VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 73984

Last change on this file since 73984 was 73984, checked in by vboxsync, 6 years ago

VMM/HM, IEM: Renamed VMX_EXIT_XDTR_ACCESS and VMX_EXIT_TR_ACCESS.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 570.7 KB
Line 
1/* $Id: HMVMXR0.cpp 73984 2018-08-31 08:30:50Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/x86.h>
25#include <iprt/asm-amd64-x86.h>
26#include <iprt/thread.h>
27
28#include <VBox/vmm/pdmapi.h>
29#include <VBox/vmm/dbgf.h>
30#include <VBox/vmm/iem.h>
31#include <VBox/vmm/iom.h>
32#include <VBox/vmm/selm.h>
33#include <VBox/vmm/tm.h>
34#include <VBox/vmm/em.h>
35#include <VBox/vmm/gim.h>
36#include <VBox/vmm/apic.h>
37#ifdef VBOX_WITH_REM
38# include <VBox/vmm/rem.h>
39#endif
40#include "HMInternal.h"
41#include <VBox/vmm/vm.h>
42#include "HMVMXR0.h"
43#include "dtrace/VBoxVMM.h"
44
45#ifdef DEBUG_ramshankar
46# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
47# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
49# define HMVMX_ALWAYS_CHECK_GUEST_STATE
50# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
51# define HMVMX_ALWAYS_TRAP_PF
52# define HMVMX_ALWAYS_FLUSH_TLB
53# define HMVMX_ALWAYS_SWAP_EFER
54#endif
55
56
57/*********************************************************************************************************************************
58* Defined Constants And Macros *
59*********************************************************************************************************************************/
60/** Use the function table. */
61#define HMVMX_USE_FUNCTION_TABLE
62
63/** Determine which tagged-TLB flush handler to use. */
64#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
65#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
66#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
67#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
68
69/** @name HMVMX_READ_XXX
70 * Flags to skip redundant reads of some common VMCS fields that are not part of
71 * the guest-CPU or VCPU state but are needed while handling VM-exits.
72 */
73#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
74#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
75#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
76#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
77#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
78#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
79#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
80/** @} */
81
82/**
83 * States of the VMCS.
84 *
85 * This does not reflect all possible VMCS states but currently only those
86 * needed for maintaining the VMCS consistently even when thread-context hooks
87 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
88 */
89#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
90#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
91#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
92
93/**
94 * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
95 * guest using hardware-assisted VMX.
96 *
97 * This excludes state like GPRs (other than RSP) which are always are
98 * swapped and restored across the world-switch and also registers like EFER,
99 * MSR which cannot be modified by the guest without causing a VM-exit.
100 */
101#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
102 | CPUMCTX_EXTRN_RFLAGS \
103 | CPUMCTX_EXTRN_RSP \
104 | CPUMCTX_EXTRN_SREG_MASK \
105 | CPUMCTX_EXTRN_TABLE_MASK \
106 | CPUMCTX_EXTRN_KERNEL_GS_BASE \
107 | CPUMCTX_EXTRN_SYSCALL_MSRS \
108 | CPUMCTX_EXTRN_SYSENTER_MSRS \
109 | CPUMCTX_EXTRN_TSC_AUX \
110 | CPUMCTX_EXTRN_OTHER_MSRS \
111 | CPUMCTX_EXTRN_CR0 \
112 | CPUMCTX_EXTRN_CR3 \
113 | CPUMCTX_EXTRN_CR4 \
114 | CPUMCTX_EXTRN_DR7 \
115 | CPUMCTX_EXTRN_HM_VMX_MASK)
116
117/**
118 * Exception bitmap mask for real-mode guests (real-on-v86).
119 *
120 * We need to intercept all exceptions manually except:
121 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
122 * due to bugs in Intel CPUs.
123 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
124 * support.
125 */
126#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
127 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
128 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
129 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
130 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
131 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
132 | RT_BIT(X86_XCPT_XF))
133
134/** Maximum VM-instruction error number. */
135#define HMVMX_INSTR_ERROR_MAX 28
136
137/** Profiling macro. */
138#ifdef HM_PROFILE_EXIT_DISPATCH
139# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
140# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
141#else
142# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
143# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
144#endif
145
146/** Assert that preemption is disabled or covered by thread-context hooks. */
147#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
148 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
149
150/** Assert that we haven't migrated CPUs when thread-context hooks are not
151 * used. */
152#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
153 || (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
154 ("Illegal migration! Entered on CPU %u Current %u\n", \
155 (a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
156
157/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
158 * context. */
159#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
160 ("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
161 (a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
162
163/** Helper macro for VM-exit handlers called unexpectedly. */
164#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
165 do { \
166 (a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
167 return VERR_VMX_UNEXPECTED_EXIT; \
168 } while (0)
169
170/** Macro for importing segment registers to the VMCS from the guest-CPU context. */
171#ifdef VMX_USE_CACHED_VMCS_ACCESSES
172# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
173 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
174 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
175#else
176# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
177 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
178 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
179#endif
180
181/** Macro for exporting segment registers to the VMCS from the guest-CPU context. */
182# define HMVMX_EXPORT_SREG(Sel, a_pCtxSelReg) \
183 hmR0VmxExportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
184 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
185
186#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
187/** Macro that does the necessary privilege checks and intercepted VM-exits for
188 * guests that attempted to execute a VMX instruction. */
189#define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
190 do \
191 { \
192 VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
193 if (rcStrictTmp == VINF_SUCCESS) \
194 { /* likely */ } \
195 else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
196 { \
197 Assert((a_pVCpu)->hm.s.Event.fPending); \
198 Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
199 return VINF_SUCCESS; \
200 } \
201 else \
202 { \
203 int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
204 AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
205 } \
206 } while (0)
207#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
208
209
210/*********************************************************************************************************************************
211* Structures and Typedefs *
212*********************************************************************************************************************************/
213/**
214 * VMX transient state.
215 *
216 * A state structure for holding miscellaneous information across
217 * VMX non-root operation and restored after the transition.
218 */
219typedef struct VMXTRANSIENT
220{
221 /** The host's rflags/eflags. */
222 RTCCUINTREG fEFlags;
223#if HC_ARCH_BITS == 32
224 uint32_t u32Alignment0;
225#endif
226 /** The guest's TPR value used for TPR shadowing. */
227 uint8_t u8GuestTpr;
228 /** Alignment. */
229 uint8_t abAlignment0[7];
230
231 /** The basic VM-exit reason. */
232 uint16_t uExitReason;
233 /** Alignment. */
234 uint16_t u16Alignment0;
235 /** The VM-exit interruption error code. */
236 uint32_t uExitIntErrorCode;
237 /** The VM-exit exit code qualification. */
238 uint64_t uExitQual;
239
240 /** The VM-exit interruption-information field. */
241 uint32_t uExitIntInfo;
242 /** The VM-exit instruction-length field. */
243 uint32_t cbInstr;
244 /** The VM-exit instruction-information field. */
245 VMXEXITINSTRINFO ExitInstrInfo;
246 /** Whether the VM-entry failed or not. */
247 bool fVMEntryFailed;
248 /** Alignment. */
249 uint8_t abAlignment1[3];
250
251 /** The VM-entry interruption-information field. */
252 uint32_t uEntryIntInfo;
253 /** The VM-entry exception error code field. */
254 uint32_t uEntryXcptErrorCode;
255 /** The VM-entry instruction length field. */
256 uint32_t cbEntryInstr;
257
258 /** IDT-vectoring information field. */
259 uint32_t uIdtVectoringInfo;
260 /** IDT-vectoring error code. */
261 uint32_t uIdtVectoringErrorCode;
262
263 /** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
264 uint32_t fVmcsFieldsRead;
265
266 /** Whether the guest debug state was active at the time of VM-exit. */
267 bool fWasGuestDebugStateActive;
268 /** Whether the hyper debug state was active at the time of VM-exit. */
269 bool fWasHyperDebugStateActive;
270 /** Whether TSC-offsetting should be setup before VM-entry. */
271 bool fUpdateTscOffsettingAndPreemptTimer;
272 /** Whether the VM-exit was caused by a page-fault during delivery of a
273 * contributory exception or a page-fault. */
274 bool fVectoringDoublePF;
275 /** Whether the VM-exit was caused by a page-fault during delivery of an
276 * external interrupt or NMI. */
277 bool fVectoringPF;
278} VMXTRANSIENT;
279AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
280AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
281AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
282AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
283AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
284/** Pointer to VMX transient state. */
285typedef VMXTRANSIENT *PVMXTRANSIENT;
286
287
288/**
289 * MSR-bitmap read permissions.
290 */
291typedef enum VMXMSREXITREAD
292{
293 /** Reading this MSR causes a VM-exit. */
294 VMXMSREXIT_INTERCEPT_READ = 0xb,
295 /** Reading this MSR does not cause a VM-exit. */
296 VMXMSREXIT_PASSTHRU_READ
297} VMXMSREXITREAD;
298/** Pointer to MSR-bitmap read permissions. */
299typedef VMXMSREXITREAD* PVMXMSREXITREAD;
300
301/**
302 * MSR-bitmap write permissions.
303 */
304typedef enum VMXMSREXITWRITE
305{
306 /** Writing to this MSR causes a VM-exit. */
307 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
308 /** Writing to this MSR does not cause a VM-exit. */
309 VMXMSREXIT_PASSTHRU_WRITE
310} VMXMSREXITWRITE;
311/** Pointer to MSR-bitmap write permissions. */
312typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
313
314
315/**
316 * VMX VM-exit handler.
317 *
318 * @returns Strict VBox status code (i.e. informational status codes too).
319 * @param pVCpu The cross context virtual CPU structure.
320 * @param pVmxTransient Pointer to the VMX-transient structure.
321 */
322#ifndef HMVMX_USE_FUNCTION_TABLE
323typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
324#else
325typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
326/** Pointer to VM-exit handler. */
327typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
328#endif
329
330/**
331 * VMX VM-exit handler, non-strict status code.
332 *
333 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
334 *
335 * @returns VBox status code, no informational status code returned.
336 * @param pVCpu The cross context virtual CPU structure.
337 * @param pVmxTransient Pointer to the VMX-transient structure.
338 *
339 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
340 * use of that status code will be replaced with VINF_EM_SOMETHING
341 * later when switching over to IEM.
342 */
343#ifndef HMVMX_USE_FUNCTION_TABLE
344typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
345#else
346typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
347#endif
348
349
350/*********************************************************************************************************************************
351* Internal Functions *
352*********************************************************************************************************************************/
353static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush);
354static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr);
355static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
356static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat);
357static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
358 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState);
359#if HC_ARCH_BITS == 32
360static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
361#endif
362#ifndef HMVMX_USE_FUNCTION_TABLE
363DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
364# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
365# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
366#else
367# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
368# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
369#endif
370
371
372/** @name VM-exit handlers.
373 * @{
374 */
375static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
376static FNVMXEXITHANDLER hmR0VmxExitExtInt;
377static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
378static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
379static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
380static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
381static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
382static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
383static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
384static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
385static FNVMXEXITHANDLER hmR0VmxExitCpuid;
386static FNVMXEXITHANDLER hmR0VmxExitGetsec;
387static FNVMXEXITHANDLER hmR0VmxExitHlt;
388static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
389static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
390static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
391static FNVMXEXITHANDLER hmR0VmxExitVmcall;
392#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
393static FNVMXEXITHANDLER hmR0VmxExitVmclear;
394static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
395static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
396static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
397static FNVMXEXITHANDLER hmR0VmxExitVmread;
398static FNVMXEXITHANDLER hmR0VmxExitVmresume;
399static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
400static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
401static FNVMXEXITHANDLER hmR0VmxExitVmxon;
402#endif
403static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
404static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
405static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
406static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
407static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
408static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
409static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
410static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
411static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
412static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
413static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
414static FNVMXEXITHANDLER hmR0VmxExitMwait;
415static FNVMXEXITHANDLER hmR0VmxExitMtf;
416static FNVMXEXITHANDLER hmR0VmxExitMonitor;
417static FNVMXEXITHANDLER hmR0VmxExitPause;
418static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
419static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
420static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
421static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
422static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
423static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
424static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
425static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
426static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
427static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
428static FNVMXEXITHANDLER hmR0VmxExitRdrand;
429static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
430/** @} */
431
432static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
433static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
434static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
435static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
436static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
437static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
438static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
439static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu);
440
441
442/*********************************************************************************************************************************
443* Global Variables *
444*********************************************************************************************************************************/
445#ifdef HMVMX_USE_FUNCTION_TABLE
446
447/**
448 * VMX_EXIT dispatch table.
449 */
450static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
451{
452 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
453 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
454 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
455 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
456 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
457 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
458 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
459 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
460 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
461 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
462 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
463 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
464 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
465 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
466 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
467 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
468 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
469 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
470 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
471#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
472 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
473 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
474 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
475 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
476 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
477 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
478 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
479 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
480 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
481#else
482 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
483 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
484 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
485 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
486 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
487 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
488 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
489 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
490 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
491#endif
492 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
493 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
494 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
495 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
496 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
497 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
498 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
499 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
500 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
501 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
502 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
503 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
504 /* 40 UNDEFINED */ hmR0VmxExitPause,
505 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
506 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
507 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
508 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
509 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
510 /* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitXdtrAccess,
511 /* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitXdtrAccess,
512 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
513 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
514 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
515 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
516 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
517 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
518 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
519 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
520 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
521 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
522 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
523 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
524 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
525 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
526 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
527 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
528 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
529};
530#endif /* HMVMX_USE_FUNCTION_TABLE */
531
532#ifdef VBOX_STRICT
533static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
534{
535 /* 0 */ "(Not Used)",
536 /* 1 */ "VMCALL executed in VMX root operation.",
537 /* 2 */ "VMCLEAR with invalid physical address.",
538 /* 3 */ "VMCLEAR with VMXON pointer.",
539 /* 4 */ "VMLAUNCH with non-clear VMCS.",
540 /* 5 */ "VMRESUME with non-launched VMCS.",
541 /* 6 */ "VMRESUME after VMXOFF",
542 /* 7 */ "VM-entry with invalid control fields.",
543 /* 8 */ "VM-entry with invalid host state fields.",
544 /* 9 */ "VMPTRLD with invalid physical address.",
545 /* 10 */ "VMPTRLD with VMXON pointer.",
546 /* 11 */ "VMPTRLD with incorrect revision identifier.",
547 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
548 /* 13 */ "VMWRITE to read-only VMCS component.",
549 /* 14 */ "(Not Used)",
550 /* 15 */ "VMXON executed in VMX root operation.",
551 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
552 /* 17 */ "VM-entry with non-launched executing VMCS.",
553 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
554 /* 19 */ "VMCALL with non-clear VMCS.",
555 /* 20 */ "VMCALL with invalid VM-exit control fields.",
556 /* 21 */ "(Not Used)",
557 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
558 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
559 /* 24 */ "VMCALL with invalid SMM-monitor features.",
560 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
561 /* 26 */ "VM-entry with events blocked by MOV SS.",
562 /* 27 */ "(Not Used)",
563 /* 28 */ "Invalid operand to INVEPT/INVVPID."
564};
565#endif /* VBOX_STRICT */
566
567
568
569/**
570 * Updates the VM's last error record.
571 *
572 * If there was a VMX instruction error, reads the error data from the VMCS and
573 * updates VCPU's last error record as well.
574 *
575 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
576 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
577 * VERR_VMX_INVALID_VMCS_FIELD.
578 * @param rc The error code.
579 */
580static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
581{
582 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
583 || rc == VERR_VMX_UNABLE_TO_START_VM)
584 {
585 AssertPtrReturnVoid(pVCpu);
586 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
587 }
588 pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
589}
590
591
592/**
593 * Reads the VM-entry interruption-information field from the VMCS into the VMX
594 * transient structure.
595 *
596 * @returns VBox status code.
597 * @param pVmxTransient Pointer to the VMX transient structure.
598 *
599 * @remarks No-long-jump zone!!!
600 */
601DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
602{
603 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
604 AssertRCReturn(rc, rc);
605 return VINF_SUCCESS;
606}
607
608#ifdef VBOX_STRICT
609/**
610 * Reads the VM-entry exception error code field from the VMCS into
611 * the VMX transient structure.
612 *
613 * @returns VBox status code.
614 * @param pVmxTransient Pointer to the VMX transient structure.
615 *
616 * @remarks No-long-jump zone!!!
617 */
618DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
619{
620 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
621 AssertRCReturn(rc, rc);
622 return VINF_SUCCESS;
623}
624
625
626/**
627 * Reads the VM-entry exception error code field from the VMCS into
628 * the VMX transient structure.
629 *
630 * @returns VBox status code.
631 * @param pVmxTransient Pointer to the VMX transient structure.
632 *
633 * @remarks No-long-jump zone!!!
634 */
635DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
636{
637 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
638 AssertRCReturn(rc, rc);
639 return VINF_SUCCESS;
640}
641#endif /* VBOX_STRICT */
642
643
644/**
645 * Reads the VM-exit interruption-information field from the VMCS into the VMX
646 * transient structure.
647 *
648 * @returns VBox status code.
649 * @param pVmxTransient Pointer to the VMX transient structure.
650 */
651DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
652{
653 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
654 {
655 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
656 AssertRCReturn(rc,rc);
657 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_INFO;
658 }
659 return VINF_SUCCESS;
660}
661
662
663/**
664 * Reads the VM-exit interruption error code from the VMCS into the VMX
665 * transient structure.
666 *
667 * @returns VBox status code.
668 * @param pVmxTransient Pointer to the VMX transient structure.
669 */
670DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
671{
672 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
673 {
674 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
675 AssertRCReturn(rc, rc);
676 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
677 }
678 return VINF_SUCCESS;
679}
680
681
682/**
683 * Reads the VM-exit instruction length field from the VMCS into the VMX
684 * transient structure.
685 *
686 * @returns VBox status code.
687 * @param pVmxTransient Pointer to the VMX transient structure.
688 */
689DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
690{
691 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
692 {
693 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
694 AssertRCReturn(rc, rc);
695 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_LEN;
696 }
697 return VINF_SUCCESS;
698}
699
700
701/**
702 * Reads the VM-exit instruction-information field from the VMCS into
703 * the VMX transient structure.
704 *
705 * @returns VBox status code.
706 * @param pVmxTransient Pointer to the VMX transient structure.
707 */
708DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
709{
710 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
711 {
712 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
713 AssertRCReturn(rc, rc);
714 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_INFO;
715 }
716 return VINF_SUCCESS;
717}
718
719
720/**
721 * Reads the exit code qualification from the VMCS into the VMX transient
722 * structure.
723 *
724 * @returns VBox status code.
725 * @param pVCpu The cross context virtual CPU structure of the
726 * calling EMT. (Required for the VMCS cache case.)
727 * @param pVmxTransient Pointer to the VMX transient structure.
728 */
729DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
730{
731 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
732 {
733 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
734 AssertRCReturn(rc, rc);
735 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION;
736 }
737 return VINF_SUCCESS;
738}
739
740
741/**
742 * Reads the IDT-vectoring information field from the VMCS into the VMX
743 * transient structure.
744 *
745 * @returns VBox status code.
746 * @param pVmxTransient Pointer to the VMX transient structure.
747 *
748 * @remarks No-long-jump zone!!!
749 */
750DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
751{
752 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
753 {
754 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
755 AssertRCReturn(rc, rc);
756 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_INFO;
757 }
758 return VINF_SUCCESS;
759}
760
761
762/**
763 * Reads the IDT-vectoring error code from the VMCS into the VMX
764 * transient structure.
765 *
766 * @returns VBox status code.
767 * @param pVmxTransient Pointer to the VMX transient structure.
768 */
769DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
770{
771 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
772 {
773 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
774 AssertRCReturn(rc, rc);
775 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
776 }
777 return VINF_SUCCESS;
778}
779
780
781/**
782 * Enters VMX root mode operation on the current CPU.
783 *
784 * @returns VBox status code.
785 * @param pVM The cross context VM structure. Can be
786 * NULL, after a resume.
787 * @param HCPhysCpuPage Physical address of the VMXON region.
788 * @param pvCpuPage Pointer to the VMXON region.
789 */
790static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
791{
792 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
793 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
794 Assert(pvCpuPage);
795 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
796
797 if (pVM)
798 {
799 /* Write the VMCS revision dword to the VMXON region. */
800 *(uint32_t *)pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
801 }
802
803 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
804 RTCCUINTREG fEFlags = ASMIntDisableFlags();
805
806 /* Enable the VMX bit in CR4 if necessary. */
807 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
808
809 /* Enter VMX root mode. */
810 int rc = VMXEnable(HCPhysCpuPage);
811 if (RT_FAILURE(rc))
812 {
813 if (!(uOldCr4 & X86_CR4_VMXE))
814 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
815
816 if (pVM)
817 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
818 }
819
820 /* Restore interrupts. */
821 ASMSetFlags(fEFlags);
822 return rc;
823}
824
825
826/**
827 * Exits VMX root mode operation on the current CPU.
828 *
829 * @returns VBox status code.
830 */
831static int hmR0VmxLeaveRootMode(void)
832{
833 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
834
835 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
836 RTCCUINTREG fEFlags = ASMIntDisableFlags();
837
838 /* If we're for some reason not in VMX root mode, then don't leave it. */
839 RTCCUINTREG uHostCR4 = ASMGetCR4();
840
841 int rc;
842 if (uHostCR4 & X86_CR4_VMXE)
843 {
844 /* Exit VMX root mode and clear the VMX bit in CR4. */
845 VMXDisable();
846 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
847 rc = VINF_SUCCESS;
848 }
849 else
850 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
851
852 /* Restore interrupts. */
853 ASMSetFlags(fEFlags);
854 return rc;
855}
856
857
858/**
859 * Allocates and maps one physically contiguous page. The allocated page is
860 * zero'd out. (Used by various VT-x structures).
861 *
862 * @returns IPRT status code.
863 * @param pMemObj Pointer to the ring-0 memory object.
864 * @param ppVirt Where to store the virtual address of the
865 * allocation.
866 * @param pHCPhys Where to store the physical address of the
867 * allocation.
868 */
869static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
870{
871 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
872 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
873 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
874
875 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
876 if (RT_FAILURE(rc))
877 return rc;
878 *ppVirt = RTR0MemObjAddress(*pMemObj);
879 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
880 ASMMemZero32(*ppVirt, PAGE_SIZE);
881 return VINF_SUCCESS;
882}
883
884
885/**
886 * Frees and unmaps an allocated physical page.
887 *
888 * @param pMemObj Pointer to the ring-0 memory object.
889 * @param ppVirt Where to re-initialize the virtual address of
890 * allocation as 0.
891 * @param pHCPhys Where to re-initialize the physical address of the
892 * allocation as 0.
893 */
894static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
895{
896 AssertPtr(pMemObj);
897 AssertPtr(ppVirt);
898 AssertPtr(pHCPhys);
899 if (*pMemObj != NIL_RTR0MEMOBJ)
900 {
901 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
902 AssertRC(rc);
903 *pMemObj = NIL_RTR0MEMOBJ;
904 *ppVirt = 0;
905 *pHCPhys = 0;
906 }
907}
908
909
910/**
911 * Worker function to free VT-x related structures.
912 *
913 * @returns IPRT status code.
914 * @param pVM The cross context VM structure.
915 */
916static void hmR0VmxStructsFree(PVM pVM)
917{
918 for (VMCPUID i = 0; i < pVM->cCpus; i++)
919 {
920 PVMCPU pVCpu = &pVM->aCpus[i];
921 AssertPtr(pVCpu);
922
923 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
924 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
925
926 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
927 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
928
929 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
930 }
931
932 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
933#ifdef VBOX_WITH_CRASHDUMP_MAGIC
934 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
935#endif
936}
937
938
939/**
940 * Worker function to allocate VT-x related VM structures.
941 *
942 * @returns IPRT status code.
943 * @param pVM The cross context VM structure.
944 */
945static int hmR0VmxStructsAlloc(PVM pVM)
946{
947 /*
948 * Initialize members up-front so we can cleanup properly on allocation failure.
949 */
950#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
951 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
952 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
953 pVM->hm.s.vmx.HCPhys##a_Name = 0;
954
955#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
956 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
957 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
958 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
959
960#ifdef VBOX_WITH_CRASHDUMP_MAGIC
961 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
962#endif
963 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
964
965 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
966 for (VMCPUID i = 0; i < pVM->cCpus; i++)
967 {
968 PVMCPU pVCpu = &pVM->aCpus[i];
969 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
970 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
971 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
972 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
973 }
974#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
975#undef VMXLOCAL_INIT_VM_MEMOBJ
976
977 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
978 AssertReturnStmt(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE) <= PAGE_SIZE,
979 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
980 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
981
982 /*
983 * Allocate all the VT-x structures.
984 */
985 int rc = VINF_SUCCESS;
986#ifdef VBOX_WITH_CRASHDUMP_MAGIC
987 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
988 if (RT_FAILURE(rc))
989 goto cleanup;
990 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
991 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
992#endif
993
994 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
995 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
996 {
997 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
998 &pVM->hm.s.vmx.HCPhysApicAccess);
999 if (RT_FAILURE(rc))
1000 goto cleanup;
1001 }
1002
1003 /*
1004 * Initialize per-VCPU VT-x structures.
1005 */
1006 for (VMCPUID i = 0; i < pVM->cCpus; i++)
1007 {
1008 PVMCPU pVCpu = &pVM->aCpus[i];
1009 AssertPtr(pVCpu);
1010
1011 /* Allocate the VM control structure (VMCS). */
1012 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1013 if (RT_FAILURE(rc))
1014 goto cleanup;
1015
1016 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1017 if ( PDMHasApic(pVM)
1018 && (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1019 {
1020 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1021 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1022 if (RT_FAILURE(rc))
1023 goto cleanup;
1024 }
1025
1026 /*
1027 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1028 * transparent accesses of specific MSRs.
1029 *
1030 * If the condition for enabling MSR bitmaps changes here, don't forget to
1031 * update HMAreMsrBitmapsAvailable().
1032 */
1033 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1034 {
1035 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1036 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1037 if (RT_FAILURE(rc))
1038 goto cleanup;
1039 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1040 }
1041
1042 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1043 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1044 if (RT_FAILURE(rc))
1045 goto cleanup;
1046
1047 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1048 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1049 if (RT_FAILURE(rc))
1050 goto cleanup;
1051 }
1052
1053 return VINF_SUCCESS;
1054
1055cleanup:
1056 hmR0VmxStructsFree(pVM);
1057 return rc;
1058}
1059
1060
1061/**
1062 * Does global VT-x initialization (called during module initialization).
1063 *
1064 * @returns VBox status code.
1065 */
1066VMMR0DECL(int) VMXR0GlobalInit(void)
1067{
1068#ifdef HMVMX_USE_FUNCTION_TABLE
1069 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1070# ifdef VBOX_STRICT
1071 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1072 Assert(g_apfnVMExitHandlers[i]);
1073# endif
1074#endif
1075 return VINF_SUCCESS;
1076}
1077
1078
1079/**
1080 * Does global VT-x termination (called during module termination).
1081 */
1082VMMR0DECL(void) VMXR0GlobalTerm()
1083{
1084 /* Nothing to do currently. */
1085}
1086
1087
1088/**
1089 * Sets up and activates VT-x on the current CPU.
1090 *
1091 * @returns VBox status code.
1092 * @param pHostCpu Pointer to the global CPU info struct.
1093 * @param pVM The cross context VM structure. Can be
1094 * NULL after a host resume operation.
1095 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1096 * fEnabledByHost is @c true).
1097 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1098 * @a fEnabledByHost is @c true).
1099 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1100 * enable VT-x on the host.
1101 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1102 */
1103VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1104 void *pvMsrs)
1105{
1106 Assert(pHostCpu);
1107 Assert(pvMsrs);
1108 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1109
1110 /* Enable VT-x if it's not already enabled by the host. */
1111 if (!fEnabledByHost)
1112 {
1113 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1114 if (RT_FAILURE(rc))
1115 return rc;
1116 }
1117
1118 /*
1119 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
1120 * using EPTPs) so we don't retain any stale guest-physical mappings which won't get
1121 * invalidated when flushing by VPID.
1122 */
1123 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1124 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1125 {
1126 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
1127 pHostCpu->fFlushAsidBeforeUse = false;
1128 }
1129 else
1130 pHostCpu->fFlushAsidBeforeUse = true;
1131
1132 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1133 ++pHostCpu->cTlbFlushes;
1134
1135 return VINF_SUCCESS;
1136}
1137
1138
1139/**
1140 * Deactivates VT-x on the current CPU.
1141 *
1142 * @returns VBox status code.
1143 * @param pHostCpu Pointer to the global CPU info struct.
1144 * @param pvCpuPage Pointer to the VMXON region.
1145 * @param HCPhysCpuPage Physical address of the VMXON region.
1146 *
1147 * @remarks This function should never be called when SUPR0EnableVTx() or
1148 * similar was used to enable VT-x on the host.
1149 */
1150VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1151{
1152 RT_NOREF3(pHostCpu, pvCpuPage, HCPhysCpuPage);
1153
1154 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1155 return hmR0VmxLeaveRootMode();
1156}
1157
1158
1159/**
1160 * Sets the permission bits for the specified MSR in the MSR bitmap.
1161 *
1162 * @param pVCpu The cross context virtual CPU structure.
1163 * @param uMsr The MSR value.
1164 * @param enmRead Whether reading this MSR causes a VM-exit.
1165 * @param enmWrite Whether writing this MSR causes a VM-exit.
1166 */
1167static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1168{
1169 int32_t iBit;
1170 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1171
1172 /*
1173 * Layout:
1174 * 0x000 - 0x3ff - Low MSR read bits
1175 * 0x400 - 0x7ff - High MSR read bits
1176 * 0x800 - 0xbff - Low MSR write bits
1177 * 0xc00 - 0xfff - High MSR write bits
1178 */
1179 if (uMsr <= 0x00001fff)
1180 iBit = uMsr;
1181 else if (uMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1182 {
1183 iBit = uMsr - UINT32_C(0xc0000000);
1184 pbMsrBitmap += 0x400;
1185 }
1186 else
1187 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1188
1189 Assert(iBit <= 0x1fff);
1190 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1191 ASMBitSet(pbMsrBitmap, iBit);
1192 else
1193 ASMBitClear(pbMsrBitmap, iBit);
1194
1195 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1196 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1197 else
1198 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1199}
1200
1201
1202#ifdef VBOX_STRICT
1203/**
1204 * Gets the permission bits for the specified MSR in the MSR bitmap.
1205 *
1206 * @returns VBox status code.
1207 * @retval VINF_SUCCESS if the specified MSR is found.
1208 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1209 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1210 *
1211 * @param pVCpu The cross context virtual CPU structure.
1212 * @param uMsr The MSR.
1213 * @param penmRead Where to store the read permissions.
1214 * @param penmWrite Where to store the write permissions.
1215 */
1216static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1217{
1218 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1219 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1220 int32_t iBit;
1221 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1222
1223 /* See hmR0VmxSetMsrPermission() for the layout. */
1224 if (uMsr <= 0x00001fff)
1225 iBit = uMsr;
1226 else if ( uMsr >= 0xc0000000
1227 && uMsr <= 0xc0001fff)
1228 {
1229 iBit = (uMsr - 0xc0000000);
1230 pbMsrBitmap += 0x400;
1231 }
1232 else
1233 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1234
1235 Assert(iBit <= 0x1fff);
1236 if (ASMBitTest(pbMsrBitmap, iBit))
1237 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1238 else
1239 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1240
1241 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1242 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1243 else
1244 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1245 return VINF_SUCCESS;
1246}
1247#endif /* VBOX_STRICT */
1248
1249
1250/**
1251 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1252 * area.
1253 *
1254 * @returns VBox status code.
1255 * @param pVCpu The cross context virtual CPU structure.
1256 * @param cMsrs The number of MSRs.
1257 */
1258static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1259{
1260 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1261 uint64_t const uVmxMiscMsr = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc;
1262 uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(uVmxMiscMsr);
1263 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1264 {
1265 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1266 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1267 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1268 }
1269
1270 /* Update number of guest MSRs to load/store across the world-switch. */
1271 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1272 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1273
1274 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1275 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1276 AssertRCReturn(rc, rc);
1277
1278 /* Update the VCPU's copy of the MSR count. */
1279 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1280
1281 return VINF_SUCCESS;
1282}
1283
1284
1285/**
1286 * Adds a new (or updates the value of an existing) guest/host MSR
1287 * pair to be swapped during the world-switch as part of the
1288 * auto-load/store MSR area in the VMCS.
1289 *
1290 * @returns VBox status code.
1291 * @param pVCpu The cross context virtual CPU structure.
1292 * @param uMsr The MSR.
1293 * @param uGuestMsrValue Value of the guest MSR.
1294 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1295 * necessary.
1296 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1297 * its value was updated. Optional, can be NULL.
1298 */
1299static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1300 bool *pfAddedAndUpdated)
1301{
1302 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1303 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1304 uint32_t i;
1305 for (i = 0; i < cMsrs; i++)
1306 {
1307 if (pGuestMsr->u32Msr == uMsr)
1308 break;
1309 pGuestMsr++;
1310 }
1311
1312 bool fAdded = false;
1313 if (i == cMsrs)
1314 {
1315 ++cMsrs;
1316 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1317 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1318
1319 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1320 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1321 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1322
1323 fAdded = true;
1324 }
1325
1326 /* Update the MSR values in the auto-load/store MSR area. */
1327 pGuestMsr->u32Msr = uMsr;
1328 pGuestMsr->u64Value = uGuestMsrValue;
1329
1330 /* Create/update the MSR slot in the host MSR area. */
1331 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1332 pHostMsr += i;
1333 pHostMsr->u32Msr = uMsr;
1334
1335 /*
1336 * Update the host MSR only when requested by the caller AND when we're
1337 * adding it to the auto-load/store area. Otherwise, it would have been
1338 * updated by hmR0VmxExportHostMsrs(). We do this for performance reasons.
1339 */
1340 bool fUpdatedMsrValue = false;
1341 if ( fAdded
1342 && fUpdateHostMsr)
1343 {
1344 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1345 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1346 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1347 fUpdatedMsrValue = true;
1348 }
1349
1350 if (pfAddedAndUpdated)
1351 *pfAddedAndUpdated = fUpdatedMsrValue;
1352 return VINF_SUCCESS;
1353}
1354
1355
1356/**
1357 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1358 * auto-load/store MSR area in the VMCS.
1359 *
1360 * @returns VBox status code.
1361 * @param pVCpu The cross context virtual CPU structure.
1362 * @param uMsr The MSR.
1363 */
1364static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1365{
1366 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1367 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1368 for (uint32_t i = 0; i < cMsrs; i++)
1369 {
1370 /* Find the MSR. */
1371 if (pGuestMsr->u32Msr == uMsr)
1372 {
1373 /* If it's the last MSR, simply reduce the count. */
1374 if (i == cMsrs - 1)
1375 {
1376 --cMsrs;
1377 break;
1378 }
1379
1380 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1381 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1382 pLastGuestMsr += cMsrs - 1;
1383 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1384 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1385
1386 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1387 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1388 pLastHostMsr += cMsrs - 1;
1389 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1390 pHostMsr->u64Value = pLastHostMsr->u64Value;
1391 --cMsrs;
1392 break;
1393 }
1394 pGuestMsr++;
1395 }
1396
1397 /* Update the VMCS if the count changed (meaning the MSR was found). */
1398 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1399 {
1400 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1401 AssertRCReturn(rc, rc);
1402
1403 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1404 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1405 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1406
1407 Log4Func(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1408 return VINF_SUCCESS;
1409 }
1410
1411 return VERR_NOT_FOUND;
1412}
1413
1414
1415/**
1416 * Checks if the specified guest MSR is part of the auto-load/store area in
1417 * the VMCS.
1418 *
1419 * @returns true if found, false otherwise.
1420 * @param pVCpu The cross context virtual CPU structure.
1421 * @param uMsr The MSR to find.
1422 */
1423static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1424{
1425 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1426 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1427
1428 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1429 {
1430 if (pGuestMsr->u32Msr == uMsr)
1431 return true;
1432 }
1433 return false;
1434}
1435
1436
1437/**
1438 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1439 *
1440 * @param pVCpu The cross context virtual CPU structure.
1441 *
1442 * @remarks No-long-jump zone!!!
1443 */
1444static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1445{
1446 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1447 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1448 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1449 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1450
1451 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1452 {
1453 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1454
1455 /*
1456 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1457 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1458 */
1459 if (pHostMsr->u32Msr == MSR_K6_EFER)
1460 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1461 else
1462 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1463 }
1464
1465 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1466}
1467
1468
1469/**
1470 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1471 * perform lazy restoration of the host MSRs while leaving VT-x.
1472 *
1473 * @param pVCpu The cross context virtual CPU structure.
1474 *
1475 * @remarks No-long-jump zone!!!
1476 */
1477static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1478{
1479 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1480
1481 /*
1482 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1483 */
1484 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1485 {
1486 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1487#if HC_ARCH_BITS == 64
1488 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1489 {
1490 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1491 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1492 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1493 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1494 }
1495#endif
1496 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1497 }
1498}
1499
1500
1501/**
1502 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1503 * lazily while leaving VT-x.
1504 *
1505 * @returns true if it does, false otherwise.
1506 * @param pVCpu The cross context virtual CPU structure.
1507 * @param uMsr The MSR to check.
1508 */
1509static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1510{
1511 NOREF(pVCpu);
1512#if HC_ARCH_BITS == 64
1513 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1514 {
1515 switch (uMsr)
1516 {
1517 case MSR_K8_LSTAR:
1518 case MSR_K6_STAR:
1519 case MSR_K8_SF_MASK:
1520 case MSR_K8_KERNEL_GS_BASE:
1521 return true;
1522 }
1523 }
1524#else
1525 RT_NOREF(pVCpu, uMsr);
1526#endif
1527 return false;
1528}
1529
1530
1531/**
1532 * Loads a set of guests MSRs to allow read/passthru to the guest.
1533 *
1534 * The name of this function is slightly confusing. This function does NOT
1535 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1536 * common prefix for functions dealing with "lazy restoration" of the shared
1537 * MSRs.
1538 *
1539 * @param pVCpu The cross context virtual CPU structure.
1540 *
1541 * @remarks No-long-jump zone!!!
1542 */
1543static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1544{
1545 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1546 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1547
1548 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1549#if HC_ARCH_BITS == 64
1550 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1551 {
1552 /*
1553 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1554 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1555 * we can skip a few MSR writes.
1556 *
1557 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1558 * guest MSR values in the guest-CPU context might be different to what's currently
1559 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1560 * CPU, see @bugref{8728}.
1561 */
1562 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1563 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1564 && pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1565 && pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1566 && pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1567 && pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1568 {
1569#ifdef VBOX_STRICT
1570 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1571 Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1572 Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
1573 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
1574#endif
1575 }
1576 else
1577 {
1578 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1579 ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
1580 ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
1581 ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
1582 }
1583 }
1584#endif
1585 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1586}
1587
1588
1589/**
1590 * Performs lazy restoration of the set of host MSRs if they were previously
1591 * loaded with guest MSR values.
1592 *
1593 * @param pVCpu The cross context virtual CPU structure.
1594 *
1595 * @remarks No-long-jump zone!!!
1596 * @remarks The guest MSRs should have been saved back into the guest-CPU
1597 * context by hmR0VmxImportGuestState()!!!
1598 */
1599static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1600{
1601 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1602 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1603
1604 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1605 {
1606 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1607#if HC_ARCH_BITS == 64
1608 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1609 {
1610 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1611 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1612 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1613 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1614 }
1615#endif
1616 }
1617 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1618}
1619
1620
1621/**
1622 * Verifies that our cached values of the VMCS fields are all consistent with
1623 * what's actually present in the VMCS.
1624 *
1625 * @returns VBox status code.
1626 * @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
1627 * @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
1628 * VMCS content. HMCPU error-field is
1629 * updated, see VMX_VCI_XXX.
1630 * @param pVCpu The cross context virtual CPU structure.
1631 */
1632static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1633{
1634 uint32_t u32Val;
1635 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1636 AssertRCReturn(rc, rc);
1637 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32EntryCtls == u32Val,
1638 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1639 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
1640 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1641
1642 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1643 AssertRCReturn(rc, rc);
1644 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ExitCtls == u32Val,
1645 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1646 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
1647 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1648
1649 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1650 AssertRCReturn(rc, rc);
1651 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32PinCtls == u32Val,
1652 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1653 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
1654 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1655
1656 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1657 AssertRCReturn(rc, rc);
1658 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls == u32Val,
1659 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1660 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
1661 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1662
1663 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
1664 {
1665 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1666 AssertRCReturn(rc, rc);
1667 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1668 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1669 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
1670 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1671 }
1672
1673 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
1674 AssertRCReturn(rc, rc);
1675 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32XcptBitmap == u32Val,
1676 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap, u32Val),
1677 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
1678 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1679
1680 uint64_t u64Val;
1681 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
1682 AssertRCReturn(rc, rc);
1683 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u64TscOffset == u64Val,
1684 ("Cache=%#RX64 VMCS=%#RX64\n", pVCpu->hm.s.vmx.u64TscOffset, u64Val),
1685 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
1686 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1687
1688 return VINF_SUCCESS;
1689}
1690
1691
1692#ifdef VBOX_STRICT
1693/**
1694 * Verifies that our cached host EFER value has not changed
1695 * since we cached it.
1696 *
1697 * @param pVCpu The cross context virtual CPU structure.
1698 */
1699static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1700{
1701 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1702
1703 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
1704 {
1705 uint64_t u64Val;
1706 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1707 AssertRC(rc);
1708
1709 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1710 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1711 }
1712}
1713
1714
1715/**
1716 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1717 * VMCS are correct.
1718 *
1719 * @param pVCpu The cross context virtual CPU structure.
1720 */
1721static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1722{
1723 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1724
1725 /* Verify MSR counts in the VMCS are what we think it should be. */
1726 uint32_t cMsrs;
1727 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1728 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1729
1730 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1731 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1732
1733 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1734 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1735
1736 PCVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1737 PCVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1738 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1739 {
1740 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1741 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1742 pGuestMsr->u32Msr, cMsrs));
1743
1744 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1745 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1746 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1747
1748 /* Verify that the permissions are as expected in the MSR bitmap. */
1749 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1750 {
1751 VMXMSREXITREAD enmRead;
1752 VMXMSREXITWRITE enmWrite;
1753 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1754 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1755 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1756 {
1757 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1758 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1759 }
1760 else
1761 {
1762 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1763 pGuestMsr->u32Msr, cMsrs));
1764 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1765 pGuestMsr->u32Msr, cMsrs));
1766 }
1767 }
1768 }
1769}
1770#endif /* VBOX_STRICT */
1771
1772
1773/**
1774 * Flushes the TLB using EPT.
1775 *
1776 * @returns VBox status code.
1777 * @param pVCpu The cross context virtual CPU structure of the calling
1778 * EMT. Can be NULL depending on @a enmTlbFlush.
1779 * @param enmTlbFlush Type of flush.
1780 *
1781 * @remarks Caller is responsible for making sure this function is called only
1782 * when NestedPaging is supported and providing @a enmTlbFlush that is
1783 * supported by the CPU.
1784 * @remarks Can be called with interrupts disabled.
1785 */
1786static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush)
1787{
1788 uint64_t au64Descriptor[2];
1789 if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
1790 au64Descriptor[0] = 0;
1791 else
1792 {
1793 Assert(pVCpu);
1794 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1795 }
1796 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1797
1798 int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
1799 AssertMsg(rc == VINF_SUCCESS,
1800 ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0, rc));
1801
1802 if ( RT_SUCCESS(rc)
1803 && pVCpu)
1804 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1805}
1806
1807
1808/**
1809 * Flushes the TLB using VPID.
1810 *
1811 * @returns VBox status code.
1812 * @param pVCpu The cross context virtual CPU structure of the calling
1813 * EMT. Can be NULL depending on @a enmTlbFlush.
1814 * @param enmTlbFlush Type of flush.
1815 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1816 * on @a enmTlbFlush).
1817 *
1818 * @remarks Can be called with interrupts disabled.
1819 */
1820static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
1821{
1822 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
1823
1824 uint64_t au64Descriptor[2];
1825 if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
1826 {
1827 au64Descriptor[0] = 0;
1828 au64Descriptor[1] = 0;
1829 }
1830 else
1831 {
1832 AssertPtr(pVCpu);
1833 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1834 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1835 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1836 au64Descriptor[1] = GCPtr;
1837 }
1838
1839 int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
1840 AssertMsg(rc == VINF_SUCCESS,
1841 ("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1842
1843 if ( RT_SUCCESS(rc)
1844 && pVCpu)
1845 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1846 NOREF(rc);
1847}
1848
1849
1850/**
1851 * Invalidates a guest page by guest virtual address. Only relevant for
1852 * EPT/VPID, otherwise there is nothing really to invalidate.
1853 *
1854 * @returns VBox status code.
1855 * @param pVCpu The cross context virtual CPU structure.
1856 * @param GCVirt Guest virtual address of the page to invalidate.
1857 */
1858VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
1859{
1860 AssertPtr(pVCpu);
1861 LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
1862
1863 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1864 if (!fFlushPending)
1865 {
1866 /*
1867 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1868 * the EPT case. See @bugref{6043} and @bugref{6177}.
1869 *
1870 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1871 * as this function maybe called in a loop with individual addresses.
1872 */
1873 PVM pVM = pVCpu->CTX_SUFF(pVM);
1874 if (pVM->hm.s.vmx.fVpid)
1875 {
1876 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1877
1878#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1879 /*
1880 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1881 * where executing INVVPID outside 64-bit mode does not flush translations of
1882 * 64-bit linear addresses, see @bugref{6208#c72}.
1883 */
1884 if (RT_HI_U32(GCVirt))
1885 fVpidFlush = false;
1886#endif
1887
1888 if (fVpidFlush)
1889 {
1890 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
1891 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1892 }
1893 else
1894 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1895 }
1896 else if (pVM->hm.s.fNestedPaging)
1897 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1898 }
1899
1900 return VINF_SUCCESS;
1901}
1902
1903
1904/**
1905 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1906 * case where neither EPT nor VPID is supported by the CPU.
1907 *
1908 * @param pVCpu The cross context virtual CPU structure.
1909 * @param pCpu Pointer to the global HM struct.
1910 *
1911 * @remarks Called with interrupts disabled.
1912 */
1913static void hmR0VmxFlushTaggedTlbNone(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1914{
1915 AssertPtr(pVCpu);
1916 AssertPtr(pCpu);
1917
1918 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1919
1920 Assert(pCpu->idCpu != NIL_RTCPUID);
1921 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1922 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1923 pVCpu->hm.s.fForceTLBFlush = false;
1924 return;
1925}
1926
1927
1928/**
1929 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1930 *
1931 * @param pVCpu The cross context virtual CPU structure.
1932 * @param pCpu Pointer to the global HM CPU struct.
1933 *
1934 * @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
1935 * nomenclature. The reason is, to avoid confusion in compare statements
1936 * since the host-CPU copies are named "ASID".
1937 *
1938 * @remarks Called with interrupts disabled.
1939 */
1940static void hmR0VmxFlushTaggedTlbBoth(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1941{
1942#ifdef VBOX_WITH_STATISTICS
1943 bool fTlbFlushed = false;
1944# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1945# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1946 if (!fTlbFlushed) \
1947 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1948 } while (0)
1949#else
1950# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1951# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1952#endif
1953
1954 AssertPtr(pCpu);
1955 AssertPtr(pVCpu);
1956 Assert(pCpu->idCpu != NIL_RTCPUID);
1957
1958 PVM pVM = pVCpu->CTX_SUFF(pVM);
1959 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1960 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1961 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1962
1963 /*
1964 * Force a TLB flush for the first world-switch if the current CPU differs from the one we
1965 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1966 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1967 * cannot reuse the current ASID anymore.
1968 */
1969 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1970 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1971 {
1972 ++pCpu->uCurrentAsid;
1973 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1974 {
1975 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1976 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1977 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1978 }
1979
1980 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1981 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1982 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1983
1984 /*
1985 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1986 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1987 */
1988 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
1989 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1990 HMVMX_SET_TAGGED_TLB_FLUSHED();
1991 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1992 }
1993 else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
1994 {
1995 /*
1996 * Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
1997 * creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
1998 * tables when EPT is in use. Flushing-by-VPID will only flush linear (only
1999 * VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2000 * mappings, see @bugref{6568}.
2001 *
2002 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2003 */
2004 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
2005 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2006 HMVMX_SET_TAGGED_TLB_FLUSHED();
2007 }
2008
2009 pVCpu->hm.s.fForceTLBFlush = false;
2010 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2011
2012 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2013 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2014 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2015 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2016 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2017 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2018 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2019 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2020 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2021
2022 /* Update VMCS with the VPID. */
2023 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2024 AssertRC(rc);
2025
2026#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2027}
2028
2029
2030/**
2031 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2032 *
2033 * @returns VBox status code.
2034 * @param pVCpu The cross context virtual CPU structure.
2035 * @param pCpu Pointer to the global HM CPU struct.
2036 *
2037 * @remarks Called with interrupts disabled.
2038 */
2039static void hmR0VmxFlushTaggedTlbEpt(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2040{
2041 AssertPtr(pVCpu);
2042 AssertPtr(pCpu);
2043 Assert(pCpu->idCpu != NIL_RTCPUID);
2044 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2045 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2046
2047 /*
2048 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2049 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2050 */
2051 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2052 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2053 {
2054 pVCpu->hm.s.fForceTLBFlush = true;
2055 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2056 }
2057
2058 /* Check for explicit TLB flushes. */
2059 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2060 {
2061 pVCpu->hm.s.fForceTLBFlush = true;
2062 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2063 }
2064
2065 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2066 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2067
2068 if (pVCpu->hm.s.fForceTLBFlush)
2069 {
2070 hmR0VmxFlushEpt(pVCpu, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2071 pVCpu->hm.s.fForceTLBFlush = false;
2072 }
2073}
2074
2075
2076/**
2077 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2078 *
2079 * @returns VBox status code.
2080 * @param pVCpu The cross context virtual CPU structure.
2081 * @param pCpu Pointer to the global HM CPU struct.
2082 *
2083 * @remarks Called with interrupts disabled.
2084 */
2085static void hmR0VmxFlushTaggedTlbVpid(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2086{
2087 AssertPtr(pVCpu);
2088 AssertPtr(pCpu);
2089 Assert(pCpu->idCpu != NIL_RTCPUID);
2090 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2091 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2092
2093 /*
2094 * Force a TLB flush for the first world switch if the current CPU differs from the one we
2095 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2096 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2097 * cannot reuse the current ASID anymore.
2098 */
2099 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2100 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2101 {
2102 pVCpu->hm.s.fForceTLBFlush = true;
2103 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2104 }
2105
2106 /* Check for explicit TLB flushes. */
2107 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2108 {
2109 /*
2110 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2111 * hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2112 * fExplicitFlush = true here and change the pCpu->fFlushAsidBeforeUse check below to
2113 * include fExplicitFlush's too) - an obscure corner case.
2114 */
2115 pVCpu->hm.s.fForceTLBFlush = true;
2116 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2117 }
2118
2119 PVM pVM = pVCpu->CTX_SUFF(pVM);
2120 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2121 if (pVCpu->hm.s.fForceTLBFlush)
2122 {
2123 ++pCpu->uCurrentAsid;
2124 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2125 {
2126 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2127 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2128 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2129 }
2130
2131 pVCpu->hm.s.fForceTLBFlush = false;
2132 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2133 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2134 if (pCpu->fFlushAsidBeforeUse)
2135 {
2136 if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2137 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2138 else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2139 {
2140 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2141 pCpu->fFlushAsidBeforeUse = false;
2142 }
2143 else
2144 {
2145 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2146 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2147 }
2148 }
2149 }
2150
2151 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2152 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2153 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2154 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2155 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2156 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2157 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2158
2159 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2160 AssertRC(rc);
2161}
2162
2163
2164/**
2165 * Flushes the guest TLB entry based on CPU capabilities.
2166 *
2167 * @param pVCpu The cross context virtual CPU structure.
2168 * @param pCpu Pointer to the global HM CPU struct.
2169 */
2170DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2171{
2172#ifdef HMVMX_ALWAYS_FLUSH_TLB
2173 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2174#endif
2175 PVM pVM = pVCpu->CTX_SUFF(pVM);
2176 switch (pVM->hm.s.vmx.enmTlbFlushType)
2177 {
2178 case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVCpu, pCpu); break;
2179 case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pVCpu, pCpu); break;
2180 case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pVCpu, pCpu); break;
2181 case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pVCpu, pCpu); break;
2182 default:
2183 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2184 break;
2185 }
2186 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2187}
2188
2189
2190/**
2191 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2192 * TLB entries from the host TLB before VM-entry.
2193 *
2194 * @returns VBox status code.
2195 * @param pVM The cross context VM structure.
2196 */
2197static int hmR0VmxSetupTaggedTlb(PVM pVM)
2198{
2199 /*
2200 * Determine optimal flush type for Nested Paging.
2201 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2202 * guest execution (see hmR3InitFinalizeR0()).
2203 */
2204 if (pVM->hm.s.fNestedPaging)
2205 {
2206 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2207 {
2208 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2209 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2210 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2211 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2212 else
2213 {
2214 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2215 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2216 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2217 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2218 }
2219
2220 /* Make sure the write-back cacheable memory type for EPT is supported. */
2221 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2222 {
2223 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2224 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2225 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2226 }
2227
2228 /* EPT requires a page-walk length of 4. */
2229 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2230 {
2231 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2232 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2233 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2234 }
2235 }
2236 else
2237 {
2238 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2239 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2240 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2241 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2242 }
2243 }
2244
2245 /*
2246 * Determine optimal flush type for VPID.
2247 */
2248 if (pVM->hm.s.vmx.fVpid)
2249 {
2250 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2251 {
2252 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2253 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2254 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2255 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2256 else
2257 {
2258 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2259 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2260 LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2261 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2262 LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2263 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2264 pVM->hm.s.vmx.fVpid = false;
2265 }
2266 }
2267 else
2268 {
2269 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2270 Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2271 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2272 pVM->hm.s.vmx.fVpid = false;
2273 }
2274 }
2275
2276 /*
2277 * Setup the handler for flushing tagged-TLBs.
2278 */
2279 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2280 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2281 else if (pVM->hm.s.fNestedPaging)
2282 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2283 else if (pVM->hm.s.vmx.fVpid)
2284 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2285 else
2286 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2287 return VINF_SUCCESS;
2288}
2289
2290
2291/**
2292 * Sets up pin-based VM-execution controls in the VMCS.
2293 *
2294 * @returns VBox status code.
2295 * @param pVCpu The cross context virtual CPU structure.
2296 *
2297 * @remarks We don't really care about optimizing vmwrites here as it's done only
2298 * once per VM and hence we don't care about VMCS-field cache comparisons.
2299 */
2300static int hmR0VmxSetupPinCtls(PVMCPU pVCpu)
2301{
2302 PVM pVM = pVCpu->CTX_SUFF(pVM);
2303 uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.disallowed0; /* Bits set here must always be set. */
2304 uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2305
2306 fVal |= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2307 | VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2308
2309 if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2310 fVal |= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2311
2312 /* Enable the VMX preemption timer. */
2313 if (pVM->hm.s.vmx.fUsePreemptTimer)
2314 {
2315 Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2316 fVal |= VMX_PIN_CTLS_PREEMPT_TIMER;
2317 }
2318
2319#if 0
2320 /* Enable posted-interrupt processing. */
2321 if (pVM->hm.s.fPostedIntrs)
2322 {
2323 Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2324 Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2325 fVal |= VMX_PIN_CTL_POSTED_INT;
2326 }
2327#endif
2328
2329 if ((fVal & fZap) != fVal)
2330 {
2331 LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2332 pVM->hm.s.vmx.Msrs.PinCtls.n.disallowed0, fVal, fZap));
2333 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2334 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2335 }
2336
2337 /* Commit it to the VMCS and update our cache. */
2338 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2339 AssertRCReturn(rc, rc);
2340 pVCpu->hm.s.vmx.u32PinCtls = fVal;
2341
2342 return VINF_SUCCESS;
2343}
2344
2345
2346/**
2347 * Sets up secondary processor-based VM-execution controls in the VMCS.
2348 *
2349 * @returns VBox status code.
2350 * @param pVCpu The cross context virtual CPU structure.
2351 *
2352 * @remarks We don't really care about optimizing vmwrites here as it's done only
2353 * once per VM and hence we don't care about VMCS-field cache comparisons.
2354 */
2355static int hmR0VmxSetupProcCtls2(PVMCPU pVCpu)
2356{
2357 PVM pVM = pVCpu->CTX_SUFF(pVM);
2358 uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2359 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2360
2361 /* WBINVD causes a VM-exit. */
2362 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2363 fVal |= VMX_PROC_CTLS2_WBINVD_EXIT;
2364
2365 /* Enable EPT (aka nested-paging). */
2366 if (pVM->hm.s.fNestedPaging)
2367 fVal |= VMX_PROC_CTLS2_EPT;
2368
2369 /*
2370 * Enable the INVPCID instruction if supported by the hardware and we expose
2371 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2372 */
2373 if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID)
2374 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2375 fVal |= VMX_PROC_CTLS2_INVPCID;
2376
2377 /* Enable VPID. */
2378 if (pVM->hm.s.vmx.fVpid)
2379 fVal |= VMX_PROC_CTLS2_VPID;
2380
2381 /* Enable Unrestricted guest execution. */
2382 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2383 fVal |= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
2384
2385#if 0
2386 if (pVM->hm.s.fVirtApicRegs)
2387 {
2388 /* Enable APIC-register virtualization. */
2389 Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
2390 fVal |= VMX_PROC_CTLS2_APIC_REG_VIRT;
2391
2392 /* Enable virtual-interrupt delivery. */
2393 Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
2394 fVal |= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
2395 }
2396#endif
2397
2398 /* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is where the TPR shadow resides. */
2399 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2400 * done dynamically. */
2401 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
2402 {
2403 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2404 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2405 fVal |= VMX_PROC_CTLS2_VIRT_APIC_ACCESS; /* Virtualize APIC accesses. */
2406 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2407 AssertRCReturn(rc, rc);
2408 }
2409
2410 /* Enable RDTSCP. */
2411 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP)
2412 fVal |= VMX_PROC_CTLS2_RDTSCP;
2413
2414 /* Enable Pause-Loop exiting. */
2415 if ( pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT
2416 && pVM->hm.s.vmx.cPleGapTicks
2417 && pVM->hm.s.vmx.cPleWindowTicks)
2418 {
2419 fVal |= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
2420
2421 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2422 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2423 AssertRCReturn(rc, rc);
2424 }
2425
2426 if ((fVal & fZap) != fVal)
2427 {
2428 LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2429 pVM->hm.s.vmx.Msrs.ProcCtls2.n.disallowed0, fVal, fZap));
2430 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2431 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2432 }
2433
2434 /* Commit it to the VMCS and update our cache. */
2435 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2436 AssertRCReturn(rc, rc);
2437 pVCpu->hm.s.vmx.u32ProcCtls2 = fVal;
2438
2439 return VINF_SUCCESS;
2440}
2441
2442
2443/**
2444 * Sets up processor-based VM-execution controls in the VMCS.
2445 *
2446 * @returns VBox status code.
2447 * @param pVCpu The cross context virtual CPU structure.
2448 *
2449 * @remarks We don't really care about optimizing vmwrites here as it's done only
2450 * once per VM and hence we don't care about VMCS-field cache comparisons.
2451 */
2452static int hmR0VmxSetupProcCtls(PVMCPU pVCpu)
2453{
2454 PVM pVM = pVCpu->CTX_SUFF(pVM);
2455 uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2456 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2457
2458 fVal |= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
2459 | VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2460 | VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2461 | VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2462 | VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2463 | VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2464 | VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2465
2466 /* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2467 if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
2468 || (pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
2469 {
2470 LogRelFunc(("Unsupported VMX_PROC_CTLS_MOV_DR_EXIT combo!"));
2471 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2472 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2473 }
2474
2475 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2476 if (!pVM->hm.s.fNestedPaging)
2477 {
2478 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2479 fVal |= VMX_PROC_CTLS_INVLPG_EXIT
2480 | VMX_PROC_CTLS_CR3_LOAD_EXIT
2481 | VMX_PROC_CTLS_CR3_STORE_EXIT;
2482 }
2483
2484 /* Use TPR shadowing if supported by the CPU. */
2485 if ( PDMHasApic(pVM)
2486 && pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
2487 {
2488 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2489 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2490 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2491 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2492 AssertRCReturn(rc, rc);
2493
2494 fVal |= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2495 /* CR8 writes cause a VM-exit based on TPR threshold. */
2496 Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
2497 Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
2498 }
2499 else
2500 {
2501 /*
2502 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2503 * Set this control only for 64-bit guests.
2504 */
2505 if (pVM->hm.s.fAllow64BitGuests)
2506 {
2507 fVal |= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2508 | VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2509 }
2510 }
2511
2512 /* Use MSR-bitmaps if supported by the CPU. */
2513 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2514 {
2515 fVal |= VMX_PROC_CTLS_USE_MSR_BITMAPS;
2516
2517 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2518 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2519 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2520 AssertRCReturn(rc, rc);
2521
2522 /*
2523 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2524 * automatically using dedicated fields in the VMCS.
2525 */
2526 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2527 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2528 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2529 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2530 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2531#if HC_ARCH_BITS == 64
2532 /*
2533 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2534 */
2535 if (pVM->hm.s.fAllow64BitGuests)
2536 {
2537 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2538 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2539 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2540 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2541 }
2542#endif
2543 /*
2544 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2545 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2546 */
2547 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2548 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2549
2550 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2551 }
2552
2553 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2554 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2555 fVal |= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
2556
2557 if ((fVal & fZap) != fVal)
2558 {
2559 LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2560 pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0, fVal, fZap));
2561 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2562 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2563 }
2564
2565 /* Commit it to the VMCS and update our cache. */
2566 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2567 AssertRCReturn(rc, rc);
2568 pVCpu->hm.s.vmx.u32ProcCtls = fVal;
2569
2570 /* Set up secondary processor-based VM-execution controls if the CPU supports it. */
2571 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2572 return hmR0VmxSetupProcCtls2(pVCpu);
2573
2574 /* Sanity check, should not really happen. */
2575 if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2576 {
2577 LogRelFunc(("Unrestricted Guest enabled when secondary processor-based VM-execution controls not available\n"));
2578 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2579 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2580 }
2581
2582 /* Old CPUs without secondary processor-based VM-execution controls would end up here. */
2583 return VINF_SUCCESS;
2584}
2585
2586
2587/**
2588 * Sets up miscellaneous (everything other than Pin & Processor-based
2589 * VM-execution) control fields in the VMCS.
2590 *
2591 * @returns VBox status code.
2592 * @param pVCpu The cross context virtual CPU structure.
2593 */
2594static int hmR0VmxSetupMiscCtls(PVMCPU pVCpu)
2595{
2596 AssertPtr(pVCpu);
2597
2598 int rc = VERR_GENERAL_FAILURE;
2599
2600 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2601#if 0
2602 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxExportGuestCR3AndCR4())*/
2603 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2604 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2605
2606 /*
2607 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2608 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2609 * We thus use the exception bitmap to control it rather than use both.
2610 */
2611 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2612 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2613
2614 /* All IO & IOIO instructions cause VM-exits. */
2615 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2616 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2617
2618 /* Initialize the MSR-bitmap area. */
2619 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2620 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2621 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2622 AssertRCReturn(rc, rc);
2623#endif
2624
2625 /* Setup MSR auto-load/store area. */
2626 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2627 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2628 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2629 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2630 AssertRCReturn(rc, rc);
2631
2632 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2633 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2634 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2635 AssertRCReturn(rc, rc);
2636
2637 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2638 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2639 AssertRCReturn(rc, rc);
2640
2641 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2642#if 0
2643 /* Setup debug controls */
2644 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2645 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2646 AssertRCReturn(rc, rc);
2647#endif
2648
2649 return rc;
2650}
2651
2652
2653/**
2654 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2655 *
2656 * We shall setup those exception intercepts that don't change during the
2657 * lifetime of the VM here. The rest are done dynamically while loading the
2658 * guest state.
2659 *
2660 * @returns VBox status code.
2661 * @param pVCpu The cross context virtual CPU structure.
2662 */
2663static int hmR0VmxInitXcptBitmap(PVMCPU pVCpu)
2664{
2665 AssertPtr(pVCpu);
2666
2667 uint32_t uXcptBitmap;
2668
2669 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2670 uXcptBitmap = RT_BIT_32(X86_XCPT_AC);
2671
2672 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2673 and writes, and because recursive #DBs can cause the CPU hang, we must always
2674 intercept #DB. */
2675 uXcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2676
2677 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2678 if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
2679 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
2680
2681 /* Commit it to the VMCS. */
2682 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
2683 AssertRCReturn(rc, rc);
2684
2685 /* Update our cache of the exception bitmap. */
2686 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
2687 return VINF_SUCCESS;
2688}
2689
2690
2691/**
2692 * Does per-VM VT-x initialization.
2693 *
2694 * @returns VBox status code.
2695 * @param pVM The cross context VM structure.
2696 */
2697VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2698{
2699 LogFlowFunc(("pVM=%p\n", pVM));
2700
2701 int rc = hmR0VmxStructsAlloc(pVM);
2702 if (RT_FAILURE(rc))
2703 {
2704 LogRelFunc(("hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2705 return rc;
2706 }
2707
2708 return VINF_SUCCESS;
2709}
2710
2711
2712/**
2713 * Does per-VM VT-x termination.
2714 *
2715 * @returns VBox status code.
2716 * @param pVM The cross context VM structure.
2717 */
2718VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2719{
2720 LogFlowFunc(("pVM=%p\n", pVM));
2721
2722#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2723 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2724 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2725#endif
2726 hmR0VmxStructsFree(pVM);
2727 return VINF_SUCCESS;
2728}
2729
2730
2731/**
2732 * Sets up the VM for execution under VT-x.
2733 * This function is only called once per-VM during initialization.
2734 *
2735 * @returns VBox status code.
2736 * @param pVM The cross context VM structure.
2737 */
2738VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2739{
2740 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2741 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2742
2743 LogFlowFunc(("pVM=%p\n", pVM));
2744
2745 /*
2746 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be
2747 * allocated. We no longer support the highly unlikely case of UnrestrictedGuest without
2748 * pRealModeTSS, see hmR3InitFinalizeR0Intel().
2749 */
2750 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2751 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2752 || !pVM->hm.s.vmx.pRealModeTSS))
2753 {
2754 LogRelFunc(("Invalid real-on-v86 state.\n"));
2755 return VERR_INTERNAL_ERROR;
2756 }
2757
2758 /* Initialize these always, see hmR3InitFinalizeR0().*/
2759 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
2760 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
2761
2762 /* Setup the tagged-TLB flush handlers. */
2763 int rc = hmR0VmxSetupTaggedTlb(pVM);
2764 if (RT_FAILURE(rc))
2765 {
2766 LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2767 return rc;
2768 }
2769
2770 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2771 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2772#if HC_ARCH_BITS == 64
2773 if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
2774 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2775 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
2776 {
2777 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2778 }
2779#endif
2780
2781 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2782 RTCCUINTREG const uHostCR4 = ASMGetCR4();
2783 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2784 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2785
2786 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2787 {
2788 PVMCPU pVCpu = &pVM->aCpus[i];
2789 AssertPtr(pVCpu);
2790 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2791
2792 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2793 Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2794
2795 /* Set revision dword at the beginning of the VMCS structure. */
2796 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
2797
2798 /* Set the VMCS launch state to "clear", see Intel spec. 31.6 "Preparation and launch a virtual machine". */
2799 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2800 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc\n", rc),
2801 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2802
2803 /* Load this VMCS as the current VMCS. */
2804 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2805 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc\n", rc),
2806 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2807
2808 rc = hmR0VmxSetupPinCtls(pVCpu);
2809 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc\n", rc),
2810 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2811
2812 rc = hmR0VmxSetupProcCtls(pVCpu);
2813 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc\n", rc),
2814 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2815
2816 rc = hmR0VmxSetupMiscCtls(pVCpu);
2817 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc\n", rc),
2818 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2819
2820 rc = hmR0VmxInitXcptBitmap(pVCpu);
2821 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc\n", rc),
2822 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2823
2824#if HC_ARCH_BITS == 32
2825 rc = hmR0VmxInitVmcsReadCache(pVCpu);
2826 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc\n", rc),
2827 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2828#endif
2829
2830 /* Sync any CPU internal VMCS data back into our VMCS in memory. */
2831 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2832 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc\n", rc),
2833 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2834
2835 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2836
2837 hmR0VmxUpdateErrorRecord(pVCpu, rc);
2838 }
2839
2840 return VINF_SUCCESS;
2841}
2842
2843
2844/**
2845 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2846 * the VMCS.
2847 *
2848 * @returns VBox status code.
2849 */
2850static int hmR0VmxExportHostControlRegs(void)
2851{
2852 RTCCUINTREG uReg = ASMGetCR0();
2853 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2854 AssertRCReturn(rc, rc);
2855
2856 uReg = ASMGetCR3();
2857 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2858 AssertRCReturn(rc, rc);
2859
2860 uReg = ASMGetCR4();
2861 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2862 AssertRCReturn(rc, rc);
2863 return rc;
2864}
2865
2866
2867/**
2868 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2869 * the host-state area in the VMCS.
2870 *
2871 * @returns VBox status code.
2872 * @param pVCpu The cross context virtual CPU structure.
2873 */
2874static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
2875{
2876#if HC_ARCH_BITS == 64
2877/**
2878 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2879 * requirements. See hmR0VmxExportHostSegmentRegs().
2880 */
2881# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2882 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2883 { \
2884 bool fValidSelector = true; \
2885 if ((selValue) & X86_SEL_LDT) \
2886 { \
2887 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2888 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2889 } \
2890 if (fValidSelector) \
2891 { \
2892 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2893 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2894 } \
2895 (selValue) = 0; \
2896 }
2897
2898 /*
2899 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2900 * should -not- save the messed up state without restoring the original host-state,
2901 * see @bugref{7240}.
2902 *
2903 * This apparently can happen (most likely the FPU changes), deal with it rather than
2904 * asserting. Was observed booting Solaris 10u10 32-bit guest.
2905 */
2906 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2907 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2908 {
2909 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2910 pVCpu->idCpu));
2911 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2912 }
2913 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2914#else
2915 RT_NOREF(pVCpu);
2916#endif
2917
2918 /*
2919 * Host DS, ES, FS and GS segment registers.
2920 */
2921#if HC_ARCH_BITS == 64
2922 RTSEL uSelDS = ASMGetDS();
2923 RTSEL uSelES = ASMGetES();
2924 RTSEL uSelFS = ASMGetFS();
2925 RTSEL uSelGS = ASMGetGS();
2926#else
2927 RTSEL uSelDS = 0;
2928 RTSEL uSelES = 0;
2929 RTSEL uSelFS = 0;
2930 RTSEL uSelGS = 0;
2931#endif
2932
2933 /*
2934 * Host CS and SS segment registers.
2935 */
2936 RTSEL uSelCS = ASMGetCS();
2937 RTSEL uSelSS = ASMGetSS();
2938
2939 /*
2940 * Host TR segment register.
2941 */
2942 RTSEL uSelTR = ASMGetTR();
2943
2944#if HC_ARCH_BITS == 64
2945 /*
2946 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
2947 * gain VM-entry and restore them before we get preempted.
2948 *
2949 * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2950 */
2951 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2952 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2953 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2954 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2955# undef VMXLOCAL_ADJUST_HOST_SEG
2956#endif
2957
2958 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2959 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2960 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2961 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2962 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2963 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2964 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2965 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2966 Assert(uSelCS);
2967 Assert(uSelTR);
2968
2969 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2970#if 0
2971 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE))
2972 Assert(uSelSS != 0);
2973#endif
2974
2975 /* Write these host selector fields into the host-state area in the VMCS. */
2976 int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2977 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2978#if HC_ARCH_BITS == 64
2979 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2980 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2981 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2982 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2983#else
2984 NOREF(uSelDS);
2985 NOREF(uSelES);
2986 NOREF(uSelFS);
2987 NOREF(uSelGS);
2988#endif
2989 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2990 AssertRCReturn(rc, rc);
2991
2992 /*
2993 * Host GDTR and IDTR.
2994 */
2995 RTGDTR Gdtr;
2996 RTIDTR Idtr;
2997 RT_ZERO(Gdtr);
2998 RT_ZERO(Idtr);
2999 ASMGetGDTR(&Gdtr);
3000 ASMGetIDTR(&Idtr);
3001 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3002 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3003 AssertRCReturn(rc, rc);
3004
3005#if HC_ARCH_BITS == 64
3006 /*
3007 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
3008 * them to the maximum limit (0xffff) on every VM-exit.
3009 */
3010 if (Gdtr.cbGdt != 0xffff)
3011 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3012
3013 /*
3014 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
3015 * Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
3016 * as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
3017 * However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
3018 * uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
3019 * alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
3020 * at 0xffff on hosts where we are sure it won't cause trouble.
3021 */
3022# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3023 if (Idtr.cbIdt < 0x0fff)
3024# else
3025 if (Idtr.cbIdt != 0xffff)
3026# endif
3027 {
3028 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3029 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3030 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3031 }
3032#endif
3033
3034 /*
3035 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3036 * and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3037 * RPL should be too in most cases.
3038 */
3039 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3040 ("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3041
3042 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3043#if HC_ARCH_BITS == 64
3044 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3045
3046 /*
3047 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3048 * all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3049 * restoration if the host has something else. Task switching is not supported in 64-bit
3050 * mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3051 * limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3052 *
3053 * [1] See Intel spec. 3.5 "System Descriptor Types".
3054 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3055 */
3056 PVM pVM = pVCpu->CTX_SUFF(pVM);
3057 Assert(pDesc->System.u4Type == 11);
3058 if ( pDesc->System.u16LimitLow != 0x67
3059 || pDesc->System.u4LimitHigh)
3060 {
3061 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3062 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3063 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3064 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3065 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3066 }
3067
3068 /*
3069 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3070 */
3071 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3072 {
3073 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3074 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3075 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3076 {
3077 /* The GDT is read-only but the writable GDT is available. */
3078 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3079 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3080 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3081 AssertRCReturn(rc, rc);
3082 }
3083 }
3084#else
3085 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3086#endif
3087 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3088 AssertRCReturn(rc, rc);
3089
3090 /*
3091 * Host FS base and GS base.
3092 */
3093#if HC_ARCH_BITS == 64
3094 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3095 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3096 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3097 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3098 AssertRCReturn(rc, rc);
3099
3100 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3101 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3102 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3103 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3104 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3105#endif
3106 return VINF_SUCCESS;
3107}
3108
3109
3110/**
3111 * Exports certain host MSRs in the VM-exit MSR-load area and some in the
3112 * host-state area of the VMCS.
3113 *
3114 * Theses MSRs will be automatically restored on the host after every successful
3115 * VM-exit.
3116 *
3117 * @returns VBox status code.
3118 * @param pVCpu The cross context virtual CPU structure.
3119 *
3120 * @remarks No-long-jump zone!!!
3121 */
3122static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
3123{
3124 AssertPtr(pVCpu);
3125 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3126
3127 /*
3128 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3129 * rather than swapping them on every VM-entry.
3130 */
3131 hmR0VmxLazySaveHostMsrs(pVCpu);
3132
3133 /*
3134 * Host Sysenter MSRs.
3135 */
3136 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3137#if HC_ARCH_BITS == 32
3138 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3139 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3140#else
3141 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3142 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3143#endif
3144 AssertRCReturn(rc, rc);
3145
3146 /*
3147 * Host EFER MSR.
3148 *
3149 * If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3150 * done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3151 */
3152 PVM pVM = pVCpu->CTX_SUFF(pVM);
3153 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3154 {
3155 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3156 AssertRCReturn(rc, rc);
3157 }
3158
3159 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see hmR0VmxExportGuestExitCtls(). */
3160
3161 return VINF_SUCCESS;
3162}
3163
3164
3165/**
3166 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3167 *
3168 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3169 * these two bits are handled by VM-entry, see hmR0VmxExportGuestExitCtls() and
3170 * hmR0VMxExportGuestEntryCtls().
3171 *
3172 * @returns true if we need to load guest EFER, false otherwise.
3173 * @param pVCpu The cross context virtual CPU structure.
3174 *
3175 * @remarks Requires EFER, CR4.
3176 * @remarks No-long-jump zone!!!
3177 */
3178static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu)
3179{
3180#ifdef HMVMX_ALWAYS_SWAP_EFER
3181 return true;
3182#endif
3183
3184 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3185#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3186 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3187 if (CPUMIsGuestInLongModeEx(pCtx))
3188 return false;
3189#endif
3190
3191 PVM pVM = pVCpu->CTX_SUFF(pVM);
3192 uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3193 uint64_t const u64GuestEfer = pCtx->msrEFER;
3194
3195 /*
3196 * For 64-bit guests, if EFER.SCE bit differs, we need to swap EFER to ensure that the
3197 * guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3198 */
3199 if ( CPUMIsGuestInLongModeEx(pCtx)
3200 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3201 {
3202 return true;
3203 }
3204
3205 /*
3206 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3207 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3208 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3209 */
3210 if ( (pCtx->cr4 & X86_CR4_PAE)
3211 && (pCtx->cr0 & X86_CR0_PG)
3212 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3213 {
3214 /* Assert that host is NX capable. */
3215 Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
3216 return true;
3217 }
3218
3219 return false;
3220}
3221
3222
3223/**
3224 * Exports the guest state with appropriate VM-entry controls in the VMCS.
3225 *
3226 * These controls can affect things done on VM-exit; e.g. "load debug controls",
3227 * see Intel spec. 24.8.1 "VM-entry controls".
3228 *
3229 * @returns VBox status code.
3230 * @param pVCpu The cross context virtual CPU structure.
3231 *
3232 * @remarks Requires EFER.
3233 * @remarks No-long-jump zone!!!
3234 */
3235static int hmR0VmxExportGuestEntryCtls(PVMCPU pVCpu)
3236{
3237 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_CTLS)
3238 {
3239 PVM pVM = pVCpu->CTX_SUFF(pVM);
3240 uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
3241 uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3242
3243 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3244 fVal |= VMX_ENTRY_CTLS_LOAD_DEBUG;
3245
3246 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3247 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
3248 {
3249 fVal |= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
3250 Log4Func(("VMX_ENTRY_CTLS_IA32E_MODE_GUEST\n"));
3251 }
3252 else
3253 Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
3254
3255 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3256 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3257 && hmR0VmxShouldSwapEferMsr(pVCpu))
3258 {
3259 fVal |= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
3260 Log4Func(("VMX_ENTRY_CTLS_LOAD_EFER_MSR\n"));
3261 }
3262
3263 /*
3264 * The following should -not- be set (since we're not in SMM mode):
3265 * - VMX_ENTRY_CTLS_ENTRY_TO_SMM
3266 * - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
3267 */
3268
3269 /** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
3270 * VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
3271
3272 if ((fVal & fZap) != fVal)
3273 {
3274 Log4Func(("Invalid VM-entry controls combo! Cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3275 pVM->hm.s.vmx.Msrs.EntryCtls.n.disallowed0, fVal, fZap));
3276 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3277 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3278 }
3279
3280 /* Commit it to the VMCS and update our cache. */
3281 if (pVCpu->hm.s.vmx.u32EntryCtls != fVal)
3282 {
3283 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
3284 AssertRCReturn(rc, rc);
3285 pVCpu->hm.s.vmx.u32EntryCtls = fVal;
3286 }
3287
3288 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_CTLS);
3289 }
3290 return VINF_SUCCESS;
3291}
3292
3293
3294/**
3295 * Exports the guest state with appropriate VM-exit controls in the VMCS.
3296 *
3297 * @returns VBox status code.
3298 * @param pVCpu The cross context virtual CPU structure.
3299 *
3300 * @remarks Requires EFER.
3301 */
3302static int hmR0VmxExportGuestExitCtls(PVMCPU pVCpu)
3303{
3304 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_EXIT_CTLS)
3305 {
3306 PVM pVM = pVCpu->CTX_SUFF(pVM);
3307 uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
3308 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3309
3310 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3311 fVal |= VMX_EXIT_CTLS_SAVE_DEBUG;
3312
3313 /*
3314 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3315 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in
3316 * hmR0VmxExportHostMsrs().
3317 */
3318#if HC_ARCH_BITS == 64
3319 fVal |= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
3320 Log4Func(("VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE\n"));
3321#else
3322 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3323 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3324 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3325 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3326 {
3327 /* The switcher returns to long mode, EFER is managed by the switcher. */
3328 fVal |= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
3329 Log4Func(("VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE\n"));
3330 }
3331 else
3332 Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
3333#endif
3334
3335 /* If the newer VMCS fields for managing EFER exists, use it. */
3336 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3337 && hmR0VmxShouldSwapEferMsr(pVCpu))
3338 {
3339 fVal |= VMX_EXIT_CTLS_SAVE_EFER_MSR
3340 | VMX_EXIT_CTLS_LOAD_EFER_MSR;
3341 Log4Func(("VMX_EXIT_CTLS_SAVE_EFER_MSR and VMX_EXIT_CTLS_LOAD_EFER_MSR\n"));
3342 }
3343
3344 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3345 Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
3346
3347 /** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
3348 * VMX_EXIT_CTLS_SAVE_PAT_MSR,
3349 * VMX_EXIT_CTLS_LOAD_PAT_MSR. */
3350
3351 /* Enable saving of the VMX preemption timer value on VM-exit. */
3352 if ( pVM->hm.s.vmx.fUsePreemptTimer
3353 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_VMX_PREEMPT_TIMER))
3354 fVal |= VMX_EXIT_CTLS_SAVE_VMX_PREEMPT_TIMER;
3355
3356 if ((fVal & fZap) != fVal)
3357 {
3358 LogRelFunc(("Invalid VM-exit controls combo! cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3359 pVM->hm.s.vmx.Msrs.ExitCtls.n.disallowed0, fVal, fZap));
3360 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3361 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3362 }
3363
3364 /* Commit it to the VMCS and update our cache. */
3365 if (pVCpu->hm.s.vmx.u32ExitCtls != fVal)
3366 {
3367 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
3368 AssertRCReturn(rc, rc);
3369 pVCpu->hm.s.vmx.u32ExitCtls = fVal;
3370 }
3371
3372 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_EXIT_CTLS);
3373 }
3374 return VINF_SUCCESS;
3375}
3376
3377
3378/**
3379 * Sets the TPR threshold in the VMCS.
3380 *
3381 * @returns VBox status code.
3382 * @param pVCpu The cross context virtual CPU structure.
3383 * @param u32TprThreshold The TPR threshold (task-priority class only).
3384 */
3385DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3386{
3387 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3388 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3389 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3390}
3391
3392
3393/**
3394 * Exports the guest APIC TPR state into the VMCS.
3395 *
3396 * @returns VBox status code.
3397 * @param pVCpu The cross context virtual CPU structure.
3398 *
3399 * @remarks No-long-jump zone!!!
3400 */
3401static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu)
3402{
3403 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
3404 {
3405 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
3406
3407 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3408 && APICIsEnabled(pVCpu))
3409 {
3410 /*
3411 * Setup TPR shadowing.
3412 */
3413 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
3414 {
3415 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3416
3417 bool fPendingIntr = false;
3418 uint8_t u8Tpr = 0;
3419 uint8_t u8PendingIntr = 0;
3420 int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3421 AssertRCReturn(rc, rc);
3422
3423 /*
3424 * If there are interrupts pending but masked by the TPR, instruct VT-x to
3425 * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
3426 * priority of the pending interrupt so we can deliver the interrupt. If there
3427 * are no interrupts pending, set threshold to 0 to not cause any
3428 * TPR-below-threshold VM-exits.
3429 */
3430 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3431 uint32_t u32TprThreshold = 0;
3432 if (fPendingIntr)
3433 {
3434 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3435 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3436 const uint8_t u8TprPriority = u8Tpr >> 4;
3437 if (u8PendingPriority <= u8TprPriority)
3438 u32TprThreshold = u8PendingPriority;
3439 }
3440
3441 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3442 AssertRCReturn(rc, rc);
3443 }
3444 }
3445 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
3446 }
3447 return VINF_SUCCESS;
3448}
3449
3450
3451/**
3452 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3453 *
3454 * @returns Guest's interruptibility-state.
3455 * @param pVCpu The cross context virtual CPU structure.
3456 *
3457 * @remarks No-long-jump zone!!!
3458 */
3459static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu)
3460{
3461 /*
3462 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3463 */
3464 uint32_t fIntrState = 0;
3465 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3466 {
3467 /* If inhibition is active, RIP & RFLAGS should've been accessed
3468 (i.e. read previously from the VMCS or from ring-3). */
3469 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3470#ifdef VBOX_STRICT
3471 uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
3472 AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
3473#endif
3474 if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3475 {
3476 if (pCtx->eflags.Bits.u1IF)
3477 fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
3478 else
3479 fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
3480 }
3481 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3482 {
3483 /*
3484 * We can clear the inhibit force flag as even if we go back to the recompiler
3485 * without executing guest code in VT-x, the flag's condition to be cleared is
3486 * met and thus the cleared state is correct.
3487 */
3488 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3489 }
3490 }
3491
3492 /*
3493 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3494 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3495 * setting this would block host-NMIs and IRET will not clear the blocking.
3496 *
3497 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3498 */
3499 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3500 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
3501 {
3502 fIntrState |= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
3503 }
3504
3505 return fIntrState;
3506}
3507
3508
3509/**
3510 * Exports the guest's interruptibility-state into the guest-state area in the
3511 * VMCS.
3512 *
3513 * @returns VBox status code.
3514 * @param pVCpu The cross context virtual CPU structure.
3515 * @param fIntrState The interruptibility-state to set.
3516 */
3517static int hmR0VmxExportGuestIntrState(PVMCPU pVCpu, uint32_t fIntrState)
3518{
3519 NOREF(pVCpu);
3520 AssertMsg(!(fIntrState & 0xfffffff0), ("%#x\n", fIntrState)); /* Bits 31:4 MBZ. */
3521 Assert((fIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3522 return VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
3523}
3524
3525
3526/**
3527 * Exports the exception intercepts required for guest execution in the VMCS.
3528 *
3529 * @returns VBox status code.
3530 * @param pVCpu The cross context virtual CPU structure.
3531 *
3532 * @remarks No-long-jump zone!!!
3533 */
3534static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu)
3535{
3536 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
3537 {
3538 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3539
3540 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxExportSharedCR0(). */
3541 if (pVCpu->hm.s.fGIMTrapXcptUD)
3542 uXcptBitmap |= RT_BIT(X86_XCPT_UD);
3543#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3544 else
3545 uXcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3546#endif
3547
3548 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_AC));
3549 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_DB));
3550
3551 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3552 {
3553 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3554 AssertRCReturn(rc, rc);
3555 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3556 }
3557
3558 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3559 Log4Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64\n", uXcptBitmap));
3560 }
3561 return VINF_SUCCESS;
3562}
3563
3564
3565/**
3566 * Exports the guest's RIP into the guest-state area in the VMCS.
3567 *
3568 * @returns VBox status code.
3569 * @param pVCpu The cross context virtual CPU structure.
3570 *
3571 * @remarks No-long-jump zone!!!
3572 */
3573static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
3574{
3575 int rc = VINF_SUCCESS;
3576 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
3577 {
3578 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
3579
3580 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
3581 AssertRCReturn(rc, rc);
3582
3583 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3584 Log4Func(("RIP=%#RX64\n", pVCpu->cpum.GstCtx.rip));
3585 }
3586 return rc;
3587}
3588
3589
3590/**
3591 * Exports the guest's RSP into the guest-state area in the VMCS.
3592 *
3593 * @returns VBox status code.
3594 * @param pVCpu The cross context virtual CPU structure.
3595 *
3596 * @remarks No-long-jump zone!!!
3597 */
3598static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
3599{
3600 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3601 {
3602 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
3603
3604 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
3605 AssertRCReturn(rc, rc);
3606
3607 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3608 }
3609 return VINF_SUCCESS;
3610}
3611
3612
3613/**
3614 * Exports the guest's RFLAGS into the guest-state area in the VMCS.
3615 *
3616 * @returns VBox status code.
3617 * @param pVCpu The cross context virtual CPU structure.
3618 *
3619 * @remarks No-long-jump zone!!!
3620 */
3621static int hmR0VmxExportGuestRflags(PVMCPU pVCpu)
3622{
3623 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3624 {
3625 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
3626
3627 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3628 Let us assert it as such and use 32-bit VMWRITE. */
3629 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
3630 X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
3631 Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3632 Assert(!(fEFlags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3633
3634 /*
3635 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3636 * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3637 * can run the real-mode guest code under Virtual 8086 mode.
3638 */
3639 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3640 {
3641 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3642 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3643 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3644 fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3645 fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3646 }
3647
3648 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3649 AssertRCReturn(rc, rc);
3650
3651 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3652 Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
3653 }
3654 return VINF_SUCCESS;
3655}
3656
3657
3658/**
3659 * Exports the guest CR0 control register into the guest-state area in the VMCS.
3660 *
3661 * The guest FPU state is always pre-loaded hence we don't need to bother about
3662 * sharing FPU related CR0 bits between the guest and host.
3663 *
3664 * @returns VBox status code.
3665 * @param pVCpu The cross context virtual CPU structure.
3666 *
3667 * @remarks No-long-jump zone!!!
3668 */
3669static int hmR0VmxExportGuestCR0(PVMCPU pVCpu)
3670{
3671 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
3672 {
3673 PVM pVM = pVCpu->CTX_SUFF(pVM);
3674 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
3675 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.cr0));
3676
3677 uint32_t const u32ShadowCr0 = pVCpu->cpum.GstCtx.cr0;
3678 uint32_t u32GuestCr0 = pVCpu->cpum.GstCtx.cr0;
3679
3680 /*
3681 * Setup VT-x's view of the guest CR0.
3682 * Minimize VM-exits due to CR3 changes when we have NestedPaging.
3683 */
3684 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
3685 if (pVM->hm.s.fNestedPaging)
3686 {
3687 if (CPUMIsGuestPagingEnabled(pVCpu))
3688 {
3689 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3690 uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
3691 | VMX_PROC_CTLS_CR3_STORE_EXIT);
3692 }
3693 else
3694 {
3695 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3696 uProcCtls |= VMX_PROC_CTLS_CR3_LOAD_EXIT
3697 | VMX_PROC_CTLS_CR3_STORE_EXIT;
3698 }
3699
3700 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3701 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3702 uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
3703 }
3704 else
3705 {
3706 /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3707 u32GuestCr0 |= X86_CR0_WP;
3708 }
3709
3710 /*
3711 * Guest FPU bits.
3712 *
3713 * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3714 * using CR0.TS.
3715 *
3716 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3717 * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3718 */
3719 u32GuestCr0 |= X86_CR0_NE;
3720
3721 /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3722 bool const fInterceptMF = !(u32ShadowCr0 & X86_CR0_NE);
3723
3724 /*
3725 * Update exception intercepts.
3726 */
3727 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3728 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3729 {
3730 Assert(PDMVmmDevHeapIsEnabled(pVM));
3731 Assert(pVM->hm.s.vmx.pRealModeTSS);
3732 uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3733 }
3734 else
3735 {
3736 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3737 uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3738 if (fInterceptMF)
3739 uXcptBitmap |= RT_BIT(X86_XCPT_MF);
3740 }
3741
3742 /* Additional intercepts for debugging, define these yourself explicitly. */
3743#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3744 uXcptBitmap |= 0
3745 | RT_BIT(X86_XCPT_BP)
3746 | RT_BIT(X86_XCPT_DE)
3747 | RT_BIT(X86_XCPT_NM)
3748 | RT_BIT(X86_XCPT_TS)
3749 | RT_BIT(X86_XCPT_UD)
3750 | RT_BIT(X86_XCPT_NP)
3751 | RT_BIT(X86_XCPT_SS)
3752 | RT_BIT(X86_XCPT_GP)
3753 | RT_BIT(X86_XCPT_PF)
3754 | RT_BIT(X86_XCPT_MF)
3755 ;
3756#elif defined(HMVMX_ALWAYS_TRAP_PF)
3757 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
3758#endif
3759 Assert(pVM->hm.s.fNestedPaging || (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
3760
3761 /*
3762 * Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW).
3763 */
3764 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3765 uint32_t fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3766 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3767 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
3768 else
3769 Assert((fSetCr0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3770
3771 u32GuestCr0 |= fSetCr0;
3772 u32GuestCr0 &= fZapCr0;
3773 u32GuestCr0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3774
3775 /*
3776 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3777 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3778 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3779 */
3780 uint32_t u32Cr0Mask = X86_CR0_PE
3781 | X86_CR0_NE
3782 | (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
3783 | X86_CR0_PG
3784 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3785 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3786 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3787
3788 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3789 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3790 * and @bugref{6944}. */
3791#if 0
3792 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3793 u32Cr0Mask &= ~X86_CR0_PE;
3794#endif
3795 /*
3796 * Finally, update VMCS fields with the CR0 values and the exception bitmap.
3797 */
3798 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCr0);
3799 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32ShadowCr0);
3800 if (u32Cr0Mask != pVCpu->hm.s.vmx.u32Cr0Mask)
3801 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32Cr0Mask);
3802 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
3803 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3804 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3805 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3806 AssertRCReturn(rc, rc);
3807
3808 /* Update our caches. */
3809 pVCpu->hm.s.vmx.u32Cr0Mask = u32Cr0Mask;
3810 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
3811 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3812
3813 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3814
3815 Log4Func(("u32Cr0Mask=%#RX32 u32ShadowCr0=%#RX32 u32GuestCr0=%#RX32 (fSetCr0=%#RX32 fZapCr0=%#RX32\n", u32Cr0Mask,
3816 u32ShadowCr0, u32GuestCr0, fSetCr0, fZapCr0));
3817 }
3818
3819 return VINF_SUCCESS;
3820}
3821
3822
3823/**
3824 * Exports the guest control registers (CR3, CR4) into the guest-state area
3825 * in the VMCS.
3826 *
3827 * @returns VBox strict status code.
3828 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3829 * without unrestricted guest access and the VMMDev is not presently
3830 * mapped (e.g. EFI32).
3831 *
3832 * @param pVCpu The cross context virtual CPU structure.
3833 *
3834 * @remarks No-long-jump zone!!!
3835 */
3836static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu)
3837{
3838 int rc = VINF_SUCCESS;
3839 PVM pVM = pVCpu->CTX_SUFF(pVM);
3840
3841 /*
3842 * Guest CR2.
3843 * It's always loaded in the assembler code. Nothing to do here.
3844 */
3845
3846 /*
3847 * Guest CR3.
3848 */
3849 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
3850 {
3851 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
3852
3853 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3854 if (pVM->hm.s.fNestedPaging)
3855 {
3856 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3857
3858 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3859 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3860 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3861 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3862
3863 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3864 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3865 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3866
3867 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3868 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3869 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3870 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3871 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3872 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3873 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3874
3875 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3876 AssertRCReturn(rc, rc);
3877
3878 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3879 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3880 || CPUMIsGuestPagingEnabledEx(pCtx))
3881 {
3882 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3883 if (CPUMIsGuestInPAEModeEx(pCtx))
3884 {
3885 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3886 AssertRCReturn(rc, rc);
3887 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3888 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3889 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3890 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3891 AssertRCReturn(rc, rc);
3892 }
3893
3894 /*
3895 * The guest's view of its CR3 is unblemished with Nested Paging when the
3896 * guest is using paging or we have unrestricted guest execution to handle
3897 * the guest when it's not using paging.
3898 */
3899 GCPhysGuestCR3 = pCtx->cr3;
3900 }
3901 else
3902 {
3903 /*
3904 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
3905 * thinks it accesses physical memory directly, we use our identity-mapped
3906 * page table to map guest-linear to guest-physical addresses. EPT takes care
3907 * of translating it to host-physical addresses.
3908 */
3909 RTGCPHYS GCPhys;
3910 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3911
3912 /* We obtain it here every time as the guest could have relocated this PCI region. */
3913 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3914 if (RT_SUCCESS(rc))
3915 { /* likely */ }
3916 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3917 {
3918 Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3919 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3920 }
3921 else
3922 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3923
3924 GCPhysGuestCR3 = GCPhys;
3925 }
3926
3927 Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
3928 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3929 AssertRCReturn(rc, rc);
3930 }
3931 else
3932 {
3933 /* Non-nested paging case, just use the hypervisor's CR3. */
3934 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3935
3936 Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
3937 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3938 AssertRCReturn(rc, rc);
3939 }
3940
3941 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3942 }
3943
3944 /*
3945 * Guest CR4.
3946 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3947 */
3948 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
3949 {
3950 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3951 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
3952 Assert(!RT_HI_U32(pCtx->cr4));
3953
3954 uint32_t u32GuestCr4 = pCtx->cr4;
3955 uint32_t const u32ShadowCr4 = pCtx->cr4;
3956
3957 /*
3958 * Setup VT-x's view of the guest CR4.
3959 *
3960 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3961 * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3962 * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3963 *
3964 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3965 */
3966 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3967 {
3968 Assert(pVM->hm.s.vmx.pRealModeTSS);
3969 Assert(PDMVmmDevHeapIsEnabled(pVM));
3970 u32GuestCr4 &= ~X86_CR4_VME;
3971 }
3972
3973 if (pVM->hm.s.fNestedPaging)
3974 {
3975 if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3976 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3977 {
3978 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3979 u32GuestCr4 |= X86_CR4_PSE;
3980 /* Our identity mapping is a 32-bit page directory. */
3981 u32GuestCr4 &= ~X86_CR4_PAE;
3982 }
3983 /* else use guest CR4.*/
3984 }
3985 else
3986 {
3987 /*
3988 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3989 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3990 */
3991 switch (pVCpu->hm.s.enmShadowMode)
3992 {
3993 case PGMMODE_REAL: /* Real-mode. */
3994 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3995 case PGMMODE_32_BIT: /* 32-bit paging. */
3996 {
3997 u32GuestCr4 &= ~X86_CR4_PAE;
3998 break;
3999 }
4000
4001 case PGMMODE_PAE: /* PAE paging. */
4002 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4003 {
4004 u32GuestCr4 |= X86_CR4_PAE;
4005 break;
4006 }
4007
4008 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4009 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4010#ifdef VBOX_ENABLE_64_BITS_GUESTS
4011 break;
4012#endif
4013 default:
4014 AssertFailed();
4015 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4016 }
4017 }
4018
4019 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4020 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4021 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4022 u32GuestCr4 |= fSetCr4;
4023 u32GuestCr4 &= fZapCr4;
4024
4025 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them,
4026 that would cause a VM-exit. */
4027 uint32_t u32Cr4Mask = X86_CR4_VME
4028 | X86_CR4_PAE
4029 | X86_CR4_PGE
4030 | X86_CR4_PSE
4031 | X86_CR4_VMXE;
4032 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4033 u32Cr4Mask |= X86_CR4_OSXSAVE;
4034 if (pVM->cpum.ro.GuestFeatures.fPcid)
4035 u32Cr4Mask |= X86_CR4_PCIDE;
4036
4037 /* Write VT-x's view of the guest CR4, the CR4 modify mask and the read-only CR4 shadow
4038 into the VMCS and update our cache. */
4039 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCr4);
4040 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32ShadowCr4);
4041 if (pVCpu->hm.s.vmx.u32Cr4Mask != u32Cr4Mask)
4042 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32Cr4Mask);
4043 AssertRCReturn(rc, rc);
4044 pVCpu->hm.s.vmx.u32Cr4Mask = u32Cr4Mask;
4045
4046 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4047 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
4048
4049 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
4050
4051 Log4Func(("u32GuestCr4=%#RX32 u32ShadowCr4=%#RX32 (fSetCr4=%#RX32 fZapCr4=%#RX32)\n", u32GuestCr4, u32ShadowCr4, fSetCr4,
4052 fZapCr4));
4053 }
4054 return rc;
4055}
4056
4057
4058/**
4059 * Exports the guest debug registers into the guest-state area in the VMCS.
4060 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4061 *
4062 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4063 *
4064 * @returns VBox status code.
4065 * @param pVCpu The cross context virtual CPU structure.
4066 *
4067 * @remarks No-long-jump zone!!!
4068 */
4069static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu)
4070{
4071 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4072
4073#ifdef VBOX_STRICT
4074 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4075 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
4076 {
4077 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4078 Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0);
4079 Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
4080 }
4081#endif
4082
4083 bool fSteppingDB = false;
4084 bool fInterceptMovDRx = false;
4085 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
4086 if (pVCpu->hm.s.fSingleInstruction)
4087 {
4088 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4089 PVM pVM = pVCpu->CTX_SUFF(pVM);
4090 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
4091 {
4092 uProcCtls |= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
4093 Assert(fSteppingDB == false);
4094 }
4095 else
4096 {
4097 pVCpu->cpum.GstCtx.eflags.u32 |= X86_EFL_TF;
4098 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_RFLAGS;
4099 pVCpu->hm.s.fClearTrapFlag = true;
4100 fSteppingDB = true;
4101 }
4102 }
4103
4104 uint32_t u32GuestDr7;
4105 if ( fSteppingDB
4106 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4107 {
4108 /*
4109 * Use the combined guest and host DRx values found in the hypervisor register set
4110 * because the debugger has breakpoints active or someone is single stepping on the
4111 * host side without a monitor trap flag.
4112 *
4113 * Note! DBGF expects a clean DR6 state before executing guest code.
4114 */
4115#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4116 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4117 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4118 {
4119 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4120 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4121 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4122 }
4123 else
4124#endif
4125 if (!CPUMIsHyperDebugStateActive(pVCpu))
4126 {
4127 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4128 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4129 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4130 }
4131
4132 /* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
4133 u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
4134 pVCpu->hm.s.fUsingHyperDR7 = true;
4135 fInterceptMovDRx = true;
4136 }
4137 else
4138 {
4139 /*
4140 * If the guest has enabled debug registers, we need to load them prior to
4141 * executing guest code so they'll trigger at the right time.
4142 */
4143 if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD))
4144 {
4145#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4146 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4147 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4148 {
4149 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4150 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4151 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4152 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4153 }
4154 else
4155#endif
4156 if (!CPUMIsGuestDebugStateActive(pVCpu))
4157 {
4158 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4159 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4160 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4161 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4162 }
4163 Assert(!fInterceptMovDRx);
4164 }
4165 /*
4166 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4167 * must intercept #DB in order to maintain a correct DR6 guest value, and
4168 * because we need to intercept it to prevent nested #DBs from hanging the
4169 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4170 */
4171#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4172 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4173 && !CPUMIsGuestDebugStateActive(pVCpu))
4174#else
4175 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4176#endif
4177 {
4178 fInterceptMovDRx = true;
4179 }
4180
4181 /* Update DR7 with the actual guest value. */
4182 u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
4183 pVCpu->hm.s.fUsingHyperDR7 = false;
4184 }
4185
4186 if (fInterceptMovDRx)
4187 uProcCtls |= VMX_PROC_CTLS_MOV_DR_EXIT;
4188 else
4189 uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
4190
4191 /*
4192 * Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
4193 * monitor-trap flag and update our cache.
4194 */
4195 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
4196 {
4197 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4198 AssertRCReturn(rc2, rc2);
4199 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
4200 }
4201
4202 /*
4203 * Update guest DR7.
4204 */
4205 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
4206 AssertRCReturn(rc, rc);
4207
4208 return VINF_SUCCESS;
4209}
4210
4211
4212#ifdef VBOX_STRICT
4213/**
4214 * Strict function to validate segment registers.
4215 *
4216 * @param pVCpu The cross context virtual CPU structure.
4217 *
4218 * @remarks Will import guest CR0 on strict builds during validation of
4219 * segments.
4220 */
4221static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu)
4222{
4223 /*
4224 * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
4225 *
4226 * The reason we check for attribute value 0 in this function and not just the unusable bit is
4227 * because hmR0VmxExportGuestSegmentReg() only updates the VMCS' copy of the value with the unusable bit
4228 * and doesn't change the guest-context value.
4229 */
4230 PVM pVM = pVCpu->CTX_SUFF(pVM);
4231 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4232 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
4233 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4234 && ( !CPUMIsGuestInRealModeEx(pCtx)
4235 && !CPUMIsGuestInV86ModeEx(pCtx)))
4236 {
4237 /* Protected mode checks */
4238 /* CS */
4239 Assert(pCtx->cs.Attr.n.u1Present);
4240 Assert(!(pCtx->cs.Attr.u & 0xf00));
4241 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4242 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4243 || !(pCtx->cs.Attr.n.u1Granularity));
4244 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4245 || (pCtx->cs.Attr.n.u1Granularity));
4246 /* CS cannot be loaded with NULL in protected mode. */
4247 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4248 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4249 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4250 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4251 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4252 else
4253 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4254 /* SS */
4255 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4256 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4257 if ( !(pCtx->cr0 & X86_CR0_PE)
4258 || pCtx->cs.Attr.n.u4Type == 3)
4259 {
4260 Assert(!pCtx->ss.Attr.n.u2Dpl);
4261 }
4262 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4263 {
4264 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4265 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4266 Assert(pCtx->ss.Attr.n.u1Present);
4267 Assert(!(pCtx->ss.Attr.u & 0xf00));
4268 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4269 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4270 || !(pCtx->ss.Attr.n.u1Granularity));
4271 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4272 || (pCtx->ss.Attr.n.u1Granularity));
4273 }
4274 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmentReg(). */
4275 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4276 {
4277 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4278 Assert(pCtx->ds.Attr.n.u1Present);
4279 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4280 Assert(!(pCtx->ds.Attr.u & 0xf00));
4281 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4282 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4283 || !(pCtx->ds.Attr.n.u1Granularity));
4284 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4285 || (pCtx->ds.Attr.n.u1Granularity));
4286 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4287 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4288 }
4289 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4290 {
4291 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4292 Assert(pCtx->es.Attr.n.u1Present);
4293 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4294 Assert(!(pCtx->es.Attr.u & 0xf00));
4295 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4296 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4297 || !(pCtx->es.Attr.n.u1Granularity));
4298 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4299 || (pCtx->es.Attr.n.u1Granularity));
4300 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4301 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4302 }
4303 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4304 {
4305 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4306 Assert(pCtx->fs.Attr.n.u1Present);
4307 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4308 Assert(!(pCtx->fs.Attr.u & 0xf00));
4309 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4310 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4311 || !(pCtx->fs.Attr.n.u1Granularity));
4312 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4313 || (pCtx->fs.Attr.n.u1Granularity));
4314 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4315 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4316 }
4317 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4318 {
4319 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4320 Assert(pCtx->gs.Attr.n.u1Present);
4321 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4322 Assert(!(pCtx->gs.Attr.u & 0xf00));
4323 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4324 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4325 || !(pCtx->gs.Attr.n.u1Granularity));
4326 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4327 || (pCtx->gs.Attr.n.u1Granularity));
4328 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4329 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4330 }
4331 /* 64-bit capable CPUs. */
4332# if HC_ARCH_BITS == 64
4333 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4334 Assert(!pCtx->ss.Attr.u || !RT_HI_U32(pCtx->ss.u64Base));
4335 Assert(!pCtx->ds.Attr.u || !RT_HI_U32(pCtx->ds.u64Base));
4336 Assert(!pCtx->es.Attr.u || !RT_HI_U32(pCtx->es.u64Base));
4337# endif
4338 }
4339 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4340 || ( CPUMIsGuestInRealModeEx(pCtx)
4341 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4342 {
4343 /* Real and v86 mode checks. */
4344 /* hmR0VmxExportGuestSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4345 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4346 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4347 {
4348 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4349 }
4350 else
4351 {
4352 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4353 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4354 }
4355
4356 /* CS */
4357 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4358 Assert(pCtx->cs.u32Limit == 0xffff);
4359 Assert(u32CSAttr == 0xf3);
4360 /* SS */
4361 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4362 Assert(pCtx->ss.u32Limit == 0xffff);
4363 Assert(u32SSAttr == 0xf3);
4364 /* DS */
4365 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4366 Assert(pCtx->ds.u32Limit == 0xffff);
4367 Assert(u32DSAttr == 0xf3);
4368 /* ES */
4369 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4370 Assert(pCtx->es.u32Limit == 0xffff);
4371 Assert(u32ESAttr == 0xf3);
4372 /* FS */
4373 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4374 Assert(pCtx->fs.u32Limit == 0xffff);
4375 Assert(u32FSAttr == 0xf3);
4376 /* GS */
4377 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4378 Assert(pCtx->gs.u32Limit == 0xffff);
4379 Assert(u32GSAttr == 0xf3);
4380 /* 64-bit capable CPUs. */
4381# if HC_ARCH_BITS == 64
4382 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4383 Assert(!u32SSAttr || !RT_HI_U32(pCtx->ss.u64Base));
4384 Assert(!u32DSAttr || !RT_HI_U32(pCtx->ds.u64Base));
4385 Assert(!u32ESAttr || !RT_HI_U32(pCtx->es.u64Base));
4386# endif
4387 }
4388}
4389#endif /* VBOX_STRICT */
4390
4391
4392/**
4393 * Exports a guest segment register into the guest-state area in the VMCS.
4394 *
4395 * @returns VBox status code.
4396 * @param pVCpu The cross context virtual CPU structure.
4397 * @param idxSel Index of the selector in the VMCS.
4398 * @param idxLimit Index of the segment limit in the VMCS.
4399 * @param idxBase Index of the segment base in the VMCS.
4400 * @param idxAccess Index of the access rights of the segment in the VMCS.
4401 * @param pSelReg Pointer to the segment selector.
4402 *
4403 * @remarks No-long-jump zone!!!
4404 */
4405static int hmR0VmxExportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
4406 PCCPUMSELREG pSelReg)
4407{
4408 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4409 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4410 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4411 AssertRCReturn(rc, rc);
4412
4413 uint32_t u32Access = pSelReg->Attr.u;
4414 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4415 {
4416 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4417 u32Access = 0xf3;
4418 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4419 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4420 }
4421 else
4422 {
4423 /*
4424 * The way to differentiate between whether this is really a null selector or was just
4425 * a selector loaded with 0 in real-mode is using the segment attributes. A selector
4426 * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4427 * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4428 * NULL selectors loaded in protected-mode have their attribute as 0.
4429 */
4430 if (!u32Access)
4431 u32Access = X86DESCATTR_UNUSABLE;
4432 }
4433
4434 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4435 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4436 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4437
4438 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4439 AssertRCReturn(rc, rc);
4440 return rc;
4441}
4442
4443
4444/**
4445 * Exports the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4446 * into the guest-state area in the VMCS.
4447 *
4448 * @returns VBox status code.
4449 * @param pVCpu The cross context virtual CPU structure.
4450 *
4451 * @remarks Will import guest CR0 on strict builds during validation of
4452 * segments.
4453 * @remarks No-long-jump zone!!!
4454 */
4455static int hmR0VmxExportGuestSegmentRegs(PVMCPU pVCpu)
4456{
4457 int rc = VERR_INTERNAL_ERROR_5;
4458 PVM pVM = pVCpu->CTX_SUFF(pVM);
4459 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4460
4461 /*
4462 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4463 */
4464 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4465 {
4466#ifdef VBOX_WITH_REM
4467 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4468 {
4469 Assert(pVM->hm.s.vmx.pRealModeTSS);
4470 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4471 if ( pVCpu->hm.s.vmx.fWasInRealMode
4472 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4473 {
4474 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4475 in real-mode (e.g. OpenBSD 4.0) */
4476 REMFlushTBs(pVM);
4477 Log4Func(("Switch to protected mode detected!\n"));
4478 pVCpu->hm.s.vmx.fWasInRealMode = false;
4479 }
4480 }
4481#endif
4482 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
4483 {
4484 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
4485 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4486 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pCtx->cs.Attr.u;
4487 rc = HMVMX_EXPORT_SREG(CS, &pCtx->cs);
4488 AssertRCReturn(rc, rc);
4489 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
4490 }
4491
4492 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
4493 {
4494 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
4495 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4496 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pCtx->ss.Attr.u;
4497 rc = HMVMX_EXPORT_SREG(SS, &pCtx->ss);
4498 AssertRCReturn(rc, rc);
4499 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
4500 }
4501
4502 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
4503 {
4504 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
4505 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4506 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pCtx->ds.Attr.u;
4507 rc = HMVMX_EXPORT_SREG(DS, &pCtx->ds);
4508 AssertRCReturn(rc, rc);
4509 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
4510 }
4511
4512 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
4513 {
4514 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
4515 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4516 pVCpu->hm.s.vmx.RealMode.AttrES.u = pCtx->es.Attr.u;
4517 rc = HMVMX_EXPORT_SREG(ES, &pCtx->es);
4518 AssertRCReturn(rc, rc);
4519 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
4520 }
4521
4522 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
4523 {
4524 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
4525 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4526 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pCtx->fs.Attr.u;
4527 rc = HMVMX_EXPORT_SREG(FS, &pCtx->fs);
4528 AssertRCReturn(rc, rc);
4529 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
4530 }
4531
4532 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
4533 {
4534 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
4535 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4536 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pCtx->gs.Attr.u;
4537 rc = HMVMX_EXPORT_SREG(GS, &pCtx->gs);
4538 AssertRCReturn(rc, rc);
4539 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
4540 }
4541
4542#ifdef VBOX_STRICT
4543 hmR0VmxValidateSegmentRegs(pVCpu);
4544#endif
4545
4546 Log4Func(("CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pCtx->cs.Sel, pCtx->cs.u64Base,
4547 pCtx->cs.u32Limit, pCtx->cs.Attr.u));
4548 }
4549
4550 /*
4551 * Guest TR.
4552 */
4553 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
4554 {
4555 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
4556
4557 /*
4558 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
4559 * achieved using the interrupt redirection bitmap (all bits cleared to let the guest
4560 * handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4561 */
4562 uint16_t u16Sel = 0;
4563 uint32_t u32Limit = 0;
4564 uint64_t u64Base = 0;
4565 uint32_t u32AccessRights = 0;
4566
4567 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4568 {
4569 u16Sel = pCtx->tr.Sel;
4570 u32Limit = pCtx->tr.u32Limit;
4571 u64Base = pCtx->tr.u64Base;
4572 u32AccessRights = pCtx->tr.Attr.u;
4573 }
4574 else
4575 {
4576 Assert(pVM->hm.s.vmx.pRealModeTSS);
4577 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
4578
4579 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4580 RTGCPHYS GCPhys;
4581 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4582 AssertRCReturn(rc, rc);
4583
4584 X86DESCATTR DescAttr;
4585 DescAttr.u = 0;
4586 DescAttr.n.u1Present = 1;
4587 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4588
4589 u16Sel = 0;
4590 u32Limit = HM_VTX_TSS_SIZE;
4591 u64Base = GCPhys; /* in real-mode phys = virt. */
4592 u32AccessRights = DescAttr.u;
4593 }
4594
4595 /* Validate. */
4596 Assert(!(u16Sel & RT_BIT(2)));
4597 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4598 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4599 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4600 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4601 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4602 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4603 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4604 Assert( (u32Limit & 0xfff) == 0xfff
4605 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4606 Assert( !(pCtx->tr.u32Limit & 0xfff00000)
4607 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4608
4609 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4610 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4611 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4612 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4613 AssertRCReturn(rc, rc);
4614
4615 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
4616 Log4Func(("TR base=%#RX64\n", pCtx->tr.u64Base));
4617 }
4618
4619 /*
4620 * Guest GDTR.
4621 */
4622 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4623 {
4624 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
4625
4626 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
4627 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
4628 AssertRCReturn(rc, rc);
4629
4630 /* Validate. */
4631 Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4632
4633 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4634 Log4Func(("GDTR base=%#RX64\n", pCtx->gdtr.pGdt));
4635 }
4636
4637 /*
4638 * Guest LDTR.
4639 */
4640 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4641 {
4642 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
4643
4644 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4645 uint32_t u32Access = 0;
4646 if (!pCtx->ldtr.Attr.u)
4647 u32Access = X86DESCATTR_UNUSABLE;
4648 else
4649 u32Access = pCtx->ldtr.Attr.u;
4650
4651 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
4652 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
4653 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4654 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
4655 AssertRCReturn(rc, rc);
4656
4657 /* Validate. */
4658 if (!(u32Access & X86DESCATTR_UNUSABLE))
4659 {
4660 Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4661 Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4662 Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4663 Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4664 Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4665 Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4666 Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
4667 || !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4668 Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
4669 || pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4670 }
4671
4672 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4673 Log4Func(("LDTR base=%#RX64\n", pCtx->ldtr.u64Base));
4674 }
4675
4676 /*
4677 * Guest IDTR.
4678 */
4679 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4680 {
4681 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
4682
4683 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
4684 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
4685 AssertRCReturn(rc, rc);
4686
4687 /* Validate. */
4688 Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4689
4690 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4691 Log4Func(("IDTR base=%#RX64\n", pCtx->idtr.pIdt));
4692 }
4693
4694 return VINF_SUCCESS;
4695}
4696
4697
4698/**
4699 * Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4700 * areas.
4701 *
4702 * These MSRs will automatically be loaded to the host CPU on every successful
4703 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4704 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4705 * -not- updated here for performance reasons. See hmR0VmxExportHostMsrs().
4706 *
4707 * Also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4708 *
4709 * @returns VBox status code.
4710 * @param pVCpu The cross context virtual CPU structure.
4711 *
4712 * @remarks No-long-jump zone!!!
4713 */
4714static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu)
4715{
4716 AssertPtr(pVCpu);
4717 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4718
4719 /*
4720 * MSRs that we use the auto-load/store MSR area in the VMCS.
4721 * For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
4722 */
4723 PVM pVM = pVCpu->CTX_SUFF(pVM);
4724 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4725 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4726 {
4727 if (pVM->hm.s.fAllow64BitGuests)
4728 {
4729#if HC_ARCH_BITS == 32
4730 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS | CPUMCTX_EXTRN_KERNEL_GS_BASE);
4731
4732 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pCtx->msrLSTAR, false, NULL);
4733 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pCtx->msrSTAR, false, NULL);
4734 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pCtx->msrSFMASK, false, NULL);
4735 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, false, NULL);
4736 AssertRCReturn(rc, rc);
4737# ifdef LOG_ENABLED
4738 PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4739 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4740 Log4Func(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4741# endif
4742#endif
4743 }
4744 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4745 }
4746
4747 /*
4748 * Guest Sysenter MSRs.
4749 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4750 * VM-exits on WRMSRs for these MSRs.
4751 */
4752 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4753 {
4754 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
4755
4756 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4757 {
4758 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
4759 AssertRCReturn(rc, rc);
4760 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4761 }
4762
4763 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4764 {
4765 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
4766 AssertRCReturn(rc, rc);
4767 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4768 }
4769
4770 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4771 {
4772 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
4773 AssertRCReturn(rc, rc);
4774 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4775 }
4776 }
4777
4778 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4779 {
4780 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
4781
4782 if (hmR0VmxShouldSwapEferMsr(pVCpu))
4783 {
4784 /*
4785 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4786 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4787 */
4788 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4789 {
4790 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
4791 AssertRCReturn(rc,rc);
4792 Log4Func(("EFER=%#RX64\n", pCtx->msrEFER));
4793 }
4794 else
4795 {
4796 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pCtx->msrEFER, false /* fUpdateHostMsr */,
4797 NULL /* pfAddedAndUpdated */);
4798 AssertRCReturn(rc, rc);
4799
4800 /* We need to intercept reads too, see @bugref{7386#c16}. */
4801 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
4802 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4803 Log4Func(("MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pCtx->msrEFER,
4804 pVCpu->hm.s.vmx.cMsrs));
4805 }
4806 }
4807 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4808 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4809 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4810 }
4811
4812 return VINF_SUCCESS;
4813}
4814
4815
4816#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4817/**
4818 * Check if guest state allows safe use of 32-bit switcher again.
4819 *
4820 * Segment bases and protected mode structures must be 32-bit addressable
4821 * because the 32-bit switcher will ignore high dword when writing these VMCS
4822 * fields. See @bugref{8432} for details.
4823 *
4824 * @returns true if safe, false if must continue to use the 64-bit switcher.
4825 * @param pCtx Pointer to the guest-CPU context.
4826 *
4827 * @remarks No-long-jump zone!!!
4828 */
4829static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
4830{
4831 if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4832 if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4833 if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4834 if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4835 if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4836 if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4837 if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4838 if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4839 if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4840 if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4841
4842 /* All good, bases are 32-bit. */
4843 return true;
4844}
4845#endif
4846
4847
4848/**
4849 * Selects up the appropriate function to run guest code.
4850 *
4851 * @returns VBox status code.
4852 * @param pVCpu The cross context virtual CPU structure.
4853 *
4854 * @remarks No-long-jump zone!!!
4855 */
4856static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu)
4857{
4858 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4859 if (CPUMIsGuestInLongModeEx(pCtx))
4860 {
4861#ifndef VBOX_ENABLE_64_BITS_GUESTS
4862 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4863#endif
4864 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4865#if HC_ARCH_BITS == 32
4866 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4867 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4868 {
4869#ifdef VBOX_STRICT
4870 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4871 {
4872 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4873 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4874 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4875 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4876 | HM_CHANGED_VMX_ENTRY_CTLS
4877 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4878 }
4879#endif
4880 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4881
4882 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4883 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4884 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4885 Log4Func(("Selected 64-bit switcher\n"));
4886 }
4887#else
4888 /* 64-bit host. */
4889 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4890#endif
4891 }
4892 else
4893 {
4894 /* Guest is not in long mode, use the 32-bit handler. */
4895#if HC_ARCH_BITS == 32
4896 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4897 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4898 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4899 {
4900# ifdef VBOX_STRICT
4901 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4902 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4903 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4904 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4905 | HM_CHANGED_VMX_ENTRY_CTLS
4906 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4907# endif
4908 }
4909# ifdef VBOX_ENABLE_64_BITS_GUESTS
4910 /*
4911 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
4912 * design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
4913 * switcher flag because now we know the guest is in a sane state where it's safe
4914 * to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4915 * the much faster 32-bit switcher again.
4916 */
4917 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4918 {
4919 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4920 Log4Func(("Selected 32-bit switcher\n"));
4921 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4922 }
4923 else
4924 {
4925 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4926 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4927 || hmR0VmxIs32BitSwitcherSafe(pCtx))
4928 {
4929 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4930 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4931 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
4932 | HM_CHANGED_VMX_ENTRY_CTLS
4933 | HM_CHANGED_VMX_EXIT_CTLS
4934 | HM_CHANGED_HOST_CONTEXT);
4935 Log4Func(("Selected 32-bit switcher (safe)\n"));
4936 }
4937 }
4938# else
4939 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4940# endif
4941#else
4942 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4943#endif
4944 }
4945 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4946 return VINF_SUCCESS;
4947}
4948
4949
4950/**
4951 * Wrapper for running the guest code in VT-x.
4952 *
4953 * @returns VBox status code, no informational status codes.
4954 * @param pVCpu The cross context virtual CPU structure.
4955 *
4956 * @remarks No-long-jump zone!!!
4957 */
4958DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu)
4959{
4960 /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4961 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4962 pCtx->fExtrn |= HMVMX_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
4963
4964 /*
4965 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4966 * floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
4967 * callee-saved and thus the need for this XMM wrapper.
4968 *
4969 * See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
4970 */
4971 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4972 /** @todo Add stats for resume vs launch. */
4973 PVM pVM = pVCpu->CTX_SUFF(pVM);
4974#ifdef VBOX_WITH_KERNEL_USING_XMM
4975 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4976#else
4977 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4978#endif
4979 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4980 return rc;
4981}
4982
4983
4984/**
4985 * Reports world-switch error and dumps some useful debug info.
4986 *
4987 * @param pVCpu The cross context virtual CPU structure.
4988 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4989 * @param pVmxTransient Pointer to the VMX transient structure (only
4990 * exitReason updated).
4991 */
4992static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
4993{
4994 Assert(pVCpu);
4995 Assert(pVmxTransient);
4996 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
4997
4998 Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
4999 switch (rcVMRun)
5000 {
5001 case VERR_VMX_INVALID_VMXON_PTR:
5002 AssertFailed();
5003 break;
5004 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5005 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5006 {
5007 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5008 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5009 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
5010 AssertRC(rc);
5011
5012 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5013 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5014 Cannot do it here as we may have been long preempted. */
5015
5016#ifdef VBOX_STRICT
5017 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5018 pVmxTransient->uExitReason));
5019 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
5020 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5021 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5022 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5023 else
5024 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5025 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5026 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5027
5028 /* VMX control bits. */
5029 uint32_t u32Val;
5030 uint64_t u64Val;
5031 RTHCUINTREG uHCReg;
5032 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5033 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5034 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5035 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5036 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
5037 {
5038 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5039 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5040 }
5041 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5042 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5043 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5044 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5045 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5046 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5047 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5048 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5049 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5050 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5051 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5052 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5053 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5054 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5055 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5056 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5057 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5058 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5059 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5060 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5061 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5062 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5063 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5064 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5065 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5066 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5067 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5068 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5069 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5070 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5071 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5072 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5073 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5074 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5075 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5076 {
5077 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5078 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5079 }
5080
5081 /* Guest bits. */
5082 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5083 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
5084 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5085 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
5086 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5087 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
5088 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
5089 {
5090 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5091 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5092 }
5093
5094 /* Host bits. */
5095 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5096 Log4(("Host CR0 %#RHr\n", uHCReg));
5097 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5098 Log4(("Host CR3 %#RHr\n", uHCReg));
5099 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5100 Log4(("Host CR4 %#RHr\n", uHCReg));
5101
5102 RTGDTR HostGdtr;
5103 PCX86DESCHC pDesc;
5104 ASMGetGDTR(&HostGdtr);
5105 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5106 Log4(("Host CS %#08x\n", u32Val));
5107 if (u32Val < HostGdtr.cbGdt)
5108 {
5109 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5110 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5111 }
5112
5113 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5114 Log4(("Host DS %#08x\n", u32Val));
5115 if (u32Val < HostGdtr.cbGdt)
5116 {
5117 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5118 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5119 }
5120
5121 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5122 Log4(("Host ES %#08x\n", u32Val));
5123 if (u32Val < HostGdtr.cbGdt)
5124 {
5125 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5126 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5127 }
5128
5129 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5130 Log4(("Host FS %#08x\n", u32Val));
5131 if (u32Val < HostGdtr.cbGdt)
5132 {
5133 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5134 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5135 }
5136
5137 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5138 Log4(("Host GS %#08x\n", u32Val));
5139 if (u32Val < HostGdtr.cbGdt)
5140 {
5141 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5142 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5143 }
5144
5145 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5146 Log4(("Host SS %#08x\n", u32Val));
5147 if (u32Val < HostGdtr.cbGdt)
5148 {
5149 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5150 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5151 }
5152
5153 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5154 Log4(("Host TR %#08x\n", u32Val));
5155 if (u32Val < HostGdtr.cbGdt)
5156 {
5157 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5158 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5159 }
5160
5161 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5162 Log4(("Host TR Base %#RHv\n", uHCReg));
5163 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5164 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5165 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5166 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5167 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5168 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5169 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5170 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5171 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5172 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5173 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5174 Log4(("Host RSP %#RHv\n", uHCReg));
5175 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5176 Log4(("Host RIP %#RHv\n", uHCReg));
5177# if HC_ARCH_BITS == 64
5178 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5179 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5180 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5181 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5182 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5183 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5184# endif
5185#endif /* VBOX_STRICT */
5186 break;
5187 }
5188
5189 default:
5190 /* Impossible */
5191 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5192 break;
5193 }
5194}
5195
5196
5197#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5198#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5199# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5200#endif
5201#ifdef VBOX_STRICT
5202static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5203{
5204 switch (idxField)
5205 {
5206 case VMX_VMCS_GUEST_RIP:
5207 case VMX_VMCS_GUEST_RSP:
5208 case VMX_VMCS_GUEST_SYSENTER_EIP:
5209 case VMX_VMCS_GUEST_SYSENTER_ESP:
5210 case VMX_VMCS_GUEST_GDTR_BASE:
5211 case VMX_VMCS_GUEST_IDTR_BASE:
5212 case VMX_VMCS_GUEST_CS_BASE:
5213 case VMX_VMCS_GUEST_DS_BASE:
5214 case VMX_VMCS_GUEST_ES_BASE:
5215 case VMX_VMCS_GUEST_FS_BASE:
5216 case VMX_VMCS_GUEST_GS_BASE:
5217 case VMX_VMCS_GUEST_SS_BASE:
5218 case VMX_VMCS_GUEST_LDTR_BASE:
5219 case VMX_VMCS_GUEST_TR_BASE:
5220 case VMX_VMCS_GUEST_CR3:
5221 return true;
5222 }
5223 return false;
5224}
5225
5226static bool hmR0VmxIsValidReadField(uint32_t idxField)
5227{
5228 switch (idxField)
5229 {
5230 /* Read-only fields. */
5231 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5232 return true;
5233 }
5234 /* Remaining readable fields should also be writable. */
5235 return hmR0VmxIsValidWriteField(idxField);
5236}
5237#endif /* VBOX_STRICT */
5238
5239
5240/**
5241 * Executes the specified handler in 64-bit mode.
5242 *
5243 * @returns VBox status code (no informational status codes).
5244 * @param pVCpu The cross context virtual CPU structure.
5245 * @param enmOp The operation to perform.
5246 * @param cParams Number of parameters.
5247 * @param paParam Array of 32-bit parameters.
5248 */
5249VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
5250{
5251 PVM pVM = pVCpu->CTX_SUFF(pVM);
5252 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5253 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5254 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5255 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5256
5257#ifdef VBOX_STRICT
5258 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5259 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5260
5261 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5262 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5263#endif
5264
5265 /* Disable interrupts. */
5266 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5267
5268#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5269 RTCPUID idHostCpu = RTMpCpuId();
5270 CPUMR0SetLApic(pVCpu, idHostCpu);
5271#endif
5272
5273 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5274 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5275
5276 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5277 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5278 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5279
5280 /* Leave VMX Root Mode. */
5281 VMXDisable();
5282
5283 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5284
5285 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5286 CPUMSetHyperEIP(pVCpu, enmOp);
5287 for (int i = (int)cParams - 1; i >= 0; i--)
5288 CPUMPushHyper(pVCpu, paParam[i]);
5289
5290 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5291
5292 /* Call the switcher. */
5293 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
5294 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5295
5296 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5297 /* Make sure the VMX instructions don't cause #UD faults. */
5298 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5299
5300 /* Re-enter VMX Root Mode */
5301 int rc2 = VMXEnable(HCPhysCpuPage);
5302 if (RT_FAILURE(rc2))
5303 {
5304 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5305 ASMSetFlags(fOldEFlags);
5306 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5307 return rc2;
5308 }
5309
5310 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5311 AssertRC(rc2);
5312 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5313 Assert(!(ASMGetFlags() & X86_EFL_IF));
5314 ASMSetFlags(fOldEFlags);
5315 return rc;
5316}
5317
5318
5319/**
5320 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5321 * supporting 64-bit guests.
5322 *
5323 * @returns VBox status code.
5324 * @param fResume Whether to VMLAUNCH or VMRESUME.
5325 * @param pCtx Pointer to the guest-CPU context.
5326 * @param pCache Pointer to the VMCS cache.
5327 * @param pVM The cross context VM structure.
5328 * @param pVCpu The cross context virtual CPU structure.
5329 */
5330DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5331{
5332 NOREF(fResume);
5333
5334 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5335 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5336
5337#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5338 pCache->uPos = 1;
5339 pCache->interPD = PGMGetInterPaeCR3(pVM);
5340 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5341#endif
5342
5343#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5344 pCache->TestIn.HCPhysCpuPage = 0;
5345 pCache->TestIn.HCPhysVmcs = 0;
5346 pCache->TestIn.pCache = 0;
5347 pCache->TestOut.HCPhysVmcs = 0;
5348 pCache->TestOut.pCache = 0;
5349 pCache->TestOut.pCtx = 0;
5350 pCache->TestOut.eflags = 0;
5351#else
5352 NOREF(pCache);
5353#endif
5354
5355 uint32_t aParam[10];
5356 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5357 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5358 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5359 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5360 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5361 aParam[5] = 0;
5362 aParam[6] = VM_RC_ADDR(pVM, pVM);
5363 aParam[7] = 0;
5364 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5365 aParam[9] = 0;
5366
5367#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5368 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5369 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5370#endif
5371 int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5372
5373#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5374 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5375 Assert(pCtx->dr[4] == 10);
5376 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5377#endif
5378
5379#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5380 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5381 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5382 pVCpu->hm.s.vmx.HCPhysVmcs));
5383 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5384 pCache->TestOut.HCPhysVmcs));
5385 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5386 pCache->TestOut.pCache));
5387 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5388 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5389 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5390 pCache->TestOut.pCtx));
5391 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5392#endif
5393 NOREF(pCtx);
5394 return rc;
5395}
5396
5397
5398/**
5399 * Initialize the VMCS-Read cache.
5400 *
5401 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5402 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5403 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5404 * (those that have a 32-bit FULL & HIGH part).
5405 *
5406 * @returns VBox status code.
5407 * @param pVCpu The cross context virtual CPU structure.
5408 */
5409static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
5410{
5411#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5412 do { \
5413 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5414 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5415 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5416 ++cReadFields; \
5417 } while (0)
5418
5419 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5420 uint32_t cReadFields = 0;
5421
5422 /*
5423 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5424 * and serve to indicate exceptions to the rules.
5425 */
5426
5427 /* Guest-natural selector base fields. */
5428#if 0
5429 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5430 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5431 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5432#endif
5433 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5434 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5435 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5436 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5437 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5438 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5439 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5440 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5441 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5442 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5443 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5444 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5445#if 0
5446 /* Unused natural width guest-state fields. */
5447 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5448 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5449#endif
5450 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5451 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5452
5453 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
5454 these 64-bit fields (using "FULL" and "HIGH" fields). */
5455#if 0
5456 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5457 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5458 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5459 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5460 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5461 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5462 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5463 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5464 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5465#endif
5466
5467 /* Natural width guest-state fields. */
5468 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5469#if 0
5470 /* Currently unused field. */
5471 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5472#endif
5473
5474 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5475 {
5476 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5477 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5478 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5479 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5480 }
5481 else
5482 {
5483 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5484 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5485 }
5486
5487#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5488 return VINF_SUCCESS;
5489}
5490
5491
5492/**
5493 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5494 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5495 * darwin, running 64-bit guests).
5496 *
5497 * @returns VBox status code.
5498 * @param pVCpu The cross context virtual CPU structure.
5499 * @param idxField The VMCS field encoding.
5500 * @param u64Val 16, 32 or 64-bit value.
5501 */
5502VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5503{
5504 int rc;
5505 switch (idxField)
5506 {
5507 /*
5508 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5509 */
5510 /* 64-bit Control fields. */
5511 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5512 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5513 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5514 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5515 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5516 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5517 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5518 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5519 case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
5520 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5521 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5522 case VMX_VMCS64_CTRL_EPTP_FULL:
5523 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5524 /* 64-bit Guest-state fields. */
5525 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5526 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5527 case VMX_VMCS64_GUEST_PAT_FULL:
5528 case VMX_VMCS64_GUEST_EFER_FULL:
5529 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5530 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5531 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5532 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5533 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5534 /* 64-bit Host-state fields. */
5535 case VMX_VMCS64_HOST_PAT_FULL:
5536 case VMX_VMCS64_HOST_EFER_FULL:
5537 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5538 {
5539 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5540 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5541 break;
5542 }
5543
5544 /*
5545 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5546 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5547 */
5548 /* Natural-width Guest-state fields. */
5549 case VMX_VMCS_GUEST_CR3:
5550 case VMX_VMCS_GUEST_ES_BASE:
5551 case VMX_VMCS_GUEST_CS_BASE:
5552 case VMX_VMCS_GUEST_SS_BASE:
5553 case VMX_VMCS_GUEST_DS_BASE:
5554 case VMX_VMCS_GUEST_FS_BASE:
5555 case VMX_VMCS_GUEST_GS_BASE:
5556 case VMX_VMCS_GUEST_LDTR_BASE:
5557 case VMX_VMCS_GUEST_TR_BASE:
5558 case VMX_VMCS_GUEST_GDTR_BASE:
5559 case VMX_VMCS_GUEST_IDTR_BASE:
5560 case VMX_VMCS_GUEST_RSP:
5561 case VMX_VMCS_GUEST_RIP:
5562 case VMX_VMCS_GUEST_SYSENTER_ESP:
5563 case VMX_VMCS_GUEST_SYSENTER_EIP:
5564 {
5565 if (!(RT_HI_U32(u64Val)))
5566 {
5567 /* If this field is 64-bit, VT-x will zero out the top bits. */
5568 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5569 }
5570 else
5571 {
5572 /* Assert that only the 32->64 switcher case should ever come here. */
5573 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5574 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5575 }
5576 break;
5577 }
5578
5579 default:
5580 {
5581 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5582 rc = VERR_INVALID_PARAMETER;
5583 break;
5584 }
5585 }
5586 AssertRCReturn(rc, rc);
5587 return rc;
5588}
5589
5590
5591/**
5592 * Queue up a VMWRITE by using the VMCS write cache.
5593 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5594 *
5595 * @param pVCpu The cross context virtual CPU structure.
5596 * @param idxField The VMCS field encoding.
5597 * @param u64Val 16, 32 or 64-bit value.
5598 */
5599VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5600{
5601 AssertPtr(pVCpu);
5602 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5603
5604 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5605 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5606
5607 /* Make sure there are no duplicates. */
5608 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5609 {
5610 if (pCache->Write.aField[i] == idxField)
5611 {
5612 pCache->Write.aFieldVal[i] = u64Val;
5613 return VINF_SUCCESS;
5614 }
5615 }
5616
5617 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5618 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5619 pCache->Write.cValidEntries++;
5620 return VINF_SUCCESS;
5621}
5622#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5623
5624
5625/**
5626 * Sets up the usage of TSC-offsetting and updates the VMCS.
5627 *
5628 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5629 * VMX preemption timer.
5630 *
5631 * @returns VBox status code.
5632 * @param pVCpu The cross context virtual CPU structure.
5633 *
5634 * @remarks No-long-jump zone!!!
5635 */
5636static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5637{
5638 bool fOffsettedTsc;
5639 bool fParavirtTsc;
5640 PVM pVM = pVCpu->CTX_SUFF(pVM);
5641 uint64_t uTscOffset;
5642 if (pVM->hm.s.vmx.fUsePreemptTimer)
5643 {
5644 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
5645
5646 /* Make sure the returned values have sane upper and lower boundaries. */
5647 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5648 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5649 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5650 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5651
5652 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5653 int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
5654 AssertRC(rc);
5655 }
5656 else
5657 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
5658
5659 if (fParavirtTsc)
5660 {
5661 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5662 information before every VM-entry, hence disable it for performance sake. */
5663#if 0
5664 int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5665 AssertRC(rc);
5666#endif
5667 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5668 }
5669
5670 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
5671 if ( fOffsettedTsc
5672 && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5673 {
5674 if (pVCpu->hm.s.vmx.u64TscOffset != uTscOffset)
5675 {
5676 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
5677 AssertRC(rc);
5678 pVCpu->hm.s.vmx.u64TscOffset = uTscOffset;
5679 }
5680
5681 if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
5682 {
5683 uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
5684 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5685 AssertRC(rc);
5686 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5687 }
5688 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5689 }
5690 else
5691 {
5692 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5693 if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
5694 {
5695 uProcCtls |= VMX_PROC_CTLS_RDTSC_EXIT;
5696 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5697 AssertRC(rc);
5698 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5699 }
5700 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5701 }
5702}
5703
5704
5705/**
5706 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5707 * VM-exit interruption info type.
5708 *
5709 * @returns The IEM exception flags.
5710 * @param uVector The event vector.
5711 * @param uVmxVectorType The VMX event type.
5712 *
5713 * @remarks This function currently only constructs flags required for
5714 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5715 * and CR2 aspects of an exception are not included).
5716 */
5717static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5718{
5719 uint32_t fIemXcptFlags;
5720 switch (uVmxVectorType)
5721 {
5722 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5723 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5724 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5725 break;
5726
5727 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5728 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5729 break;
5730
5731 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5732 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5733 break;
5734
5735 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5736 {
5737 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5738 if (uVector == X86_XCPT_BP)
5739 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5740 else if (uVector == X86_XCPT_OF)
5741 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5742 else
5743 {
5744 fIemXcptFlags = 0;
5745 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5746 }
5747 break;
5748 }
5749
5750 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5751 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5752 break;
5753
5754 default:
5755 fIemXcptFlags = 0;
5756 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5757 break;
5758 }
5759 return fIemXcptFlags;
5760}
5761
5762
5763/**
5764 * Sets an event as a pending event to be injected into the guest.
5765 *
5766 * @param pVCpu The cross context virtual CPU structure.
5767 * @param u32IntInfo The VM-entry interruption-information field.
5768 * @param cbInstr The VM-entry instruction length in bytes (for software
5769 * interrupts, exceptions and privileged software
5770 * exceptions).
5771 * @param u32ErrCode The VM-entry exception error code.
5772 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5773 * page-fault.
5774 *
5775 * @remarks Statistics counter assumes this is a guest event being injected or
5776 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5777 * always incremented.
5778 */
5779DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5780 RTGCUINTPTR GCPtrFaultAddress)
5781{
5782 Assert(!pVCpu->hm.s.Event.fPending);
5783 pVCpu->hm.s.Event.fPending = true;
5784 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5785 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5786 pVCpu->hm.s.Event.cbInstr = cbInstr;
5787 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5788}
5789
5790
5791/**
5792 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5793 *
5794 * @param pVCpu The cross context virtual CPU structure.
5795 */
5796DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
5797{
5798 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
5799 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5800 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
5801 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5802 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5803}
5804
5805
5806/**
5807 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
5808 *
5809 * @param pVCpu The cross context virtual CPU structure.
5810 */
5811DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
5812{
5813 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
5814 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5815 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
5816 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5817 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5818}
5819
5820
5821/**
5822 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
5823 *
5824 * @param pVCpu The cross context virtual CPU structure.
5825 */
5826DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
5827{
5828 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
5829 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5830 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
5831 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5832 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5833}
5834
5835
5836#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5837/**
5838 * Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
5839 *
5840 * @param pVCpu The cross context virtual CPU structure.
5841 * @param u32ErrCode The error code for the general-protection exception.
5842 */
5843DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
5844{
5845 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
5846 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5847 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
5848 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5849 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrCode, 0 /* GCPtrFaultAddress */);
5850}
5851
5852
5853/**
5854 * Sets a stack (\#SS) exception as pending-for-injection into the VM.
5855 *
5856 * @param pVCpu The cross context virtual CPU structure.
5857 * @param u32ErrCode The error code for the stack exception.
5858 */
5859DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
5860{
5861 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
5862 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5863 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
5864 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5865 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrCode, 0 /* GCPtrFaultAddress */);
5866}
5867
5868
5869/**
5870 * Decodes the memory operand of an instruction that caused a VM-exit.
5871 *
5872 * The VM-exit qualification field provides the displacement field for memory
5873 * operand instructions, if any.
5874 *
5875 * @returns Strict VBox status code (i.e. informational status codes too).
5876 * @retval VINF_SUCCESS if the operand was successfully decoded.
5877 * @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
5878 * operand.
5879 * @param pVCpu The cross context virtual CPU structure.
5880 * @param pExitInstrInfo Pointer to the VM-exit instruction information.
5881 * @param fIsDstOperand Whether the operand is a destination memory
5882 * operand (i.e. writeable memory location) or not.
5883 * @param GCPtrDisp The instruction displacement field, if any. For
5884 * RIP-relative addressing pass RIP + displacement here.
5885 * @param pGCPtrMem Where to store the effective destination memory address.
5886 */
5887static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, PCVMXEXITINSTRINFO pExitInstrInfo, RTGCPTR GCPtrDisp,
5888 bool fIsDstOperand, PRTGCPTR pGCPtrMem)
5889{
5890 Assert(pExitInstrInfo);
5891 Assert(pGCPtrMem);
5892 Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
5893
5894 static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
5895 static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
5896 AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
5897
5898 uint8_t const uAddrSize = pExitInstrInfo->All.u3AddrSize;
5899 uint8_t const iSegReg = pExitInstrInfo->All.iSegReg;
5900 bool const fIdxRegValid = !pExitInstrInfo->All.fIdxRegInvalid;
5901 uint8_t const iIdxReg = pExitInstrInfo->All.iIdxReg;
5902 uint8_t const uScale = pExitInstrInfo->All.u2Scaling;
5903 bool const fBaseRegValid = !pExitInstrInfo->All.fBaseRegInvalid;
5904 uint8_t const iBaseReg = pExitInstrInfo->All.iBaseReg;
5905 bool const fIsMemOperand = !pExitInstrInfo->All.fIsRegOperand;
5906 bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
5907
5908 /*
5909 * Validate instruction information.
5910 * This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
5911 */
5912 AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
5913 ("Invalid address size. ExitInstrInfo=%#RX32\n", pExitInstrInfo->u), VERR_VMX_IPE_1);
5914 AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
5915 ("Invalid segment register. ExitInstrInfo=%#RX32\n", pExitInstrInfo->u), VERR_VMX_IPE_2);
5916 AssertLogRelMsgReturn(fIsMemOperand,
5917 ("Expected memory operand. ExitInstrInfo=%#RX32\n", pExitInstrInfo->u), VERR_VMX_IPE_3);
5918
5919 /*
5920 * Compute the complete effective address.
5921 *
5922 * See AMD instruction spec. 1.4.2 "SIB Byte Format"
5923 * See AMD spec. 4.5.2 "Segment Registers".
5924 */
5925 RTGCPTR GCPtrMem = GCPtrDisp;
5926 if (fBaseRegValid)
5927 GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
5928 if (fIdxRegValid)
5929 GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
5930
5931 RTGCPTR const GCPtrOff = GCPtrMem;
5932 if ( !fIsLongMode
5933 || iSegReg >= X86_SREG_FS)
5934 GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
5935 GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
5936
5937 /*
5938 * Validate effective address.
5939 * See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
5940 */
5941 uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
5942 Assert(cbAccess > 0);
5943 if (fIsLongMode)
5944 {
5945 if (X86_IS_CANONICAL(GCPtrMem))
5946 {
5947 *pGCPtrMem = GCPtrMem;
5948 return VINF_SUCCESS;
5949 }
5950
5951 /** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
5952 * "Data Limit Checks in 64-bit Mode". */
5953 Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
5954 hmR0VmxSetPendingXcptGP(pVCpu, 0);
5955 return VINF_HM_PENDING_XCPT;
5956 }
5957
5958 /*
5959 * This is a watered down version of iemMemApplySegment().
5960 * Parts that are not applicable for VMX instructions like real-or-v8086 mode
5961 * and segment CPL/DPL checks are skipped.
5962 */
5963 RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
5964 RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
5965 PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
5966
5967 /* Check if the segment is present and usable. */
5968 if ( pSel->Attr.n.u1Present
5969 && !pSel->Attr.n.u1Unusable)
5970 {
5971 Assert(pSel->Attr.n.u1DescType);
5972 if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
5973 {
5974 /* Check permissions for the data segment. */
5975 if ( fIsDstOperand
5976 && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
5977 {
5978 Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
5979 hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
5980 return VINF_HM_PENDING_XCPT;
5981 }
5982
5983 /* Check limits if it's a normal data segment. */
5984 if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
5985 {
5986 if ( GCPtrFirst32 > pSel->u32Limit
5987 || GCPtrLast32 > pSel->u32Limit)
5988 {
5989 Log4Func(("Data segment limit exceeded."
5990 "iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
5991 GCPtrLast32, pSel->u32Limit));
5992 if (iSegReg == X86_SREG_SS)
5993 hmR0VmxSetPendingXcptSS(pVCpu, 0);
5994 else
5995 hmR0VmxSetPendingXcptGP(pVCpu, 0);
5996 return VINF_HM_PENDING_XCPT;
5997 }
5998 }
5999 else
6000 {
6001 /* Check limits if it's an expand-down data segment.
6002 Note! The upper boundary is defined by the B bit, not the G bit! */
6003 if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
6004 || GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
6005 {
6006 Log4Func(("Expand-down data segment limit exceeded."
6007 "iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6008 GCPtrLast32, pSel->u32Limit));
6009 if (iSegReg == X86_SREG_SS)
6010 hmR0VmxSetPendingXcptSS(pVCpu, 0);
6011 else
6012 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6013 return VINF_HM_PENDING_XCPT;
6014 }
6015 }
6016 }
6017 else
6018 {
6019 /* Check permissions for the code segment. */
6020 if ( fIsDstOperand
6021 || !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ))
6022 {
6023 Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
6024 Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
6025 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6026 return VINF_HM_PENDING_XCPT;
6027 }
6028
6029 /* Check limits for the code segment (normal/expand-down not applicable for code segments). */
6030 if ( GCPtrFirst32 > pSel->u32Limit
6031 || GCPtrLast32 > pSel->u32Limit)
6032 {
6033 Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
6034 GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
6035 if (iSegReg == X86_SREG_SS)
6036 hmR0VmxSetPendingXcptSS(pVCpu, 0);
6037 else
6038 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6039 return VINF_HM_PENDING_XCPT;
6040 }
6041 }
6042 }
6043 else
6044 {
6045 Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6046 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6047 return VINF_HM_PENDING_XCPT;
6048 }
6049
6050 *pGCPtrMem = GCPtrMem;
6051 return VINF_SUCCESS;
6052}
6053
6054
6055/**
6056 * Perform the relevant VMX instruction checks for VM-exits that occurred due to the
6057 * guest attempting to execute a VMX instruction.
6058 *
6059 * @returns Strict VBox status code (i.e. informational status codes too).
6060 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
6061 * @retval VINF_HM_PENDING_XCPT if an exception was raised.
6062 *
6063 * @param pVCpu The cross context virtual CPU structure.
6064 * @param uExitReason The VM-exit reason.
6065 *
6066 * @todo NstVmx: Document other error codes when VM-exit is implemented.
6067 * @remarks No-long-jump zone!!!
6068 */
6069static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
6070{
6071 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS
6072 | CPUMCTX_EXTRN_HWVIRT);
6073
6074 if ( CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx)
6075 || ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6076 && !CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx)))
6077 {
6078 Log4Func(("In real/v86-mode or long-mode outside 64-bit code segment -> #UD\n"));
6079 hmR0VmxSetPendingXcptUD(pVCpu);
6080 return VINF_HM_PENDING_XCPT;
6081 }
6082
6083 if (uExitReason == VMX_EXIT_VMXON)
6084 {
6085 /*
6086 * We check CR4.VMXE because it is required to be always set while in VMX operation
6087 * by physical CPUs and our CR4 read shadow is only consulted when executing specific
6088 * instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
6089 * otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
6090 */
6091 if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
6092 {
6093 Log4Func(("CR4.VMXE is not set -> #UD\n"));
6094 hmR0VmxSetPendingXcptUD(pVCpu);
6095 return VINF_HM_PENDING_XCPT;
6096 }
6097 }
6098 else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
6099 {
6100 /*
6101 * The guest has not entered VMX operation but attempted to execute a VMX instruction
6102 * (other than VMXON), we need to raise a #UD.
6103 */
6104 Log4Func(("Not in VMX root mode -> #UD\n"));
6105 hmR0VmxSetPendingXcptUD(pVCpu);
6106 return VINF_HM_PENDING_XCPT;
6107 }
6108
6109 if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
6110 {
6111 /*
6112 * The nested-guest attempted to execute a VMX instruction, cause a VM-exit and let
6113 * the guest hypervisor deal with it.
6114 */
6115 /** @todo NSTVMX: Trigger a VM-exit */
6116 }
6117
6118 /*
6119 * VMX instructions require CPL 0 except in VMX non-root mode where the VM-exit intercept
6120 * (above) takes preceedence over the CPL check.
6121 */
6122 if (CPUMGetGuestCPL(pVCpu) > 0)
6123 {
6124 Log4Func(("CPL > 0 -> #GP(0)\n"));
6125 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6126 return VINF_HM_PENDING_XCPT;
6127 }
6128
6129 return VINF_SUCCESS;
6130}
6131
6132#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
6133
6134
6135/**
6136 * Handle a condition that occurred while delivering an event through the guest
6137 * IDT.
6138 *
6139 * @returns Strict VBox status code (i.e. informational status codes too).
6140 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
6141 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
6142 * to continue execution of the guest which will delivery the \#DF.
6143 * @retval VINF_EM_RESET if we detected a triple-fault condition.
6144 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
6145 *
6146 * @param pVCpu The cross context virtual CPU structure.
6147 * @param pVmxTransient Pointer to the VMX transient structure.
6148 *
6149 * @remarks No-long-jump zone!!!
6150 */
6151static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6152{
6153 uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
6154
6155 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
6156 rc2 |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
6157 AssertRCReturn(rc2, rc2);
6158
6159 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
6160 if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6161 {
6162 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
6163 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
6164
6165 /*
6166 * If the event was a software interrupt (generated with INT n) or a software exception
6167 * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
6168 * can handle the VM-exit and continue guest execution which will re-execute the
6169 * instruction rather than re-injecting the exception, as that can cause premature
6170 * trips to ring-3 before injection and involve TRPM which currently has no way of
6171 * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
6172 * the problem).
6173 */
6174 IEMXCPTRAISE enmRaise;
6175 IEMXCPTRAISEINFO fRaiseInfo;
6176 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6177 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6178 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6179 {
6180 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
6181 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6182 }
6183 else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
6184 {
6185 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
6186 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
6187 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
6188 /** @todo Make AssertMsgReturn as just AssertMsg later. */
6189 AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
6190 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
6191 uExitVectorType), VERR_VMX_IPE_5);
6192
6193 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
6194
6195 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
6196 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
6197 {
6198 pVmxTransient->fVectoringPF = true;
6199 enmRaise = IEMXCPTRAISE_PREV_EVENT;
6200 }
6201 }
6202 else
6203 {
6204 /*
6205 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
6206 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
6207 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
6208 */
6209 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6210 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6211 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
6212 enmRaise = IEMXCPTRAISE_PREV_EVENT;
6213 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6214 }
6215
6216 /*
6217 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
6218 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
6219 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
6220 * subsequent VM-entry would fail.
6221 *
6222 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
6223 */
6224 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
6225 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6226 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
6227 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
6228 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
6229 {
6230 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6231 }
6232
6233 switch (enmRaise)
6234 {
6235 case IEMXCPTRAISE_CURRENT_XCPT:
6236 {
6237 Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
6238 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
6239 Assert(rcStrict == VINF_SUCCESS);
6240 break;
6241 }
6242
6243 case IEMXCPTRAISE_PREV_EVENT:
6244 {
6245 uint32_t u32ErrCode;
6246 if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
6247 {
6248 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6249 AssertRCReturn(rc2, rc2);
6250 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6251 }
6252 else
6253 u32ErrCode = 0;
6254
6255 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
6256 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6257 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6258 0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
6259
6260 Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
6261 pVCpu->hm.s.Event.u32ErrCode));
6262 Assert(rcStrict == VINF_SUCCESS);
6263 break;
6264 }
6265
6266 case IEMXCPTRAISE_REEXEC_INSTR:
6267 Assert(rcStrict == VINF_SUCCESS);
6268 break;
6269
6270 case IEMXCPTRAISE_DOUBLE_FAULT:
6271 {
6272 /*
6273 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
6274 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
6275 */
6276 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6277 {
6278 pVmxTransient->fVectoringDoublePF = true;
6279 Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
6280 pVCpu->cpum.GstCtx.cr2));
6281 rcStrict = VINF_SUCCESS;
6282 }
6283 else
6284 {
6285 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6286 hmR0VmxSetPendingXcptDF(pVCpu);
6287 Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
6288 uIdtVector, uExitVector));
6289 rcStrict = VINF_HM_DOUBLE_FAULT;
6290 }
6291 break;
6292 }
6293
6294 case IEMXCPTRAISE_TRIPLE_FAULT:
6295 {
6296 Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
6297 rcStrict = VINF_EM_RESET;
6298 break;
6299 }
6300
6301 case IEMXCPTRAISE_CPU_HANG:
6302 {
6303 Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
6304 rcStrict = VERR_EM_GUEST_CPU_HANG;
6305 break;
6306 }
6307
6308 default:
6309 {
6310 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6311 rcStrict = VERR_VMX_IPE_2;
6312 break;
6313 }
6314 }
6315 }
6316 else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6317 && VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6318 && uExitVector != X86_XCPT_DF
6319 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
6320 {
6321 /*
6322 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6323 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6324 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6325 */
6326 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6327 {
6328 Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
6329 VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6330 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6331 }
6332 }
6333
6334 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
6335 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
6336 return rcStrict;
6337}
6338
6339
6340/**
6341 * Imports a guest segment register from the current VMCS into
6342 * the guest-CPU context.
6343 *
6344 * @returns VBox status code.
6345 * @param pVCpu The cross context virtual CPU structure.
6346 * @param idxSel Index of the selector in the VMCS.
6347 * @param idxLimit Index of the segment limit in the VMCS.
6348 * @param idxBase Index of the segment base in the VMCS.
6349 * @param idxAccess Index of the access rights of the segment in the VMCS.
6350 * @param pSelReg Pointer to the segment selector.
6351 *
6352 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6353 * do not log!
6354 *
6355 * @remarks Never call this function directly!!! Use the
6356 * HMVMX_IMPORT_SREG() macro as that takes care
6357 * of whether to read from the VMCS cache or not.
6358 */
6359static int hmR0VmxImportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6360 PCPUMSELREG pSelReg)
6361{
6362 NOREF(pVCpu);
6363
6364 uint32_t u32Sel;
6365 uint32_t u32Limit;
6366 uint32_t u32Attr;
6367 uint64_t u64Base;
6368 int rc = VMXReadVmcs32(idxSel, &u32Sel);
6369 rc |= VMXReadVmcs32(idxLimit, &u32Limit);
6370 rc |= VMXReadVmcs32(idxAccess, &u32Attr);
6371 rc |= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
6372 AssertRCReturn(rc, rc);
6373
6374 pSelReg->Sel = (uint16_t)u32Sel;
6375 pSelReg->ValidSel = (uint16_t)u32Sel;
6376 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6377 pSelReg->u32Limit = u32Limit;
6378 pSelReg->u64Base = u64Base;
6379 pSelReg->Attr.u = u32Attr;
6380
6381 /*
6382 * If VT-x marks the segment as unusable, most other bits remain undefined:
6383 * - For CS the L, D and G bits have meaning.
6384 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6385 * - For the remaining data segments no bits are defined.
6386 *
6387 * The present bit and the unusable bit has been observed to be set at the
6388 * same time (the selector was supposed to be invalid as we started executing
6389 * a V8086 interrupt in ring-0).
6390 *
6391 * What should be important for the rest of the VBox code, is that the P bit is
6392 * cleared. Some of the other VBox code recognizes the unusable bit, but
6393 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6394 * safe side here, we'll strip off P and other bits we don't care about. If
6395 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6396 *
6397 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6398 */
6399 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6400 {
6401 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6402
6403 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6404 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6405 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6406#ifdef VBOX_STRICT
6407 VMMRZCallRing3Disable(pVCpu);
6408 Log4Func(("Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Sel, pSelReg->Attr.u));
6409# ifdef DEBUG_bird
6410 AssertMsg((u32Attr & ~X86DESCATTR_P) == pSelReg->Attr.u,
6411 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6412 idxSel, u32Sel, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6413# endif
6414 VMMRZCallRing3Enable(pVCpu);
6415#endif
6416 }
6417 return VINF_SUCCESS;
6418}
6419
6420
6421/**
6422 * Imports the guest RIP from the VMCS back into the guest-CPU context.
6423 *
6424 * @returns VBox status code.
6425 * @param pVCpu The cross context virtual CPU structure.
6426 *
6427 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6428 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6429 * instead!!!
6430 */
6431DECLINLINE(int) hmR0VmxImportGuestRip(PVMCPU pVCpu)
6432{
6433 uint64_t u64Val;
6434 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6435 if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
6436 {
6437 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6438 if (RT_SUCCESS(rc))
6439 {
6440 pCtx->rip = u64Val;
6441 EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
6442 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
6443 }
6444 return rc;
6445 }
6446 return VINF_SUCCESS;
6447}
6448
6449
6450/**
6451 * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
6452 *
6453 * @returns VBox status code.
6454 * @param pVCpu The cross context virtual CPU structure.
6455 *
6456 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6457 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6458 * instead!!!
6459 */
6460DECLINLINE(int) hmR0VmxImportGuestRFlags(PVMCPU pVCpu)
6461{
6462 uint32_t u32Val;
6463 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6464 if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
6465 {
6466 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
6467 if (RT_SUCCESS(rc))
6468 {
6469 pCtx->eflags.u32 = u32Val;
6470
6471 /* Restore eflags for real-on-v86-mode hack. */
6472 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6473 {
6474 pCtx->eflags.Bits.u1VM = 0;
6475 pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6476 }
6477 }
6478 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
6479 return rc;
6480 }
6481 return VINF_SUCCESS;
6482}
6483
6484
6485/**
6486 * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
6487 * context.
6488 *
6489 * @returns VBox status code.
6490 * @param pVCpu The cross context virtual CPU structure.
6491 *
6492 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6493 * do not log!
6494 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6495 * instead!!!
6496 */
6497DECLINLINE(int) hmR0VmxImportGuestIntrState(PVMCPU pVCpu)
6498{
6499 uint32_t u32Val;
6500 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6501 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
6502 if (RT_SUCCESS(rc))
6503 {
6504 /*
6505 * We additionally have a requirement to import RIP, RFLAGS depending on whether we
6506 * might need them in hmR0VmxEvaluatePendingEvent().
6507 */
6508 if (!u32Val)
6509 {
6510 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6511 {
6512 rc = hmR0VmxImportGuestRip(pVCpu);
6513 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6514 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6515 }
6516
6517 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6518 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6519 }
6520 else
6521 {
6522 rc = hmR0VmxImportGuestRip(pVCpu);
6523 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6524
6525 if (u32Val & ( VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
6526 | VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
6527 {
6528 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
6529 }
6530 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6531 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6532
6533 if (u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
6534 {
6535 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6536 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6537 }
6538 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6539 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6540 }
6541 }
6542 return rc;
6543}
6544
6545
6546/**
6547 * Worker for VMXR0ImportStateOnDemand.
6548 *
6549 * @returns VBox status code.
6550 * @param pVCpu The cross context virtual CPU structure.
6551 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6552 */
6553static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
6554{
6555#define VMXLOCAL_BREAK_RC(a_rc) \
6556 if (RT_FAILURE(a_rc)) \
6557 break
6558
6559 int rc = VINF_SUCCESS;
6560 PVM pVM = pVCpu->CTX_SUFF(pVM);
6561 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6562 uint64_t u64Val;
6563 uint32_t u32Val;
6564
6565 Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
6566 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
6567
6568 /*
6569 * We disable interrupts to make the updating of the state and in particular
6570 * the fExtrn modification atomic wrt to preemption hooks.
6571 */
6572 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
6573
6574 fWhat &= pCtx->fExtrn;
6575 if (fWhat)
6576 {
6577 do
6578 {
6579 if (fWhat & CPUMCTX_EXTRN_RIP)
6580 {
6581 rc = hmR0VmxImportGuestRip(pVCpu);
6582 VMXLOCAL_BREAK_RC(rc);
6583 }
6584
6585 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
6586 {
6587 rc = hmR0VmxImportGuestRFlags(pVCpu);
6588 VMXLOCAL_BREAK_RC(rc);
6589 }
6590
6591 if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
6592 {
6593 rc = hmR0VmxImportGuestIntrState(pVCpu);
6594 VMXLOCAL_BREAK_RC(rc);
6595 }
6596
6597 if (fWhat & CPUMCTX_EXTRN_RSP)
6598 {
6599 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6600 VMXLOCAL_BREAK_RC(rc);
6601 pCtx->rsp = u64Val;
6602 }
6603
6604 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
6605 {
6606 if (fWhat & CPUMCTX_EXTRN_CS)
6607 {
6608 rc = HMVMX_IMPORT_SREG(CS, &pCtx->cs);
6609 rc |= hmR0VmxImportGuestRip(pVCpu);
6610 VMXLOCAL_BREAK_RC(rc);
6611 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6612 pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6613 EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true);
6614 }
6615 if (fWhat & CPUMCTX_EXTRN_SS)
6616 {
6617 rc = HMVMX_IMPORT_SREG(SS, &pCtx->ss);
6618 VMXLOCAL_BREAK_RC(rc);
6619 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6620 pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6621 }
6622 if (fWhat & CPUMCTX_EXTRN_DS)
6623 {
6624 rc = HMVMX_IMPORT_SREG(DS, &pCtx->ds);
6625 VMXLOCAL_BREAK_RC(rc);
6626 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6627 pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6628 }
6629 if (fWhat & CPUMCTX_EXTRN_ES)
6630 {
6631 rc = HMVMX_IMPORT_SREG(ES, &pCtx->es);
6632 VMXLOCAL_BREAK_RC(rc);
6633 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6634 pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6635 }
6636 if (fWhat & CPUMCTX_EXTRN_FS)
6637 {
6638 rc = HMVMX_IMPORT_SREG(FS, &pCtx->fs);
6639 VMXLOCAL_BREAK_RC(rc);
6640 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6641 pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6642 }
6643 if (fWhat & CPUMCTX_EXTRN_GS)
6644 {
6645 rc = HMVMX_IMPORT_SREG(GS, &pCtx->gs);
6646 VMXLOCAL_BREAK_RC(rc);
6647 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6648 pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6649 }
6650 }
6651
6652 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
6653 {
6654 if (fWhat & CPUMCTX_EXTRN_LDTR)
6655 {
6656 rc = HMVMX_IMPORT_SREG(LDTR, &pCtx->ldtr);
6657 VMXLOCAL_BREAK_RC(rc);
6658 }
6659
6660 if (fWhat & CPUMCTX_EXTRN_GDTR)
6661 {
6662 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6663 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
6664 VMXLOCAL_BREAK_RC(rc);
6665 pCtx->gdtr.pGdt = u64Val;
6666 pCtx->gdtr.cbGdt = u32Val;
6667 }
6668
6669 /* Guest IDTR. */
6670 if (fWhat & CPUMCTX_EXTRN_IDTR)
6671 {
6672 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6673 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
6674 VMXLOCAL_BREAK_RC(rc);
6675 pCtx->idtr.pIdt = u64Val;
6676 pCtx->idtr.cbIdt = u32Val;
6677 }
6678
6679 /* Guest TR. */
6680 if (fWhat & CPUMCTX_EXTRN_TR)
6681 {
6682 /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
6683 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6684 {
6685 rc = HMVMX_IMPORT_SREG(TR, &pCtx->tr);
6686 VMXLOCAL_BREAK_RC(rc);
6687 }
6688 }
6689 }
6690
6691 if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
6692 {
6693 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
6694 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
6695 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
6696 pCtx->SysEnter.cs = u32Val;
6697 VMXLOCAL_BREAK_RC(rc);
6698 }
6699
6700#if HC_ARCH_BITS == 64
6701 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
6702 {
6703 if ( pVM->hm.s.fAllow64BitGuests
6704 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6705 pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
6706 }
6707
6708 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
6709 {
6710 if ( pVM->hm.s.fAllow64BitGuests
6711 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6712 {
6713 pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
6714 pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
6715 pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
6716 }
6717 }
6718#endif
6719
6720 if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
6721#if HC_ARCH_BITS == 32
6722 || (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS))
6723#endif
6724 )
6725 {
6726 PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6727 uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
6728 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6729 {
6730 switch (pMsr->u32Msr)
6731 {
6732#if HC_ARCH_BITS == 32
6733 case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
6734 case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
6735 case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
6736 case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6737#endif
6738 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6739 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6740 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit */ break;
6741 default:
6742 {
6743 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6744 ASMSetFlags(fEFlags);
6745 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
6746 cMsrs));
6747 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6748 }
6749 }
6750 }
6751 }
6752
6753 if (fWhat & CPUMCTX_EXTRN_DR7)
6754 {
6755 if (!pVCpu->hm.s.fUsingHyperDR7)
6756 {
6757 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6758 rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
6759 VMXLOCAL_BREAK_RC(rc);
6760 pCtx->dr[7] = u32Val;
6761 }
6762 }
6763
6764 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
6765 {
6766 uint32_t u32Shadow;
6767 if (fWhat & CPUMCTX_EXTRN_CR0)
6768 {
6769 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
6770 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
6771 VMXLOCAL_BREAK_RC(rc);
6772 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr0Mask)
6773 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr0Mask);
6774 VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
6775 CPUMSetGuestCR0(pVCpu, u32Val);
6776 VMMRZCallRing3Enable(pVCpu);
6777 }
6778
6779 if (fWhat & CPUMCTX_EXTRN_CR4)
6780 {
6781 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
6782 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
6783 VMXLOCAL_BREAK_RC(rc);
6784 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr4Mask)
6785 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr4Mask);
6786 CPUMSetGuestCR4(pVCpu, u32Val);
6787 }
6788
6789 if (fWhat & CPUMCTX_EXTRN_CR3)
6790 {
6791 /* CR0.PG bit changes are always intercepted, so it's up to date. */
6792 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6793 || ( pVM->hm.s.fNestedPaging
6794 && CPUMIsGuestPagingEnabledEx(pCtx)))
6795 {
6796 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6797 if (pCtx->cr3 != u64Val)
6798 {
6799 CPUMSetGuestCR3(pVCpu, u64Val);
6800 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6801 }
6802
6803 /* If the guest is in PAE mode, sync back the PDPE's into the guest state.
6804 Note: CR4.PAE, CR0.PG, EFER bit changes are always intercepted, so they're up to date. */
6805 if (CPUMIsGuestInPAEModeEx(pCtx))
6806 {
6807 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6808 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6809 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6810 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6811 VMXLOCAL_BREAK_RC(rc);
6812 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6813 }
6814 }
6815 }
6816 }
6817 } while (0);
6818
6819 if (RT_SUCCESS(rc))
6820 {
6821 /* Update fExtrn. */
6822 pCtx->fExtrn &= ~fWhat;
6823
6824 /* If everything has been imported, clear the HM keeper bit. */
6825 if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
6826 {
6827 pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
6828 Assert(!pCtx->fExtrn);
6829 }
6830 }
6831 }
6832 else
6833 AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
6834
6835 ASMSetFlags(fEFlags);
6836
6837 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
6838
6839 /*
6840 * Honor any pending CR3 updates.
6841 *
6842 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6843 * -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6844 * -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
6845 *
6846 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6847 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6848 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6849 * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
6850 *
6851 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6852 */
6853 if (VMMRZCallRing3IsEnabled(pVCpu))
6854 {
6855 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6856 {
6857 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
6858 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6859 }
6860
6861 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6862 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6863
6864 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6865 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6866 }
6867
6868 return VINF_SUCCESS;
6869#undef VMXLOCAL_BREAK_RC
6870}
6871
6872
6873/**
6874 * Saves the guest state from the VMCS into the guest-CPU context.
6875 *
6876 * @returns VBox status code.
6877 * @param pVCpu The cross context virtual CPU structure.
6878 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6879 */
6880VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
6881{
6882 return hmR0VmxImportGuestState(pVCpu, fWhat);
6883}
6884
6885
6886/**
6887 * Check per-VM and per-VCPU force flag actions that require us to go back to
6888 * ring-3 for one reason or another.
6889 *
6890 * @returns Strict VBox status code (i.e. informational status codes too)
6891 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6892 * ring-3.
6893 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6894 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6895 * interrupts)
6896 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6897 * all EMTs to be in ring-3.
6898 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6899 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6900 * to the EM loop.
6901 *
6902 * @param pVCpu The cross context virtual CPU structure.
6903 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6904 */
6905static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
6906{
6907 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6908
6909 /*
6910 * Anything pending? Should be more likely than not if we're doing a good job.
6911 */
6912 PVM pVM = pVCpu->CTX_SUFF(pVM);
6913 if ( !fStepping
6914 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6915 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6916 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6917 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6918 return VINF_SUCCESS;
6919
6920 /* Pending PGM C3 sync. */
6921 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6922 {
6923 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6924 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
6925 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
6926 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6927 if (rcStrict2 != VINF_SUCCESS)
6928 {
6929 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6930 Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6931 return rcStrict2;
6932 }
6933 }
6934
6935 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6936 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6937 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6938 {
6939 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6940 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6941 Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6942 return rc2;
6943 }
6944
6945 /* Pending VM request packets, such as hardware interrupts. */
6946 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6947 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6948 {
6949 Log4Func(("Pending VM request forcing us back to ring-3\n"));
6950 return VINF_EM_PENDING_REQUEST;
6951 }
6952
6953 /* Pending PGM pool flushes. */
6954 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6955 {
6956 Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
6957 return VINF_PGM_POOL_FLUSH_PENDING;
6958 }
6959
6960 /* Pending DMA requests. */
6961 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6962 {
6963 Log4Func(("Pending DMA request forcing us back to ring-3\n"));
6964 return VINF_EM_RAW_TO_R3;
6965 }
6966
6967 return VINF_SUCCESS;
6968}
6969
6970
6971/**
6972 * Converts any TRPM trap into a pending HM event. This is typically used when
6973 * entering from ring-3 (not longjmp returns).
6974 *
6975 * @param pVCpu The cross context virtual CPU structure.
6976 */
6977static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6978{
6979 Assert(TRPMHasTrap(pVCpu));
6980 Assert(!pVCpu->hm.s.Event.fPending);
6981
6982 uint8_t uVector;
6983 TRPMEVENT enmTrpmEvent;
6984 RTGCUINT uErrCode;
6985 RTGCUINTPTR GCPtrFaultAddress;
6986 uint8_t cbInstr;
6987
6988 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6989 AssertRC(rc);
6990
6991 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6992 uint32_t u32IntInfo = uVector | VMX_EXIT_INT_INFO_VALID;
6993 if (enmTrpmEvent == TRPM_TRAP)
6994 {
6995 switch (uVector)
6996 {
6997 case X86_XCPT_NMI:
6998 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6999 break;
7000
7001 case X86_XCPT_BP:
7002 case X86_XCPT_OF:
7003 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7004 break;
7005
7006 case X86_XCPT_PF:
7007 case X86_XCPT_DF:
7008 case X86_XCPT_TS:
7009 case X86_XCPT_NP:
7010 case X86_XCPT_SS:
7011 case X86_XCPT_GP:
7012 case X86_XCPT_AC:
7013 u32IntInfo |= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7014 RT_FALL_THRU();
7015 default:
7016 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7017 break;
7018 }
7019 }
7020 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7021 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7022 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7023 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7024 else
7025 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7026
7027 rc = TRPMResetTrap(pVCpu);
7028 AssertRC(rc);
7029 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7030 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7031
7032 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7033}
7034
7035
7036/**
7037 * Converts the pending HM event into a TRPM trap.
7038 *
7039 * @param pVCpu The cross context virtual CPU structure.
7040 */
7041static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7042{
7043 Assert(pVCpu->hm.s.Event.fPending);
7044
7045 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7046 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7047 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
7048 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7049
7050 /* If a trap was already pending, we did something wrong! */
7051 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7052
7053 TRPMEVENT enmTrapType;
7054 switch (uVectorType)
7055 {
7056 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7057 enmTrapType = TRPM_HARDWARE_INT;
7058 break;
7059
7060 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7061 enmTrapType = TRPM_SOFTWARE_INT;
7062 break;
7063
7064 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7065 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7066 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7067 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7068 enmTrapType = TRPM_TRAP;
7069 break;
7070
7071 default:
7072 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7073 enmTrapType = TRPM_32BIT_HACK;
7074 break;
7075 }
7076
7077 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7078
7079 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7080 AssertRC(rc);
7081
7082 if (fErrorCodeValid)
7083 TRPMSetErrorCode(pVCpu, uErrorCode);
7084
7085 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7086 && uVector == X86_XCPT_PF)
7087 {
7088 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7089 }
7090 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7091 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7092 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7093 {
7094 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7095 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
7096 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7097 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7098 }
7099
7100 /* Clear any pending events from the VMCS. */
7101 VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7102 VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, 0);
7103
7104 /* We're now done converting the pending event. */
7105 pVCpu->hm.s.Event.fPending = false;
7106}
7107
7108
7109/**
7110 * Does the necessary state syncing before returning to ring-3 for any reason
7111 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7112 *
7113 * @returns VBox status code.
7114 * @param pVCpu The cross context virtual CPU structure.
7115 * @param fImportState Whether to import the guest state from the VMCS back
7116 * to the guest-CPU context.
7117 *
7118 * @remarks No-long-jmp zone!!!
7119 */
7120static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
7121{
7122 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7123 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7124
7125 RTCPUID idCpu = RTMpCpuId();
7126 Log4Func(("HostCpuId=%u\n", idCpu));
7127
7128 /*
7129 * !!! IMPORTANT !!!
7130 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7131 */
7132
7133 /* Save the guest state if necessary. */
7134 if (fImportState)
7135 {
7136 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
7137 AssertRCReturn(rc, rc);
7138 }
7139
7140 /* Restore host FPU state if necessary. We will resync on next R0 reentry. */
7141 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7142 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
7143
7144 /* Restore host debug registers if necessary. We will resync on next R0 reentry. */
7145#ifdef VBOX_STRICT
7146 if (CPUMIsHyperDebugStateActive(pVCpu))
7147 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
7148#endif
7149 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7150 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7151 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7152
7153#if HC_ARCH_BITS == 64
7154 /* Restore host-state bits that VT-x only restores partially. */
7155 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7156 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7157 {
7158 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7159 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7160 }
7161 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7162#endif
7163
7164 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7165 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7166 {
7167 /* We shouldn't restore the host MSRs without saving the guest MSRs first. */
7168 if (!fImportState)
7169 {
7170 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS);
7171 AssertRCReturn(rc, rc);
7172 }
7173 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7174 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7175 }
7176 else
7177 pVCpu->hm.s.vmx.fLazyMsrs = 0;
7178
7179 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7180 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7181
7182 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7183 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
7184 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
7185 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
7186 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
7187 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7188 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7189 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7190 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7191
7192 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7193
7194 /** @todo This partially defeats the purpose of having preemption hooks.
7195 * The problem is, deregistering the hooks should be moved to a place that
7196 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7197 * context.
7198 */
7199 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7200 {
7201 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7202 AssertRCReturn(rc, rc);
7203
7204 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7205 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7206 }
7207 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7208 NOREF(idCpu);
7209
7210 return VINF_SUCCESS;
7211}
7212
7213
7214/**
7215 * Leaves the VT-x session.
7216 *
7217 * @returns VBox status code.
7218 * @param pVCpu The cross context virtual CPU structure.
7219 *
7220 * @remarks No-long-jmp zone!!!
7221 */
7222static int hmR0VmxLeaveSession(PVMCPU pVCpu)
7223{
7224 HM_DISABLE_PREEMPT(pVCpu);
7225 HMVMX_ASSERT_CPU_SAFE(pVCpu);
7226 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7227 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7228
7229 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7230 and done this from the VMXR0ThreadCtxCallback(). */
7231 if (!pVCpu->hm.s.fLeaveDone)
7232 {
7233 int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
7234 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7235 pVCpu->hm.s.fLeaveDone = true;
7236 }
7237 Assert(!pVCpu->cpum.GstCtx.fExtrn);
7238
7239 /*
7240 * !!! IMPORTANT !!!
7241 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7242 */
7243
7244 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7245 /** @todo Deregistering here means we need to VMCLEAR always
7246 * (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
7247 * for calling VMMR0ThreadCtxHookDisable here! */
7248 VMMR0ThreadCtxHookDisable(pVCpu);
7249
7250 /* Leave HM context. This takes care of local init (term). */
7251 int rc = HMR0LeaveCpu(pVCpu);
7252
7253 HM_RESTORE_PREEMPT();
7254 return rc;
7255}
7256
7257
7258/**
7259 * Does the necessary state syncing before doing a longjmp to ring-3.
7260 *
7261 * @returns VBox status code.
7262 * @param pVCpu The cross context virtual CPU structure.
7263 *
7264 * @remarks No-long-jmp zone!!!
7265 */
7266DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
7267{
7268 return hmR0VmxLeaveSession(pVCpu);
7269}
7270
7271
7272/**
7273 * Take necessary actions before going back to ring-3.
7274 *
7275 * An action requires us to go back to ring-3. This function does the necessary
7276 * steps before we can safely return to ring-3. This is not the same as longjmps
7277 * to ring-3, this is voluntary and prepares the guest so it may continue
7278 * executing outside HM (recompiler/IEM).
7279 *
7280 * @returns VBox status code.
7281 * @param pVCpu The cross context virtual CPU structure.
7282 * @param rcExit The reason for exiting to ring-3. Can be
7283 * VINF_VMM_UNKNOWN_RING3_CALL.
7284 */
7285static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
7286{
7287 Assert(pVCpu);
7288 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7289
7290 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7291 {
7292 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7293 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7294 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7295 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7296 }
7297
7298 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7299 VMMRZCallRing3Disable(pVCpu);
7300 Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
7301
7302 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7303 if (pVCpu->hm.s.Event.fPending)
7304 {
7305 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7306 Assert(!pVCpu->hm.s.Event.fPending);
7307 }
7308
7309 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7310 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7311
7312 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7313 and if we're injecting an event we should have a TRPM trap pending. */
7314 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7315#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
7316 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7317#endif
7318
7319 /* Save guest state and restore host state bits. */
7320 int rc = hmR0VmxLeaveSession(pVCpu);
7321 AssertRCReturn(rc, rc);
7322 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7323 /* Thread-context hooks are unregistered at this point!!! */
7324
7325 /* Sync recompiler state. */
7326 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7327 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7328 | CPUM_CHANGED_LDTR
7329 | CPUM_CHANGED_GDTR
7330 | CPUM_CHANGED_IDTR
7331 | CPUM_CHANGED_TR
7332 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7333 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
7334 && CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
7335 {
7336 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7337 }
7338
7339 Assert(!pVCpu->hm.s.fClearTrapFlag);
7340
7341 /* Update the exit-to-ring 3 reason. */
7342 pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
7343
7344 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7345 if (rcExit != VINF_EM_RAW_INTERRUPT)
7346 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7347
7348 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7349
7350 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7351 VMMRZCallRing3RemoveNotification(pVCpu);
7352 VMMRZCallRing3Enable(pVCpu);
7353
7354 return rc;
7355}
7356
7357
7358/**
7359 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7360 * longjump to ring-3 and possibly get preempted.
7361 *
7362 * @returns VBox status code.
7363 * @param pVCpu The cross context virtual CPU structure.
7364 * @param enmOperation The operation causing the ring-3 longjump.
7365 * @param pvUser User argument, currently unused, NULL.
7366 */
7367static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7368{
7369 RT_NOREF(pvUser);
7370 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7371 {
7372 /*
7373 * !!! IMPORTANT !!!
7374 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7375 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7376 */
7377 VMMRZCallRing3RemoveNotification(pVCpu);
7378 VMMRZCallRing3Disable(pVCpu);
7379 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7380 RTThreadPreemptDisable(&PreemptState);
7381
7382 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
7383 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7384 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7385
7386#if HC_ARCH_BITS == 64
7387 /* Restore host-state bits that VT-x only restores partially. */
7388 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7389 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7390 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7391 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7392#endif
7393
7394 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7395 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7396 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7397
7398 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7399 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7400 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7401 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7402 {
7403 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7404 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7405 }
7406
7407 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7408 VMMR0ThreadCtxHookDisable(pVCpu);
7409 HMR0LeaveCpu(pVCpu);
7410 RTThreadPreemptRestore(&PreemptState);
7411 return VINF_SUCCESS;
7412 }
7413
7414 Assert(pVCpu);
7415 Assert(pvUser);
7416 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7417 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7418
7419 VMMRZCallRing3Disable(pVCpu);
7420 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7421
7422 Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
7423
7424 int rc = hmR0VmxLongJmpToRing3(pVCpu);
7425 AssertRCReturn(rc, rc);
7426
7427 VMMRZCallRing3Enable(pVCpu);
7428 return VINF_SUCCESS;
7429}
7430
7431
7432/**
7433 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7434 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7435 *
7436 * @param pVCpu The cross context virtual CPU structure.
7437 */
7438DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7439{
7440 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7441 {
7442 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7443 {
7444 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_PROC_CTLS_INT_WINDOW_EXIT;
7445 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7446 AssertRC(rc);
7447 Log4Func(("Setup interrupt-window exiting\n"));
7448 }
7449 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7450}
7451
7452
7453/**
7454 * Clears the interrupt-window exiting control in the VMCS.
7455 *
7456 * @param pVCpu The cross context virtual CPU structure.
7457 */
7458DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7459{
7460 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT);
7461 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
7462 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7463 AssertRC(rc);
7464 Log4Func(("Cleared interrupt-window exiting\n"));
7465}
7466
7467
7468/**
7469 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7470 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7471 *
7472 * @param pVCpu The cross context virtual CPU structure.
7473 */
7474DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7475{
7476 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7477 {
7478 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7479 {
7480 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7481 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7482 AssertRC(rc);
7483 Log4Func(("Setup NMI-window exiting\n"));
7484 }
7485 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7486}
7487
7488
7489/**
7490 * Clears the NMI-window exiting control in the VMCS.
7491 *
7492 * @param pVCpu The cross context virtual CPU structure.
7493 */
7494DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7495{
7496 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT);
7497 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7498 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7499 AssertRC(rc);
7500 Log4Func(("Cleared NMI-window exiting\n"));
7501}
7502
7503
7504/**
7505 * Evaluates the event to be delivered to the guest and sets it as the pending
7506 * event.
7507 *
7508 * @returns The VT-x guest-interruptibility state.
7509 * @param pVCpu The cross context virtual CPU structure.
7510 */
7511static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu)
7512{
7513 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7514 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7515 uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu);
7516 bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7517 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7518 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7519
7520 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7521 Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7522 Assert(!fBlockSti || pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7523 Assert(!TRPMHasTrap(pVCpu));
7524
7525 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7526 APICUpdatePendingInterrupts(pVCpu);
7527
7528 /*
7529 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7530 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7531 */
7532 /** @todo SMI. SMIs take priority over NMIs. */
7533 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7534 {
7535 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7536 if ( !pVCpu->hm.s.Event.fPending
7537 && !fBlockNmi
7538 && !fBlockSti
7539 && !fBlockMovSS)
7540 {
7541 Log4Func(("Pending NMI\n"));
7542 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INT_INFO_VALID;
7543 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7544
7545 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7546 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7547 }
7548 else
7549 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7550 }
7551 /*
7552 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7553 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7554 */
7555 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7556 && !pVCpu->hm.s.fSingleInstruction)
7557 {
7558 Assert(!DBGFIsStepping(pVCpu));
7559 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7560 AssertRCReturn(rc, 0);
7561 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7562 if ( !pVCpu->hm.s.Event.fPending
7563 && !fBlockInt
7564 && !fBlockSti
7565 && !fBlockMovSS)
7566 {
7567 uint8_t u8Interrupt;
7568 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7569 if (RT_SUCCESS(rc))
7570 {
7571 Log4Func(("Pending external interrupt u8Interrupt=%#x\n", u8Interrupt));
7572 uint32_t u32IntInfo = u8Interrupt
7573 | VMX_EXIT_INT_INFO_VALID
7574 | (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7575
7576 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7577 }
7578 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7579 {
7580 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
7581 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7582 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7583
7584 /*
7585 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7586 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7587 * need to re-set this force-flag here.
7588 */
7589 }
7590 else
7591 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7592 }
7593 else
7594 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7595 }
7596
7597 return fIntrState;
7598}
7599
7600
7601/**
7602 * Sets a pending-debug exception to be delivered to the guest if the guest is
7603 * single-stepping in the VMCS.
7604 *
7605 * @param pVCpu The cross context virtual CPU structure.
7606 */
7607DECLINLINE(int) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7608{
7609 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7610 RT_NOREF(pVCpu);
7611 return VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
7612}
7613
7614
7615/**
7616 * Injects any pending events into the guest if the guest is in a state to
7617 * receive them.
7618 *
7619 * @returns Strict VBox status code (i.e. informational status codes too).
7620 * @param pVCpu The cross context virtual CPU structure.
7621 * @param fIntrState The VT-x guest-interruptibility state.
7622 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7623 * return VINF_EM_DBG_STEPPED if the event was
7624 * dispatched directly.
7625 */
7626static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, uint32_t fIntrState, bool fStepping)
7627{
7628 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7629 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7630
7631 bool fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7632 bool fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7633
7634 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7635 Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7636 Assert(!fBlockSti || pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7637 Assert(!TRPMHasTrap(pVCpu));
7638
7639 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7640 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7641 if (pVCpu->hm.s.Event.fPending)
7642 {
7643 /*
7644 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7645 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7646 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7647 *
7648 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7649 */
7650 uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7651#ifdef VBOX_STRICT
7652 if (uIntType == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
7653 {
7654 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7655 Assert(!fBlockInt);
7656 Assert(!fBlockSti);
7657 Assert(!fBlockMovSS);
7658 }
7659 else if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
7660 {
7661 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7662 Assert(!fBlockSti);
7663 Assert(!fBlockMovSS);
7664 Assert(!fBlockNmi);
7665 }
7666#endif
7667 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7668 uIntType));
7669 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7670 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping,
7671 &fIntrState);
7672 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7673
7674 /* Update the interruptibility-state as it could have been changed by
7675 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7676 fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7677 fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7678
7679 if (uIntType == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
7680 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7681 else
7682 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7683 }
7684
7685 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7686 if ( fBlockSti
7687 || fBlockMovSS)
7688 {
7689 if (!pVCpu->hm.s.fSingleInstruction)
7690 {
7691 /*
7692 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7693 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7694 * See Intel spec. 27.3.4 "Saving Non-Register State".
7695 */
7696 Assert(!DBGFIsStepping(pVCpu));
7697 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7698 AssertRCReturn(rc, rc);
7699 if (pCtx->eflags.Bits.u1TF)
7700 {
7701 int rc2 = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7702 AssertRCReturn(rc2, rc2);
7703 }
7704 }
7705 else if (pCtx->eflags.Bits.u1TF)
7706 {
7707 /*
7708 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7709 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7710 */
7711 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
7712 fIntrState = 0;
7713 }
7714 }
7715
7716 /*
7717 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7718 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7719 */
7720 int rc3 = hmR0VmxExportGuestIntrState(pVCpu, fIntrState);
7721 AssertRCReturn(rc3, rc3);
7722
7723 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7724 NOREF(fBlockMovSS); NOREF(fBlockSti);
7725 return rcStrict;
7726}
7727
7728
7729/**
7730 * Injects a double-fault (\#DF) exception into the VM.
7731 *
7732 * @returns Strict VBox status code (i.e. informational status codes too).
7733 * @param pVCpu The cross context virtual CPU structure.
7734 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7735 * and should return VINF_EM_DBG_STEPPED if the event
7736 * is injected directly (register modified by us, not
7737 * by hardware on VM-entry).
7738 * @param pfIntrState Pointer to the current guest interruptibility-state.
7739 * This interruptibility-state will be updated if
7740 * necessary. This cannot not be NULL.
7741 */
7742DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, bool fStepping, uint32_t *pfIntrState)
7743{
7744 uint32_t const u32IntInfo = X86_XCPT_DF | VMX_EXIT_INT_INFO_VALID
7745 | (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT)
7746 | VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7747 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */, fStepping,
7748 pfIntrState);
7749}
7750
7751
7752/**
7753 * Injects a general-protection (\#GP) fault into the VM.
7754 *
7755 * @returns Strict VBox status code (i.e. informational status codes too).
7756 * @param pVCpu The cross context virtual CPU structure.
7757 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7758 * mode, i.e. in real-mode it's not valid).
7759 * @param u32ErrorCode The error code associated with the \#GP.
7760 * @param fStepping Whether we're running in
7761 * hmR0VmxRunGuestCodeStep() and should return
7762 * VINF_EM_DBG_STEPPED if the event is injected
7763 * directly (register modified by us, not by
7764 * hardware on VM-entry).
7765 * @param pfIntrState Pointer to the current guest interruptibility-state.
7766 * This interruptibility-state will be updated if
7767 * necessary. This cannot not be NULL.
7768 */
7769DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, bool fErrorCodeValid, uint32_t u32ErrorCode, bool fStepping,
7770 uint32_t *pfIntrState)
7771{
7772 uint32_t const u32IntInfo = X86_XCPT_GP | VMX_EXIT_INT_INFO_VALID
7773 | (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT)
7774 | (fErrorCodeValid ? VMX_EXIT_INT_INFO_ERROR_CODE_VALID : 0);
7775 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */, fStepping,
7776 pfIntrState);
7777}
7778
7779
7780/**
7781 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7782 * stack.
7783 *
7784 * @returns Strict VBox status code (i.e. informational status codes too).
7785 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7786 * @param pVCpu The cross context virtual CPU structure.
7787 * @param uValue The value to push to the guest stack.
7788 */
7789static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
7790{
7791 /*
7792 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7793 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7794 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7795 */
7796 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7797 if (pCtx->sp == 1)
7798 return VINF_EM_RESET;
7799 pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7800 int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
7801 AssertRC(rc);
7802 return rc;
7803}
7804
7805
7806/**
7807 * Injects an event into the guest upon VM-entry by updating the relevant fields
7808 * in the VM-entry area in the VMCS.
7809 *
7810 * @returns Strict VBox status code (i.e. informational status codes too).
7811 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7812 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7813 *
7814 * @param pVCpu The cross context virtual CPU structure.
7815 * @param u64IntInfo The VM-entry interruption-information field.
7816 * @param cbInstr The VM-entry instruction length in bytes (for
7817 * software interrupts, exceptions and privileged
7818 * software exceptions).
7819 * @param u32ErrCode The VM-entry exception error code.
7820 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7821 * @param pfIntrState Pointer to the current guest interruptibility-state.
7822 * This interruptibility-state will be updated if
7823 * necessary. This cannot not be NULL.
7824 * @param fStepping Whether we're running in
7825 * hmR0VmxRunGuestCodeStep() and should return
7826 * VINF_EM_DBG_STEPPED if the event is injected
7827 * directly (register modified by us, not by
7828 * hardware on VM-entry).
7829 */
7830static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7831 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState)
7832{
7833 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7834 AssertMsg(!RT_HI_U32(u64IntInfo), ("%#RX64\n", u64IntInfo));
7835 Assert(pfIntrState);
7836
7837 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7838 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7839 uint32_t const uVector = VMX_EXIT_INT_INFO_VECTOR(u32IntInfo);
7840 uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(u32IntInfo);
7841
7842#ifdef VBOX_STRICT
7843 /*
7844 * Validate the error-code-valid bit for hardware exceptions.
7845 * No error codes for exceptions in real-mode.
7846 *
7847 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7848 */
7849 if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
7850 && !CPUMIsGuestInRealModeEx(pCtx))
7851 {
7852 switch (uVector)
7853 {
7854 case X86_XCPT_PF:
7855 case X86_XCPT_DF:
7856 case X86_XCPT_TS:
7857 case X86_XCPT_NP:
7858 case X86_XCPT_SS:
7859 case X86_XCPT_GP:
7860 case X86_XCPT_AC:
7861 AssertMsg(VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
7862 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7863 RT_FALL_THRU();
7864 default:
7865 break;
7866 }
7867 }
7868#endif
7869
7870 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7871 Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
7872 || !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
7873
7874 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7875
7876 /*
7877 * Hardware interrupts & exceptions cannot be delivered through the software interrupt
7878 * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
7879 * interrupt handler in the (real-mode) guest.
7880 *
7881 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
7882 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7883 */
7884 if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
7885 {
7886 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
7887 {
7888 /*
7889 * For unrestricted execution enabled CPUs running real-mode guests, we must not
7890 * set the deliver-error-code bit.
7891 *
7892 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7893 */
7894 u32IntInfo &= ~VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7895 }
7896 else
7897 {
7898 PVM pVM = pVCpu->CTX_SUFF(pVM);
7899 Assert(PDMVmmDevHeapIsEnabled(pVM));
7900 Assert(pVM->hm.s.vmx.pRealModeTSS);
7901
7902 /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
7903 int rc2 = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_TABLE_MASK | CPUMCTX_EXTRN_RIP
7904 | CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_RFLAGS);
7905 AssertRCReturn(rc2, rc2);
7906
7907 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7908 size_t const cbIdtEntry = sizeof(X86IDTR16);
7909 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
7910 {
7911 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7912 if (uVector == X86_XCPT_DF)
7913 return VINF_EM_RESET;
7914
7915 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7916 if (uVector == X86_XCPT_GP)
7917 return hmR0VmxInjectXcptDF(pVCpu, fStepping, pfIntrState);
7918
7919 /*
7920 * If we're injecting an event with no valid IDT entry, inject a #GP.
7921 * No error codes for exceptions in real-mode.
7922 *
7923 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7924 */
7925 return hmR0VmxInjectXcptGP(pVCpu, false /* fErrCodeValid */, 0 /* u32ErrCode */, fStepping, pfIntrState);
7926 }
7927
7928 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7929 uint16_t uGuestIp = pCtx->ip;
7930 if (uIntType == VMX_EXIT_INT_INFO_TYPE_SW_XCPT)
7931 {
7932 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7933 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7934 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7935 }
7936 else if (uIntType == VMX_EXIT_INT_INFO_TYPE_SW_INT)
7937 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7938
7939 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7940 X86IDTR16 IdtEntry;
7941 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
7942 rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7943 AssertRCReturn(rc2, rc2);
7944
7945 /* Construct the stack frame for the interrupt/exception handler. */
7946 VBOXSTRICTRC rcStrict;
7947 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
7948 if (rcStrict == VINF_SUCCESS)
7949 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
7950 if (rcStrict == VINF_SUCCESS)
7951 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
7952
7953 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7954 if (rcStrict == VINF_SUCCESS)
7955 {
7956 pCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7957 pCtx->rip = IdtEntry.offSel;
7958 pCtx->cs.Sel = IdtEntry.uSel;
7959 pCtx->cs.ValidSel = IdtEntry.uSel;
7960 pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7961 if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
7962 && uVector == X86_XCPT_PF)
7963 pCtx->cr2 = GCPtrFaultAddress;
7964
7965 /* If any other guest-state bits are changed here, make sure to update
7966 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7967 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS | HM_CHANGED_GUEST_CR2
7968 | HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
7969 | HM_CHANGED_GUEST_RSP);
7970
7971 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7972 if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
7973 {
7974 Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
7975 && uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
7976 Log4Func(("Clearing inhibition due to STI\n"));
7977 *pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
7978 }
7979 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7980 u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
7981
7982 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7983 it, if we are returning to ring-3 before executing guest code. */
7984 pVCpu->hm.s.Event.fPending = false;
7985
7986 /* Make hmR0VmxPreRunGuest() return if we're stepping since we've changed cs:rip. */
7987 if (fStepping)
7988 rcStrict = VINF_EM_DBG_STEPPED;
7989 }
7990 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7991 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7992 return rcStrict;
7993 }
7994 }
7995
7996 /* Validate. */
7997 Assert(VMX_EXIT_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7998 Assert(!VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7999 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8000
8001 /* Inject. */
8002 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8003 if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8004 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8005 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8006 AssertRCReturn(rc, rc);
8007
8008 /* Update CR2. */
8009 if ( VMX_EXIT_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8010 && uVector == X86_XCPT_PF)
8011 pCtx->cr2 = GCPtrFaultAddress;
8012
8013 Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8014
8015 return VINF_SUCCESS;
8016}
8017
8018
8019/**
8020 * Clears the interrupt-window exiting control in the VMCS and if necessary
8021 * clears the current event in the VMCS as well.
8022 *
8023 * @returns VBox status code.
8024 * @param pVCpu The cross context virtual CPU structure.
8025 *
8026 * @remarks Use this function only to clear events that have not yet been
8027 * delivered to the guest but are injected in the VMCS!
8028 * @remarks No-long-jump zone!!!
8029 */
8030static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8031{
8032 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8033 {
8034 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8035 Log4Func(("Cleared interrupt widow\n"));
8036 }
8037
8038 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8039 {
8040 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8041 Log4Func(("Cleared interrupt widow\n"));
8042 }
8043}
8044
8045
8046/**
8047 * Enters the VT-x session.
8048 *
8049 * @returns VBox status code.
8050 * @param pVCpu The cross context virtual CPU structure.
8051 * @param pHostCpu Pointer to the global CPU info struct.
8052 */
8053VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu, PHMGLOBALCPUINFO pHostCpu)
8054{
8055 AssertPtr(pVCpu);
8056 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8057 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8058 RT_NOREF(pHostCpu);
8059
8060 LogFlowFunc(("pVCpu=%p\n", pVCpu));
8061 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8062 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8063
8064#ifdef VBOX_STRICT
8065 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8066 RTCCUINTREG uHostCR4 = ASMGetCR4();
8067 if (!(uHostCR4 & X86_CR4_VMXE))
8068 {
8069 LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
8070 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8071 }
8072#endif
8073
8074 /*
8075 * Load the VCPU's VMCS as the current (and active) one.
8076 */
8077 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8078 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8079 if (RT_FAILURE(rc))
8080 return rc;
8081
8082 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8083 pVCpu->hm.s.fLeaveDone = false;
8084 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8085
8086 return VINF_SUCCESS;
8087}
8088
8089
8090/**
8091 * The thread-context callback (only on platforms which support it).
8092 *
8093 * @param enmEvent The thread-context event.
8094 * @param pVCpu The cross context virtual CPU structure.
8095 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8096 * @thread EMT(pVCpu)
8097 */
8098VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8099{
8100 NOREF(fGlobalInit);
8101
8102 switch (enmEvent)
8103 {
8104 case RTTHREADCTXEVENT_OUT:
8105 {
8106 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8107 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8108 VMCPU_ASSERT_EMT(pVCpu);
8109
8110 /* No longjmps (logger flushes, locks) in this fragile context. */
8111 VMMRZCallRing3Disable(pVCpu);
8112 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8113
8114 /*
8115 * Restore host-state (FPU, debug etc.)
8116 */
8117 if (!pVCpu->hm.s.fLeaveDone)
8118 {
8119 /*
8120 * Do -not- import the guest-state here as we might already be in the middle of importing
8121 * it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
8122 */
8123 hmR0VmxLeave(pVCpu, false /* fImportState */);
8124 pVCpu->hm.s.fLeaveDone = true;
8125 }
8126
8127 /* Leave HM context, takes care of local init (term). */
8128 int rc = HMR0LeaveCpu(pVCpu);
8129 AssertRC(rc); NOREF(rc);
8130
8131 /* Restore longjmp state. */
8132 VMMRZCallRing3Enable(pVCpu);
8133 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8134 break;
8135 }
8136
8137 case RTTHREADCTXEVENT_IN:
8138 {
8139 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8140 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8141 VMCPU_ASSERT_EMT(pVCpu);
8142
8143 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8144 VMMRZCallRing3Disable(pVCpu);
8145 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8146
8147 /* Initialize the bare minimum state required for HM. This takes care of
8148 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8149 int rc = hmR0EnterCpu(pVCpu);
8150 AssertRC(rc);
8151 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8152 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8153
8154 /* Load the active VMCS as the current one. */
8155 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8156 {
8157 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8158 AssertRC(rc); NOREF(rc);
8159 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8160 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8161 }
8162 pVCpu->hm.s.fLeaveDone = false;
8163
8164 /* Restore longjmp state. */
8165 VMMRZCallRing3Enable(pVCpu);
8166 break;
8167 }
8168
8169 default:
8170 break;
8171 }
8172}
8173
8174
8175/**
8176 * Exports the host state into the VMCS host-state area.
8177 * Sets up the VM-exit MSR-load area.
8178 *
8179 * The CPU state will be loaded from these fields on every successful VM-exit.
8180 *
8181 * @returns VBox status code.
8182 * @param pVCpu The cross context virtual CPU structure.
8183 *
8184 * @remarks No-long-jump zone!!!
8185 */
8186static int hmR0VmxExportHostState(PVMCPU pVCpu)
8187{
8188 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8189
8190 int rc = VINF_SUCCESS;
8191 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8192 {
8193 rc = hmR0VmxExportHostControlRegs();
8194 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8195
8196 rc = hmR0VmxExportHostSegmentRegs(pVCpu);
8197 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8198
8199 rc = hmR0VmxExportHostMsrs(pVCpu);
8200 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8201
8202 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
8203 }
8204 return rc;
8205}
8206
8207
8208/**
8209 * Saves the host state in the VMCS host-state.
8210 *
8211 * @returns VBox status code.
8212 * @param pVCpu The cross context virtual CPU structure.
8213 *
8214 * @remarks No-long-jump zone!!!
8215 */
8216VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
8217{
8218 AssertPtr(pVCpu);
8219 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8220
8221 /*
8222 * Export the host state here while entering HM context.
8223 * When thread-context hooks are used, we might get preempted and have to re-save the host
8224 * state but most of the time we won't be, so do it here before we disable interrupts.
8225 */
8226 return hmR0VmxExportHostState(pVCpu);
8227}
8228
8229
8230/**
8231 * Exports the guest state into the VMCS guest-state area.
8232 *
8233 * The will typically be done before VM-entry when the guest-CPU state and the
8234 * VMCS state may potentially be out of sync.
8235 *
8236 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8237 * VM-entry controls.
8238 * Sets up the appropriate VMX non-root function to execute guest code based on
8239 * the guest CPU mode.
8240 *
8241 * @returns VBox strict status code.
8242 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8243 * without unrestricted guest access and the VMMDev is not presently
8244 * mapped (e.g. EFI32).
8245 *
8246 * @param pVCpu The cross context virtual CPU structure.
8247 *
8248 * @remarks No-long-jump zone!!!
8249 */
8250static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu)
8251{
8252 AssertPtr(pVCpu);
8253 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8254
8255 LogFlowFunc(("pVCpu=%p\n", pVCpu));
8256
8257 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
8258
8259 /* Determine real-on-v86 mode. */
8260 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8261 if ( !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
8262 && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
8263 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8264
8265 /*
8266 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8267 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8268 */
8269 int rc = hmR0VmxSelectVMRunHandler(pVCpu);
8270 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8271
8272 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8273 rc = hmR0VmxExportGuestEntryCtls(pVCpu);
8274 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8275
8276 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8277 rc = hmR0VmxExportGuestExitCtls(pVCpu);
8278 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8279
8280 rc = hmR0VmxExportGuestCR0(pVCpu);
8281 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8282
8283 VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu);
8284 if (rcStrict == VINF_SUCCESS)
8285 { /* likely */ }
8286 else
8287 {
8288 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8289 return rcStrict;
8290 }
8291
8292 rc = hmR0VmxExportGuestSegmentRegs(pVCpu);
8293 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8294
8295 /* This needs to be done after hmR0VmxExportGuestEntryCtls() and hmR0VmxExportGuestExitCtls() as it
8296 may alter controls if we determine we don't have to swap EFER after all. */
8297 rc = hmR0VmxExportGuestMsrs(pVCpu);
8298 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8299
8300 rc = hmR0VmxExportGuestApicTpr(pVCpu);
8301 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8302
8303 rc = hmR0VmxExportGuestXcptIntercepts(pVCpu);
8304 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8305
8306 /* Exporting RFLAGS here is fine, even though RFLAGS.TF might depend on guest debug state which is
8307 not exported here. It is re-evaluated and updated if necessary in hmR0VmxExportSharedState(). */
8308 rc = hmR0VmxExportGuestRip(pVCpu);
8309 rc |= hmR0VmxExportGuestRsp(pVCpu);
8310 rc |= hmR0VmxExportGuestRflags(pVCpu);
8311 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8312
8313 /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
8314 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
8315 | HM_CHANGED_GUEST_CR2
8316 | (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
8317 | HM_CHANGED_GUEST_X87
8318 | HM_CHANGED_GUEST_SSE_AVX
8319 | HM_CHANGED_GUEST_OTHER_XSAVE
8320 | HM_CHANGED_GUEST_XCRx
8321 | HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
8322 | HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
8323 | HM_CHANGED_GUEST_TSC_AUX
8324 | HM_CHANGED_GUEST_OTHER_MSRS
8325 | HM_CHANGED_GUEST_HWVIRT
8326 | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
8327
8328 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
8329 return rc;
8330}
8331
8332
8333/**
8334 * Exports the state shared between the host and guest into the VMCS.
8335 *
8336 * @param pVCpu The cross context virtual CPU structure.
8337 *
8338 * @remarks No-long-jump zone!!!
8339 */
8340static void hmR0VmxExportSharedState(PVMCPU pVCpu)
8341{
8342 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8343 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8344
8345 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
8346 {
8347 int rc = hmR0VmxExportSharedDebugState(pVCpu);
8348 AssertRC(rc);
8349 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
8350
8351 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8352 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
8353 {
8354 rc = hmR0VmxExportGuestRflags(pVCpu);
8355 AssertRC(rc);
8356 }
8357 }
8358
8359 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
8360 {
8361 hmR0VmxLazyLoadGuestMsrs(pVCpu);
8362 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
8363 }
8364
8365 AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
8366 ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8367}
8368
8369
8370/**
8371 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8372 *
8373 * @returns Strict VBox status code (i.e. informational status codes too).
8374 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8375 * without unrestricted guest access and the VMMDev is not presently
8376 * mapped (e.g. EFI32).
8377 *
8378 * @param pVCpu The cross context virtual CPU structure.
8379 *
8380 * @remarks No-long-jump zone!!!
8381 */
8382static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu)
8383{
8384 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8385 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8386 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8387
8388#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8389 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8390#endif
8391
8392 /*
8393 * For many exits it's only RIP that changes and hence try to export it first
8394 * without going through a lot of change flag checks.
8395 */
8396 VBOXSTRICTRC rcStrict;
8397 uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8398 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8399 if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
8400 {
8401 rcStrict = hmR0VmxExportGuestRip(pVCpu);
8402 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8403 { /* likely */}
8404 else
8405 AssertMsgFailedReturn(("hmR0VmxExportGuestRip failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8406 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
8407 }
8408 else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8409 {
8410 rcStrict = hmR0VmxExportGuestState(pVCpu);
8411 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8412 { /* likely */}
8413 else
8414 {
8415 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("hmR0VmxExportGuestState failed! rc=%Rrc\n",
8416 VBOXSTRICTRC_VAL(rcStrict)));
8417 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8418 return rcStrict;
8419 }
8420 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
8421 }
8422 else
8423 rcStrict = VINF_SUCCESS;
8424
8425#ifdef VBOX_STRICT
8426 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8427 fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8428 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8429 AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
8430 ("fCtxChanged=%#RX64\n", fCtxChanged));
8431#endif
8432 return rcStrict;
8433}
8434
8435
8436/**
8437 * Does the preparations before executing guest code in VT-x.
8438 *
8439 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8440 * recompiler/IEM. We must be cautious what we do here regarding committing
8441 * guest-state information into the VMCS assuming we assuredly execute the
8442 * guest in VT-x mode.
8443 *
8444 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8445 * the common-state (TRPM/forceflags), we must undo those changes so that the
8446 * recompiler/IEM can (and should) use them when it resumes guest execution.
8447 * Otherwise such operations must be done when we can no longer exit to ring-3.
8448 *
8449 * @returns Strict VBox status code (i.e. informational status codes too).
8450 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8451 * have been disabled.
8452 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8453 * double-fault into the guest.
8454 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8455 * dispatched directly.
8456 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8457 *
8458 * @param pVCpu The cross context virtual CPU structure.
8459 * @param pVmxTransient Pointer to the VMX transient structure.
8460 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8461 * us ignore some of the reasons for returning to
8462 * ring-3, and return VINF_EM_DBG_STEPPED if event
8463 * dispatching took place.
8464 */
8465static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
8466{
8467 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8468
8469#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_ONLY_IN_IEM
8470 Log2(("hmR0SvmPreRunGuest: Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
8471 return VINF_EM_RESCHEDULE_REM;
8472#endif
8473
8474#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8475 PGMRZDynMapFlushAutoSet(pVCpu);
8476#endif
8477
8478 /* Check force flag actions that might require us to go back to ring-3. */
8479 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
8480 if (rcStrict == VINF_SUCCESS)
8481 { /* FFs doesn't get set all the time. */ }
8482 else
8483 return rcStrict;
8484
8485 /*
8486 * Setup the virtualized-APIC accesses.
8487 *
8488 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8489 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8490 *
8491 * This is the reason we do it here and not in hmR0VmxExportGuestState().
8492 */
8493 PVM pVM = pVCpu->CTX_SUFF(pVM);
8494 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8495 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
8496 && PDMHasApic(pVM))
8497 {
8498 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8499 Assert(u64MsrApicBase);
8500 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8501
8502 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8503
8504 /* Unalias any existing mapping. */
8505 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8506 AssertRCReturn(rc, rc);
8507
8508 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8509 Log4Func(("Mapped HC APIC-access page at %#RGp\n", GCPhysApicBase));
8510 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8511 AssertRCReturn(rc, rc);
8512
8513 /* Update the per-VCPU cache of the APIC base MSR. */
8514 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8515 }
8516
8517 if (TRPMHasTrap(pVCpu))
8518 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8519 uint32_t fIntrState = hmR0VmxEvaluatePendingEvent(pVCpu);
8520
8521 /*
8522 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
8523 * needs to be done with longjmps or interrupts + preemption enabled. Event injection might
8524 * also result in triple-faulting the VM.
8525 */
8526 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, fIntrState, fStepping);
8527 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8528 { /* likely */ }
8529 else
8530 {
8531 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8532 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8533 return rcStrict;
8534 }
8535
8536 /*
8537 * A longjump might result in importing CR3 even for VM-exits that don't necessarily
8538 * import CR3 themselves. We will need to update them here, as even as late as the above
8539 * hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
8540 * the below force flags to be set.
8541 */
8542 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8543 {
8544 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
8545 int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8546 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
8547 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
8548 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8549 }
8550 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8551 {
8552 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8553 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8554 }
8555
8556 /*
8557 * No longjmps to ring-3 from this point on!!!
8558 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8559 * This also disables flushing of the R0-logger instance (if any).
8560 */
8561 VMMRZCallRing3Disable(pVCpu);
8562
8563 /*
8564 * Export the guest state bits.
8565 *
8566 * We cannot perform longjmps while loading the guest state because we do not preserve the
8567 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8568 * CPU migration.
8569 *
8570 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8571 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8572 * Hence, loading of the guest state needs to be done -after- injection of events.
8573 */
8574 rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu);
8575 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8576 { /* likely */ }
8577 else
8578 {
8579 VMMRZCallRing3Enable(pVCpu);
8580 return rcStrict;
8581 }
8582
8583 /*
8584 * We disable interrupts so that we don't miss any interrupts that would flag preemption
8585 * (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
8586 * preemption disabled for a while. Since this is purly to aid the
8587 * RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
8588 * disable interrupt on NT.
8589 *
8590 * We need to check for force-flags that could've possible been altered since we last
8591 * checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
8592 * see @bugref{6398}).
8593 *
8594 * We also check a couple of other force-flags as a last opportunity to get the EMT back
8595 * to ring-3 before executing guest code.
8596 */
8597 pVmxTransient->fEFlags = ASMIntDisableFlags();
8598
8599 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8600 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8601 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8602 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8603 {
8604 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8605 {
8606 pVCpu->hm.s.Event.fPending = false;
8607
8608 /*
8609 * We've injected any pending events. This is really the point of no return (to ring-3).
8610 *
8611 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
8612 * returns from this function, so don't enable them here.
8613 */
8614 return VINF_SUCCESS;
8615 }
8616
8617 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
8618 rcStrict = VINF_EM_RAW_INTERRUPT;
8619 }
8620 else
8621 {
8622 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8623 rcStrict = VINF_EM_RAW_TO_R3;
8624 }
8625
8626 ASMSetFlags(pVmxTransient->fEFlags);
8627 VMMRZCallRing3Enable(pVCpu);
8628
8629 return rcStrict;
8630}
8631
8632
8633/**
8634 * Prepares to run guest code in VT-x and we've committed to doing so. This
8635 * means there is no backing out to ring-3 or anywhere else at this
8636 * point.
8637 *
8638 * @param pVCpu The cross context virtual CPU structure.
8639 * @param pVmxTransient Pointer to the VMX transient structure.
8640 *
8641 * @remarks Called with preemption disabled.
8642 * @remarks No-long-jump zone!!!
8643 */
8644static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
8645{
8646 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8647 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8648 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8649
8650 /*
8651 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8652 */
8653 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8654 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8655
8656 PVM pVM = pVCpu->CTX_SUFF(pVM);
8657 if (!CPUMIsGuestFPUStateActive(pVCpu))
8658 {
8659 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8660 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8661 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT;
8662 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8663 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
8664 }
8665
8666 /*
8667 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8668 */
8669 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8670 && pVCpu->hm.s.vmx.cMsrs > 0)
8671 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8672
8673 /*
8674 * Re-save the host state bits as we may've been preempted (only happens when
8675 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8676 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8677 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8678 * See @bugref{8432}.
8679 */
8680 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8681 {
8682 int rc = hmR0VmxExportHostState(pVCpu);
8683 AssertRC(rc);
8684 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
8685 }
8686 Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
8687
8688 /*
8689 * Export the state shared between host and guest (FPU, debug, lazy MSRs).
8690 */
8691 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
8692 hmR0VmxExportSharedState(pVCpu);
8693 AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8694
8695 /* Store status of the shared guest-host state at the time of VM-entry. */
8696#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8697 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
8698 {
8699 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8700 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8701 }
8702 else
8703#endif
8704 {
8705 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8706 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8707 }
8708
8709 /*
8710 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8711 */
8712 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
8713 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8714
8715 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8716 RTCPUID idCurrentCpu = pCpu->idCpu;
8717 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8718 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8719 {
8720 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8721 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8722 }
8723
8724 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8725 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8726 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8727 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8728
8729 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8730
8731 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8732 to start executing. */
8733
8734 /*
8735 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8736 */
8737 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
8738 {
8739 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
8740 {
8741 bool fMsrUpdated;
8742 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
8743 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8744 &fMsrUpdated);
8745 AssertRC(rc2);
8746 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8747 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8748 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8749 }
8750 else
8751 {
8752 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8753 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8754 }
8755 }
8756
8757 if (pVM->cpum.ro.GuestFeatures.fIbrs)
8758 {
8759 bool fMsrUpdated;
8760 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_OTHER_MSRS);
8761 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
8762 &fMsrUpdated);
8763 AssertRC(rc2);
8764 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8765 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8766 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8767 }
8768
8769#ifdef VBOX_STRICT
8770 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8771 hmR0VmxCheckHostEferMsr(pVCpu);
8772 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8773#endif
8774#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8775 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
8776 {
8777 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
8778 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8779 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8780 }
8781#endif
8782}
8783
8784
8785/**
8786 * Performs some essential restoration of state after running guest code in
8787 * VT-x.
8788 *
8789 * @param pVCpu The cross context virtual CPU structure.
8790 * @param pVmxTransient Pointer to the VMX transient structure.
8791 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8792 *
8793 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8794 *
8795 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8796 * unconditionally when it is safe to do so.
8797 */
8798static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8799{
8800 uint64_t const uHostTsc = ASMReadTSC();
8801 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8802
8803 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8804 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8805 pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8806 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8807 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8808 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8809
8810 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
8811 TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TscOffset);
8812
8813 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
8814 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8815 Assert(!ASMIntAreEnabled());
8816 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8817
8818#if HC_ARCH_BITS == 64
8819 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8820#endif
8821#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8822 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8823 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8824 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8825#else
8826 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8827#endif
8828#ifdef VBOX_STRICT
8829 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8830#endif
8831 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8832
8833 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8834 uint32_t uExitReason;
8835 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8836 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8837 AssertRC(rc);
8838 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8839 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INT_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8840
8841 if (rcVMRun == VINF_SUCCESS)
8842 {
8843 /*
8844 * Update the VM-exit history array here even if the VM-entry failed due to:
8845 * - Invalid guest state.
8846 * - MSR loading.
8847 * - Machine-check event.
8848 *
8849 * In any of the above cases we will still have a "valid" VM-exit reason
8850 * despite @a fVMEntryFailed being false.
8851 *
8852 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8853 *
8854 * Note! We don't have CS or RIP at this point. Will probably address that later
8855 * by amending the history entry added here.
8856 */
8857 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
8858 UINT64_MAX, uHostTsc);
8859
8860 if (!pVmxTransient->fVMEntryFailed)
8861 {
8862 VMMRZCallRing3Enable(pVCpu);
8863
8864 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8865 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8866
8867#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8868 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
8869 AssertRC(rc);
8870#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8871 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_RFLAGS);
8872 AssertRC(rc);
8873#else
8874 /*
8875 * Import the guest-interruptibility state always as we need it while evaluating
8876 * injecting events on re-entry.
8877 *
8878 * We don't import CR0 (when Unrestricted guest execution is unavailable) despite
8879 * checking for real-mode while exporting the state because all bits that cause
8880 * mode changes wrt CR0 are intercepted.
8881 */
8882 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
8883 AssertRC(rc);
8884#endif
8885
8886 /*
8887 * Sync the TPR shadow with our APIC state.
8888 */
8889 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
8890 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8891 {
8892 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8893 AssertRC(rc);
8894 ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8895 }
8896
8897 return;
8898 }
8899 }
8900 else
8901 Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
8902
8903 VMMRZCallRing3Enable(pVCpu);
8904}
8905
8906
8907/**
8908 * Runs the guest code using VT-x the normal way.
8909 *
8910 * @returns VBox status code.
8911 * @param pVCpu The cross context virtual CPU structure.
8912 *
8913 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8914 */
8915static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu)
8916{
8917 VMXTRANSIENT VmxTransient;
8918 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8919 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8920 uint32_t cLoops = 0;
8921
8922 for (;; cLoops++)
8923 {
8924 Assert(!HMR0SuspendPending());
8925 HMVMX_ASSERT_CPU_SAFE(pVCpu);
8926
8927 /* Preparatory work for running guest code, this may force us to return
8928 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8929 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8930 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
8931 if (rcStrict != VINF_SUCCESS)
8932 break;
8933
8934 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
8935 int rcRun = hmR0VmxRunGuest(pVCpu);
8936
8937 /* Restore any residual host-state and save any bits shared between host
8938 and guest into the guest-CPU state. Re-enables interrupts! */
8939 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
8940
8941 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8942 if (RT_SUCCESS(rcRun))
8943 { /* very likely */ }
8944 else
8945 {
8946 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
8947 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
8948 return rcRun;
8949 }
8950
8951 /* Profile the VM-exit. */
8952 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8953 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8954 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8955 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
8956 HMVMX_START_EXIT_DISPATCH_PROF();
8957
8958 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
8959
8960 /* Handle the VM-exit. */
8961#ifdef HMVMX_USE_FUNCTION_TABLE
8962 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
8963#else
8964 rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient, VmxTransient.uExitReason);
8965#endif
8966 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
8967 if (rcStrict == VINF_SUCCESS)
8968 {
8969 if (cLoops <= pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
8970 continue; /* likely */
8971 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8972 rcStrict = VINF_EM_RAW_INTERRUPT;
8973 }
8974 break;
8975 }
8976
8977 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8978 return rcStrict;
8979}
8980
8981
8982
8983/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8984 * probes.
8985 *
8986 * The following few functions and associated structure contains the bloat
8987 * necessary for providing detailed debug events and dtrace probes as well as
8988 * reliable host side single stepping. This works on the principle of
8989 * "subclassing" the normal execution loop and workers. We replace the loop
8990 * method completely and override selected helpers to add necessary adjustments
8991 * to their core operation.
8992 *
8993 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8994 * any performance for debug and analysis features.
8995 *
8996 * @{
8997 */
8998
8999/**
9000 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
9001 * the debug run loop.
9002 */
9003typedef struct VMXRUNDBGSTATE
9004{
9005 /** The RIP we started executing at. This is for detecting that we stepped. */
9006 uint64_t uRipStart;
9007 /** The CS we started executing with. */
9008 uint16_t uCsStart;
9009
9010 /** Whether we've actually modified the 1st execution control field. */
9011 bool fModifiedProcCtls : 1;
9012 /** Whether we've actually modified the 2nd execution control field. */
9013 bool fModifiedProcCtls2 : 1;
9014 /** Whether we've actually modified the exception bitmap. */
9015 bool fModifiedXcptBitmap : 1;
9016
9017 /** We desire the modified the CR0 mask to be cleared. */
9018 bool fClearCr0Mask : 1;
9019 /** We desire the modified the CR4 mask to be cleared. */
9020 bool fClearCr4Mask : 1;
9021 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9022 uint32_t fCpe1Extra;
9023 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9024 uint32_t fCpe1Unwanted;
9025 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9026 uint32_t fCpe2Extra;
9027 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9028 uint32_t bmXcptExtra;
9029 /** The sequence number of the Dtrace provider settings the state was
9030 * configured against. */
9031 uint32_t uDtraceSettingsSeqNo;
9032 /** VM-exits to check (one bit per VM-exit). */
9033 uint32_t bmExitsToCheck[3];
9034
9035 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9036 uint32_t fProcCtlsInitial;
9037 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9038 uint32_t fProcCtls2Initial;
9039 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9040 uint32_t bmXcptInitial;
9041} VMXRUNDBGSTATE;
9042AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9043typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9044
9045
9046/**
9047 * Initializes the VMXRUNDBGSTATE structure.
9048 *
9049 * @param pVCpu The cross context virtual CPU structure of the
9050 * calling EMT.
9051 * @param pDbgState The structure to initialize.
9052 */
9053static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9054{
9055 pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
9056 pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
9057
9058 pDbgState->fModifiedProcCtls = false;
9059 pDbgState->fModifiedProcCtls2 = false;
9060 pDbgState->fModifiedXcptBitmap = false;
9061 pDbgState->fClearCr0Mask = false;
9062 pDbgState->fClearCr4Mask = false;
9063 pDbgState->fCpe1Extra = 0;
9064 pDbgState->fCpe1Unwanted = 0;
9065 pDbgState->fCpe2Extra = 0;
9066 pDbgState->bmXcptExtra = 0;
9067 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9068 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9069 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9070}
9071
9072
9073/**
9074 * Updates the VMSC fields with changes requested by @a pDbgState.
9075 *
9076 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9077 * immediately before executing guest code, i.e. when interrupts are disabled.
9078 * We don't check status codes here as we cannot easily assert or return in the
9079 * latter case.
9080 *
9081 * @param pVCpu The cross context virtual CPU structure.
9082 * @param pDbgState The debug state.
9083 */
9084static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9085{
9086 /*
9087 * Ensure desired flags in VMCS control fields are set.
9088 * (Ignoring write failure here, as we're committed and it's just debug extras.)
9089 *
9090 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9091 * there should be no stale data in pCtx at this point.
9092 */
9093 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9094 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9095 {
9096 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
9097 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9098 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9099 Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9100 pDbgState->fModifiedProcCtls = true;
9101 }
9102
9103 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9104 {
9105 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
9106 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9107 Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9108 pDbgState->fModifiedProcCtls2 = true;
9109 }
9110
9111 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9112 {
9113 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
9114 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9115 Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9116 pDbgState->fModifiedXcptBitmap = true;
9117 }
9118
9119 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32Cr0Mask != 0)
9120 {
9121 pVCpu->hm.s.vmx.u32Cr0Mask = 0;
9122 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9123 Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9124 }
9125
9126 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32Cr4Mask != 0)
9127 {
9128 pVCpu->hm.s.vmx.u32Cr4Mask = 0;
9129 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9130 Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9131 }
9132}
9133
9134
9135/**
9136 * Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
9137 * re-entry next time around.
9138 *
9139 * @returns Strict VBox status code (i.e. informational status codes too).
9140 * @param pVCpu The cross context virtual CPU structure.
9141 * @param pDbgState The debug state.
9142 * @param rcStrict The return code from executing the guest using single
9143 * stepping.
9144 */
9145static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9146{
9147 /*
9148 * Restore VM-exit control settings as we may not reenter this function the
9149 * next time around.
9150 */
9151 /* We reload the initial value, trigger what we can of recalculations the
9152 next time around. From the looks of things, that's all that's required atm. */
9153 if (pDbgState->fModifiedProcCtls)
9154 {
9155 if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9156 pDbgState->fProcCtlsInitial |= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9157 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9158 AssertRCReturn(rc2, rc2);
9159 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9160 }
9161
9162 /* We're currently the only ones messing with this one, so just restore the
9163 cached value and reload the field. */
9164 if ( pDbgState->fModifiedProcCtls2
9165 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9166 {
9167 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9168 AssertRCReturn(rc2, rc2);
9169 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9170 }
9171
9172 /* If we've modified the exception bitmap, we restore it and trigger
9173 reloading and partial recalculation the next time around. */
9174 if (pDbgState->fModifiedXcptBitmap)
9175 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9176
9177 return rcStrict;
9178}
9179
9180
9181/**
9182 * Configures VM-exit controls for current DBGF and DTrace settings.
9183 *
9184 * This updates @a pDbgState and the VMCS execution control fields to reflect
9185 * the necessary VM-exits demanded by DBGF and DTrace.
9186 *
9187 * @param pVCpu The cross context virtual CPU structure.
9188 * @param pDbgState The debug state.
9189 * @param pVmxTransient Pointer to the VMX transient structure. May update
9190 * fUpdateTscOffsettingAndPreemptTimer.
9191 */
9192static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9193{
9194 /*
9195 * Take down the dtrace serial number so we can spot changes.
9196 */
9197 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9198 ASMCompilerBarrier();
9199
9200 /*
9201 * We'll rebuild most of the middle block of data members (holding the
9202 * current settings) as we go along here, so start by clearing it all.
9203 */
9204 pDbgState->bmXcptExtra = 0;
9205 pDbgState->fCpe1Extra = 0;
9206 pDbgState->fCpe1Unwanted = 0;
9207 pDbgState->fCpe2Extra = 0;
9208 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9209 pDbgState->bmExitsToCheck[i] = 0;
9210
9211 /*
9212 * Software interrupts (INT XXh) - no idea how to trigger these...
9213 */
9214 PVM pVM = pVCpu->CTX_SUFF(pVM);
9215 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9216 || VBOXVMM_INT_SOFTWARE_ENABLED())
9217 {
9218 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9219 }
9220
9221 /*
9222 * INT3 breakpoints - triggered by #BP exceptions.
9223 */
9224 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9225 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9226
9227 /*
9228 * Exception bitmap and XCPT events+probes.
9229 */
9230 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9231 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9232 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9233
9234 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9235 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9236 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9237 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9238 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9239 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9240 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9241 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9242 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9243 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9244 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9245 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9246 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9247 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9248 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9249 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9250 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9251 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9252
9253 if (pDbgState->bmXcptExtra)
9254 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9255
9256 /*
9257 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9258 *
9259 * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
9260 * So, when adding/changing/removing please don't forget to update it.
9261 *
9262 * Some of the macros are picking up local variables to save horizontal space,
9263 * (being able to see it in a table is the lesser evil here).
9264 */
9265#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9266 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9267 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9268#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9269 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9270 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9271 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9272 } else do { } while (0)
9273#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9274 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9275 { \
9276 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9277 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9278 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9279 } else do { } while (0)
9280#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9281 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9282 { \
9283 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9284 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9285 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9286 } else do { } while (0)
9287#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9288 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9289 { \
9290 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9291 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9292 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9293 } else do { } while (0)
9294
9295 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9296 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9297 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9298 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9299 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9300
9301 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9302 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9303 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9304 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9305 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
9306 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9307 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9308 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9309 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
9310 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9311 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
9312 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9313 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
9314 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9315 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9316 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9317 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9318 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9319 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9320 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9321 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9322 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9323 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9324 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9325 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9326 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9327 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9328 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9329 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9330 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9331 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9332 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9333 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9334 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9335 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9336 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9337
9338 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9339 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9340 {
9341 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_APIC_TPR);
9342 AssertRC(rc);
9343
9344#if 0 /** @todo fix me */
9345 pDbgState->fClearCr0Mask = true;
9346 pDbgState->fClearCr4Mask = true;
9347#endif
9348 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9349 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_STORE_EXIT | VMX_PROC_CTLS_CR8_STORE_EXIT;
9350 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9351 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_LOAD_EXIT | VMX_PROC_CTLS_CR8_LOAD_EXIT;
9352 pDbgState->fCpe1Unwanted |= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
9353 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9354 require clearing here and in the loop if we start using it. */
9355 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9356 }
9357 else
9358 {
9359 if (pDbgState->fClearCr0Mask)
9360 {
9361 pDbgState->fClearCr0Mask = false;
9362 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
9363 }
9364 if (pDbgState->fClearCr4Mask)
9365 {
9366 pDbgState->fClearCr4Mask = false;
9367 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
9368 }
9369 }
9370 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9371 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9372
9373 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9374 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9375 {
9376 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9377 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9378 }
9379 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9380 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9381
9382 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
9383 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9384 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
9385 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9386 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
9387 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9388 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
9389 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9390#if 0 /** @todo too slow, fix handler. */
9391 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
9392#endif
9393 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9394
9395 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9396 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9397 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9398 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9399 {
9400 pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
9401 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
9402 }
9403 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9404 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9405 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9406 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9407
9408 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9409 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9410 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9411 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9412 {
9413 pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
9414 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
9415 }
9416 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
9417 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
9418 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
9419 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
9420
9421 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9422 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9423 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
9424 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9425 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9426 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9427 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
9428 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9429 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9430 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9431 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
9432 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9433 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
9434 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9435 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9436 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9437 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
9438 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9439 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9440 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9441 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9442 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9443
9444#undef IS_EITHER_ENABLED
9445#undef SET_ONLY_XBM_IF_EITHER_EN
9446#undef SET_CPE1_XBM_IF_EITHER_EN
9447#undef SET_CPEU_XBM_IF_EITHER_EN
9448#undef SET_CPE2_XBM_IF_EITHER_EN
9449
9450 /*
9451 * Sanitize the control stuff.
9452 */
9453 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
9454 if (pDbgState->fCpe2Extra)
9455 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
9456 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
9457 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0;
9458 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
9459 {
9460 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9461 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9462 }
9463
9464 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9465 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9466 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9467 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9468}
9469
9470
9471/**
9472 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9473 * appropriate.
9474 *
9475 * The caller has checked the VM-exit against the
9476 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9477 * already, so we don't have to do that either.
9478 *
9479 * @returns Strict VBox status code (i.e. informational status codes too).
9480 * @param pVCpu The cross context virtual CPU structure.
9481 * @param pVmxTransient Pointer to the VMX-transient structure.
9482 * @param uExitReason The VM-exit reason.
9483 *
9484 * @remarks The name of this function is displayed by dtrace, so keep it short
9485 * and to the point. No longer than 33 chars long, please.
9486 */
9487static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9488{
9489 /*
9490 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9491 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9492 *
9493 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9494 * does. Must add/change/remove both places. Same ordering, please.
9495 *
9496 * Added/removed events must also be reflected in the next section
9497 * where we dispatch dtrace events.
9498 */
9499 bool fDtrace1 = false;
9500 bool fDtrace2 = false;
9501 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9502 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9503 uint32_t uEventArg = 0;
9504#define SET_EXIT(a_EventSubName) \
9505 do { \
9506 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9507 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9508 } while (0)
9509#define SET_BOTH(a_EventSubName) \
9510 do { \
9511 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9512 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9513 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9514 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9515 } while (0)
9516 switch (uExitReason)
9517 {
9518 case VMX_EXIT_MTF:
9519 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9520
9521 case VMX_EXIT_XCPT_OR_NMI:
9522 {
9523 uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9524 switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
9525 {
9526 case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
9527 case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
9528 case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
9529 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9530 {
9531 if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
9532 {
9533 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9534 uEventArg = pVmxTransient->uExitIntErrorCode;
9535 }
9536 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9537 switch (enmEvent1)
9538 {
9539 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9540 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9541 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9542 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9543 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9544 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9545 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9546 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9547 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9548 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9549 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9550 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9551 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9552 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9553 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9554 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9555 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9556 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9557 default: break;
9558 }
9559 }
9560 else
9561 AssertFailed();
9562 break;
9563
9564 case VMX_EXIT_INT_INFO_TYPE_SW_INT:
9565 uEventArg = idxVector;
9566 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9567 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9568 break;
9569 }
9570 break;
9571 }
9572
9573 case VMX_EXIT_TRIPLE_FAULT:
9574 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9575 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9576 break;
9577 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9578 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9579 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9580 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9581 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9582
9583 /* Instruction specific VM-exits: */
9584 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9585 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9586 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9587 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9588 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9589 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9590 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9591 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9592 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9593 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9594 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9595 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9596 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9597 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9598 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9599 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9600 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9601 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9602 case VMX_EXIT_MOV_CRX:
9603 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9604 if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
9605 SET_BOTH(CRX_READ);
9606 else
9607 SET_BOTH(CRX_WRITE);
9608 uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
9609 break;
9610 case VMX_EXIT_MOV_DRX:
9611 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9612 if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
9613 == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
9614 SET_BOTH(DRX_READ);
9615 else
9616 SET_BOTH(DRX_WRITE);
9617 uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
9618 break;
9619 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9620 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9621 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9622 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9623 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9624 case VMX_EXIT_GDTR_IDTR_ACCESS:
9625 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9626 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
9627 {
9628 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9629 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9630 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9631 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9632 }
9633 break;
9634
9635 case VMX_EXIT_LDTR_TR_ACCESS:
9636 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9637 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
9638 {
9639 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9640 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9641 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9642 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9643 }
9644 break;
9645
9646 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9647 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9648 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9649 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9650 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9651 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9652 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9653 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9654 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9655 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9656 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9657
9658 /* Events that aren't relevant at this point. */
9659 case VMX_EXIT_EXT_INT:
9660 case VMX_EXIT_INT_WINDOW:
9661 case VMX_EXIT_NMI_WINDOW:
9662 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9663 case VMX_EXIT_PREEMPT_TIMER:
9664 case VMX_EXIT_IO_INSTR:
9665 break;
9666
9667 /* Errors and unexpected events. */
9668 case VMX_EXIT_INIT_SIGNAL:
9669 case VMX_EXIT_SIPI:
9670 case VMX_EXIT_IO_SMI:
9671 case VMX_EXIT_SMI:
9672 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9673 case VMX_EXIT_ERR_MSR_LOAD:
9674 case VMX_EXIT_ERR_MACHINE_CHECK:
9675 break;
9676
9677 default:
9678 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9679 break;
9680 }
9681#undef SET_BOTH
9682#undef SET_EXIT
9683
9684 /*
9685 * Dtrace tracepoints go first. We do them here at once so we don't
9686 * have to copy the guest state saving and stuff a few dozen times.
9687 * Down side is that we've got to repeat the switch, though this time
9688 * we use enmEvent since the probes are a subset of what DBGF does.
9689 */
9690 if (fDtrace1 || fDtrace2)
9691 {
9692 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9693 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9694 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9695 switch (enmEvent1)
9696 {
9697 /** @todo consider which extra parameters would be helpful for each probe. */
9698 case DBGFEVENT_END: break;
9699 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
9700 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
9701 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
9702 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
9703 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
9704 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
9705 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
9706 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
9707 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
9708 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
9709 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
9710 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
9711 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
9712 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
9713 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
9714 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
9715 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
9716 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
9717 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9718 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9719 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
9720 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
9721 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
9722 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
9723 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
9724 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
9725 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
9726 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9727 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9728 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9729 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9730 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9731 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
9732 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9733 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
9734 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
9735 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
9736 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
9737 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
9738 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
9739 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
9740 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
9741 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
9742 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
9743 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
9744 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
9745 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
9746 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
9747 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
9748 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
9749 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
9750 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
9751 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
9752 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
9753 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
9754 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
9755 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
9756 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
9757 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
9758 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
9759 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
9760 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
9761 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
9762 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
9763 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
9764 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
9765 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9766 }
9767 switch (enmEvent2)
9768 {
9769 /** @todo consider which extra parameters would be helpful for each probe. */
9770 case DBGFEVENT_END: break;
9771 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
9772 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9773 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
9774 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
9775 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
9776 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
9777 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
9778 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
9779 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
9780 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9781 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9782 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9783 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9784 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9785 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
9786 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9787 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
9788 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
9789 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
9790 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
9791 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
9792 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
9793 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
9794 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
9795 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
9796 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
9797 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
9798 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
9799 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
9800 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
9801 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
9802 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
9803 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
9804 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
9805 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
9806 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
9807 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
9808 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
9809 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
9810 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
9811 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
9812 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
9813 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
9814 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
9815 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
9816 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
9817 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
9818 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
9819 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
9820 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
9821 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
9822 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
9823 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9824 }
9825 }
9826
9827 /*
9828 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9829 * the DBGF call will do a full check).
9830 *
9831 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9832 * Note! If we have to events, we prioritize the first, i.e. the instruction
9833 * one, in order to avoid event nesting.
9834 */
9835 PVM pVM = pVCpu->CTX_SUFF(pVM);
9836 if ( enmEvent1 != DBGFEVENT_END
9837 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9838 {
9839 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
9840 if (rcStrict != VINF_SUCCESS)
9841 return rcStrict;
9842 }
9843 else if ( enmEvent2 != DBGFEVENT_END
9844 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9845 {
9846 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
9847 if (rcStrict != VINF_SUCCESS)
9848 return rcStrict;
9849 }
9850
9851 return VINF_SUCCESS;
9852}
9853
9854
9855/**
9856 * Single-stepping VM-exit filtering.
9857 *
9858 * This is preprocessing the VM-exits and deciding whether we've gotten far
9859 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9860 * handling is performed.
9861 *
9862 * @returns Strict VBox status code (i.e. informational status codes too).
9863 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9864 * @param pVmxTransient Pointer to the VMX-transient structure.
9865 * @param pDbgState The debug state.
9866 */
9867DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
9868{
9869 /*
9870 * Expensive (saves context) generic dtrace VM-exit probe.
9871 */
9872 uint32_t const uExitReason = pVmxTransient->uExitReason;
9873 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9874 { /* more likely */ }
9875 else
9876 {
9877 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9878 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9879 AssertRC(rc);
9880 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
9881 }
9882
9883 /*
9884 * Check for host NMI, just to get that out of the way.
9885 */
9886 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9887 { /* normally likely */ }
9888 else
9889 {
9890 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9891 AssertRCReturn(rc2, rc2);
9892 uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
9893 if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
9894 return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
9895 }
9896
9897 /*
9898 * Check for single stepping event if we're stepping.
9899 */
9900 if (pVCpu->hm.s.fSingleInstruction)
9901 {
9902 switch (uExitReason)
9903 {
9904 case VMX_EXIT_MTF:
9905 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9906
9907 /* Various events: */
9908 case VMX_EXIT_XCPT_OR_NMI:
9909 case VMX_EXIT_EXT_INT:
9910 case VMX_EXIT_TRIPLE_FAULT:
9911 case VMX_EXIT_INT_WINDOW:
9912 case VMX_EXIT_NMI_WINDOW:
9913 case VMX_EXIT_TASK_SWITCH:
9914 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9915 case VMX_EXIT_APIC_ACCESS:
9916 case VMX_EXIT_EPT_VIOLATION:
9917 case VMX_EXIT_EPT_MISCONFIG:
9918 case VMX_EXIT_PREEMPT_TIMER:
9919
9920 /* Instruction specific VM-exits: */
9921 case VMX_EXIT_CPUID:
9922 case VMX_EXIT_GETSEC:
9923 case VMX_EXIT_HLT:
9924 case VMX_EXIT_INVD:
9925 case VMX_EXIT_INVLPG:
9926 case VMX_EXIT_RDPMC:
9927 case VMX_EXIT_RDTSC:
9928 case VMX_EXIT_RSM:
9929 case VMX_EXIT_VMCALL:
9930 case VMX_EXIT_VMCLEAR:
9931 case VMX_EXIT_VMLAUNCH:
9932 case VMX_EXIT_VMPTRLD:
9933 case VMX_EXIT_VMPTRST:
9934 case VMX_EXIT_VMREAD:
9935 case VMX_EXIT_VMRESUME:
9936 case VMX_EXIT_VMWRITE:
9937 case VMX_EXIT_VMXOFF:
9938 case VMX_EXIT_VMXON:
9939 case VMX_EXIT_MOV_CRX:
9940 case VMX_EXIT_MOV_DRX:
9941 case VMX_EXIT_IO_INSTR:
9942 case VMX_EXIT_RDMSR:
9943 case VMX_EXIT_WRMSR:
9944 case VMX_EXIT_MWAIT:
9945 case VMX_EXIT_MONITOR:
9946 case VMX_EXIT_PAUSE:
9947 case VMX_EXIT_GDTR_IDTR_ACCESS:
9948 case VMX_EXIT_LDTR_TR_ACCESS:
9949 case VMX_EXIT_INVEPT:
9950 case VMX_EXIT_RDTSCP:
9951 case VMX_EXIT_INVVPID:
9952 case VMX_EXIT_WBINVD:
9953 case VMX_EXIT_XSETBV:
9954 case VMX_EXIT_RDRAND:
9955 case VMX_EXIT_INVPCID:
9956 case VMX_EXIT_VMFUNC:
9957 case VMX_EXIT_RDSEED:
9958 case VMX_EXIT_XSAVES:
9959 case VMX_EXIT_XRSTORS:
9960 {
9961 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
9962 AssertRCReturn(rc, rc);
9963 if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
9964 || pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
9965 return VINF_EM_DBG_STEPPED;
9966 break;
9967 }
9968
9969 /* Errors and unexpected events: */
9970 case VMX_EXIT_INIT_SIGNAL:
9971 case VMX_EXIT_SIPI:
9972 case VMX_EXIT_IO_SMI:
9973 case VMX_EXIT_SMI:
9974 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9975 case VMX_EXIT_ERR_MSR_LOAD:
9976 case VMX_EXIT_ERR_MACHINE_CHECK:
9977 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9978 break;
9979
9980 default:
9981 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9982 break;
9983 }
9984 }
9985
9986 /*
9987 * Check for debugger event breakpoints and dtrace probes.
9988 */
9989 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9990 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9991 {
9992 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
9993 if (rcStrict != VINF_SUCCESS)
9994 return rcStrict;
9995 }
9996
9997 /*
9998 * Normal processing.
9999 */
10000#ifdef HMVMX_USE_FUNCTION_TABLE
10001 return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
10002#else
10003 return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
10004#endif
10005}
10006
10007
10008/**
10009 * Single steps guest code using VT-x.
10010 *
10011 * @returns Strict VBox status code (i.e. informational status codes too).
10012 * @param pVCpu The cross context virtual CPU structure.
10013 *
10014 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10015 */
10016static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu)
10017{
10018 VMXTRANSIENT VmxTransient;
10019 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10020
10021 /* Set HMCPU indicators. */
10022 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10023 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
10024 pVCpu->hm.s.fDebugWantRdTscExit = false;
10025 pVCpu->hm.s.fUsingDebugLoop = true;
10026
10027 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10028 VMXRUNDBGSTATE DbgState;
10029 hmR0VmxRunDebugStateInit(pVCpu, &DbgState);
10030 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
10031
10032 /*
10033 * The loop.
10034 */
10035 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10036 for (uint32_t cLoops = 0; ; cLoops++)
10037 {
10038 Assert(!HMR0SuspendPending());
10039 HMVMX_ASSERT_CPU_SAFE(pVCpu);
10040 bool fStepping = pVCpu->hm.s.fSingleInstruction;
10041
10042 /*
10043 * Preparatory work for running guest code, this may force us to return
10044 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10045 */
10046 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10047 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10048 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
10049 if (rcStrict != VINF_SUCCESS)
10050 break;
10051
10052 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10053 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10054
10055 /*
10056 * Now we can run the guest code.
10057 */
10058 int rcRun = hmR0VmxRunGuest(pVCpu);
10059
10060 /*
10061 * Restore any residual host-state and save any bits shared between host
10062 * and guest into the guest-CPU state. Re-enables interrupts!
10063 */
10064 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10065
10066 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10067 if (RT_SUCCESS(rcRun))
10068 { /* very likely */ }
10069 else
10070 {
10071 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10072 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10073 return rcRun;
10074 }
10075
10076 /* Profile the VM-exit. */
10077 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10078 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10079 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10080 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10081 HMVMX_START_EXIT_DISPATCH_PROF();
10082
10083 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10084
10085 /*
10086 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10087 */
10088 rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
10089 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10090 if (rcStrict != VINF_SUCCESS)
10091 break;
10092 if (cLoops > pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
10093 {
10094 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10095 rcStrict = VINF_EM_RAW_INTERRUPT;
10096 break;
10097 }
10098
10099 /*
10100 * Stepping: Did the RIP change, if so, consider it a single step.
10101 * Otherwise, make sure one of the TFs gets set.
10102 */
10103 if (fStepping)
10104 {
10105 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
10106 AssertRC(rc);
10107 if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
10108 || pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
10109 {
10110 rcStrict = VINF_EM_DBG_STEPPED;
10111 break;
10112 }
10113 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
10114 }
10115
10116 /*
10117 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
10118 */
10119 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10120 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
10121 }
10122
10123 /*
10124 * Clear the X86_EFL_TF if necessary.
10125 */
10126 if (pVCpu->hm.s.fClearTrapFlag)
10127 {
10128 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
10129 AssertRC(rc);
10130 pVCpu->hm.s.fClearTrapFlag = false;
10131 pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
10132 }
10133 /** @todo there seems to be issues with the resume flag when the monitor trap
10134 * flag is pending without being used. Seen early in bios init when
10135 * accessing APIC page in protected mode. */
10136
10137 /*
10138 * Restore VM-exit control settings as we may not reenter this function the
10139 * next time around.
10140 */
10141 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10142
10143 /* Restore HMCPU indicators. */
10144 pVCpu->hm.s.fUsingDebugLoop = false;
10145 pVCpu->hm.s.fDebugWantRdTscExit = false;
10146 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10147
10148 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10149 return rcStrict;
10150}
10151
10152
10153/** @} */
10154
10155
10156/**
10157 * Checks if any expensive dtrace probes are enabled and we should go to the
10158 * debug loop.
10159 *
10160 * @returns true if we should use debug loop, false if not.
10161 */
10162static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10163{
10164 /* It's probably faster to OR the raw 32-bit counter variables together.
10165 Since the variables are in an array and the probes are next to one
10166 another (more or less), we have good locality. So, better read
10167 eight-nine cache lines ever time and only have one conditional, than
10168 128+ conditionals, right? */
10169 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10170 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10171 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10172 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10173 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10174 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10175 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10176 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10177 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10178 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10179 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10180 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10181 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10182 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10183 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10184 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10185 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10186 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10187 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10188 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10189 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10190 ) != 0
10191 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10192 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10193 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10194 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10195 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10196 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10197 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10198 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10199 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10200 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10201 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10202 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10203 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10204 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10205 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10206 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10207 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10208 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10209 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10210 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10211 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10212 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10213 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10214 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10215 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10216 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10217 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10218 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10219 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10220 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10221 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10222 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10223 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10224 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10225 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10226 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10227 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10228 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10229 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10230 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10231 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10232 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10233 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10234 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10235 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10236 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10237 ) != 0
10238 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10239 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10240 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10241 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10242 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10243 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10244 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10245 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10246 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10247 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10248 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10249 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10250 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10251 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10252 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10253 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10254 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10255 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10256 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10257 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10258 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10259 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10260 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10261 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10262 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10263 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10264 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10265 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10266 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10267 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10268 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10269 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10270 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10271 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10272 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10273 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10274 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10275 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10276 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10277 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10278 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10279 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10280 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10281 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10282 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10283 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10284 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10285 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10286 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10287 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10288 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10289 ) != 0;
10290}
10291
10292
10293/**
10294 * Runs the guest code using VT-x.
10295 *
10296 * @returns Strict VBox status code (i.e. informational status codes too).
10297 * @param pVCpu The cross context virtual CPU structure.
10298 */
10299VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
10300{
10301 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10302 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10303 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10304 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
10305
10306 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10307
10308 VBOXSTRICTRC rcStrict;
10309 if ( !pVCpu->hm.s.fUseDebugLoop
10310 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10311 && !DBGFIsStepping(pVCpu)
10312 && !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
10313 rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu);
10314 else
10315 rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu);
10316
10317 if (rcStrict == VERR_EM_INTERPRETER)
10318 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10319 else if (rcStrict == VINF_EM_RESET)
10320 rcStrict = VINF_EM_TRIPLE_FAULT;
10321
10322 int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
10323 if (RT_FAILURE(rc2))
10324 {
10325 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10326 rcStrict = rc2;
10327 }
10328 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10329 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10330 return rcStrict;
10331}
10332
10333
10334#ifndef HMVMX_USE_FUNCTION_TABLE
10335DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10336{
10337#ifdef DEBUG_ramshankar
10338#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
10339 do { \
10340 if (a_fSave != 0) \
10341 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
10342 VBOXSTRICTRC rcStrict = a_CallExpr; \
10343 if (a_fSave != 0) \
10344 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
10345 return rcStrict; \
10346 } while (0)
10347#else
10348# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
10349#endif
10350 switch (rcReason)
10351 {
10352 case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
10353 case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
10354 case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
10355 case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
10356 case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
10357 case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
10358 case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
10359 case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
10360 case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
10361 case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
10362 case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
10363 case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
10364 case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
10365 case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
10366 case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
10367 case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
10368 case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
10369 case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
10370 case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
10371 case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
10372 case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
10373 case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
10374 case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
10375 case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pVmxTransient));
10376 case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
10377 case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
10378 case VMX_EXIT_GDTR_IDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10379 case VMX_EXIT_LDTR_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10380 case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
10381 case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
10382 case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
10383 case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
10384 case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
10385 case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
10386
10387 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
10388 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
10389 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pVmxTransient);
10390 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pVmxTransient);
10391 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pVmxTransient);
10392 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pVmxTransient);
10393 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pVmxTransient);
10394 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
10395 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pVmxTransient);
10396
10397 case VMX_EXIT_VMCLEAR:
10398 case VMX_EXIT_VMLAUNCH:
10399 case VMX_EXIT_VMPTRLD:
10400 case VMX_EXIT_VMPTRST:
10401 case VMX_EXIT_VMREAD:
10402 case VMX_EXIT_VMRESUME:
10403 case VMX_EXIT_VMWRITE:
10404 case VMX_EXIT_VMXOFF:
10405 case VMX_EXIT_VMXON:
10406 case VMX_EXIT_INVEPT:
10407 case VMX_EXIT_INVVPID:
10408 case VMX_EXIT_VMFUNC:
10409 case VMX_EXIT_XSAVES:
10410 case VMX_EXIT_XRSTORS:
10411 return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
10412
10413 case VMX_EXIT_ENCLS:
10414 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10415 case VMX_EXIT_PML_FULL:
10416 default:
10417 return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
10418 }
10419#undef VMEXIT_CALL_RET
10420}
10421#endif /* !HMVMX_USE_FUNCTION_TABLE */
10422
10423
10424#ifdef VBOX_STRICT
10425/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10426# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10427 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10428
10429# define HMVMX_ASSERT_PREEMPT_CPUID() \
10430 do { \
10431 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10432 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10433 } while (0)
10434
10435# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10436 do { \
10437 AssertPtr((a_pVCpu)); \
10438 AssertPtr((a_pVmxTransient)); \
10439 Assert((a_pVmxTransient)->fVMEntryFailed == false); \
10440 Assert(ASMIntAreEnabled()); \
10441 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10442 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10443 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
10444 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10445 if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
10446 HMVMX_ASSERT_PREEMPT_CPUID(); \
10447 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10448 } while (0)
10449
10450# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10451 do { \
10452 Log4Func(("\n")); \
10453 } while (0)
10454#else
10455# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10456 do { \
10457 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10458 NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
10459 } while (0)
10460# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
10461#endif
10462
10463
10464/**
10465 * Advances the guest RIP by the specified number of bytes.
10466 *
10467 * @param pVCpu The cross context virtual CPU structure.
10468 * @param cbInstr Number of bytes to advance the RIP by.
10469 *
10470 * @remarks No-long-jump zone!!!
10471 */
10472DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
10473{
10474 /* Advance the RIP. */
10475 pVCpu->cpum.GstCtx.rip += cbInstr;
10476 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
10477
10478 /* Update interrupt inhibition. */
10479 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10480 && pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
10481 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10482}
10483
10484
10485/**
10486 * Advances the guest RIP after reading it from the VMCS.
10487 *
10488 * @returns VBox status code, no informational status codes.
10489 * @param pVCpu The cross context virtual CPU structure.
10490 * @param pVmxTransient Pointer to the VMX transient structure.
10491 *
10492 * @remarks No-long-jump zone!!!
10493 */
10494static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10495{
10496 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10497 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS);
10498 AssertRCReturn(rc, rc);
10499
10500 hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
10501
10502 /*
10503 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10504 * pending debug exception field as it takes care of priority of events.
10505 *
10506 * See Intel spec. 32.2.1 "Debug Exceptions".
10507 */
10508 if ( !pVCpu->hm.s.fSingleInstruction
10509 && pVCpu->cpum.GstCtx.eflags.Bits.u1TF)
10510 {
10511 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10512 AssertRCReturn(rc, rc);
10513 }
10514
10515 return VINF_SUCCESS;
10516}
10517
10518
10519/**
10520 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10521 * and update error record fields accordingly.
10522 *
10523 * @return VMX_IGS_* return codes.
10524 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10525 * wrong with the guest state.
10526 *
10527 * @param pVCpu The cross context virtual CPU structure.
10528 *
10529 * @remarks This function assumes our cache of the VMCS controls
10530 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10531 */
10532static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu)
10533{
10534#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10535#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10536 uError = (err); \
10537 break; \
10538 } else do { } while (0)
10539
10540 int rc;
10541 PVM pVM = pVCpu->CTX_SUFF(pVM);
10542 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10543 uint32_t uError = VMX_IGS_ERROR;
10544 uint32_t u32Val;
10545 bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10546
10547 do
10548 {
10549 /*
10550 * CR0.
10551 */
10552 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10553 uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10554 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10555 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10556 if (fUnrestrictedGuest)
10557 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
10558
10559 uint32_t u32GuestCr0;
10560 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10561 AssertRCBreak(rc);
10562 HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10563 HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10564 if ( !fUnrestrictedGuest
10565 && (u32GuestCr0 & X86_CR0_PG)
10566 && !(u32GuestCr0 & X86_CR0_PE))
10567 {
10568 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10569 }
10570
10571 /*
10572 * CR4.
10573 */
10574 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10575 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10576
10577 uint32_t u32GuestCr4;
10578 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10579 AssertRCBreak(rc);
10580 HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10581 HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10582
10583 /*
10584 * IA32_DEBUGCTL MSR.
10585 */
10586 uint64_t u64Val;
10587 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10588 AssertRCBreak(rc);
10589 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10590 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10591 {
10592 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10593 }
10594 uint64_t u64DebugCtlMsr = u64Val;
10595
10596#ifdef VBOX_STRICT
10597 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10598 AssertRCBreak(rc);
10599 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10600#endif
10601 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
10602
10603 /*
10604 * RIP and RFLAGS.
10605 */
10606 uint32_t u32Eflags;
10607#if HC_ARCH_BITS == 64
10608 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10609 AssertRCBreak(rc);
10610 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10611 if ( !fLongModeGuest
10612 || !pCtx->cs.Attr.n.u1Long)
10613 {
10614 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10615 }
10616 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10617 * must be identical if the "IA-32e mode guest" VM-entry
10618 * control is 1 and CS.L is 1. No check applies if the
10619 * CPU supports 64 linear-address bits. */
10620
10621 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10622 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10623 AssertRCBreak(rc);
10624 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10625 VMX_IGS_RFLAGS_RESERVED);
10626 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10627 u32Eflags = u64Val;
10628#else
10629 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10630 AssertRCBreak(rc);
10631 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10632 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10633#endif
10634
10635 if ( fLongModeGuest
10636 || ( fUnrestrictedGuest
10637 && !(u32GuestCr0 & X86_CR0_PE)))
10638 {
10639 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10640 }
10641
10642 uint32_t u32EntryInfo;
10643 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10644 AssertRCBreak(rc);
10645 if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
10646 && VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
10647 {
10648 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10649 }
10650
10651 /*
10652 * 64-bit checks.
10653 */
10654#if HC_ARCH_BITS == 64
10655 if (fLongModeGuest)
10656 {
10657 HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10658 HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10659 }
10660
10661 if ( !fLongModeGuest
10662 && (u32GuestCr4 & X86_CR4_PCIDE))
10663 {
10664 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10665 }
10666
10667 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10668 * 51:32 beyond the processor's physical-address width are 0. */
10669
10670 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10671 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10672 {
10673 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10674 }
10675
10676 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10677 AssertRCBreak(rc);
10678 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10679
10680 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10681 AssertRCBreak(rc);
10682 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10683#endif
10684
10685 /*
10686 * PERF_GLOBAL MSR.
10687 */
10688 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
10689 {
10690 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10691 AssertRCBreak(rc);
10692 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10693 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10694 }
10695
10696 /*
10697 * PAT MSR.
10698 */
10699 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
10700 {
10701 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10702 AssertRCBreak(rc);
10703 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10704 for (unsigned i = 0; i < 8; i++)
10705 {
10706 uint8_t u8Val = (u64Val & 0xff);
10707 if ( u8Val != 0 /* UC */
10708 && u8Val != 1 /* WC */
10709 && u8Val != 4 /* WT */
10710 && u8Val != 5 /* WP */
10711 && u8Val != 6 /* WB */
10712 && u8Val != 7 /* UC- */)
10713 {
10714 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10715 }
10716 u64Val >>= 8;
10717 }
10718 }
10719
10720 /*
10721 * EFER MSR.
10722 */
10723 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
10724 {
10725 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10726 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10727 AssertRCBreak(rc);
10728 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10729 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10730 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10731 & VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
10732 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10733 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10734 || !(u32GuestCr0 & X86_CR0_PG)
10735 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10736 VMX_IGS_EFER_LMA_LME_MISMATCH);
10737 }
10738
10739 /*
10740 * Segment registers.
10741 */
10742 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10743 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10744 if (!(u32Eflags & X86_EFL_VM))
10745 {
10746 /* CS */
10747 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10748 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10749 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10750 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10751 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10752 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10753 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10754 /* CS cannot be loaded with NULL in protected mode. */
10755 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10756 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10757 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10758 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10759 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10760 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10761 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10762 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10763 else
10764 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10765
10766 /* SS */
10767 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10768 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10769 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10770 if ( !(pCtx->cr0 & X86_CR0_PE)
10771 || pCtx->cs.Attr.n.u4Type == 3)
10772 {
10773 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10774 }
10775 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10776 {
10777 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10778 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10779 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10780 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10781 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10782 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10783 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10784 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10785 }
10786
10787 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmenReg(). */
10788 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10789 {
10790 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10791 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10792 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10793 || pCtx->ds.Attr.n.u4Type > 11
10794 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10795 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10796 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10797 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10798 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10799 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10800 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10801 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10802 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10803 }
10804 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10805 {
10806 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10807 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10808 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10809 || pCtx->es.Attr.n.u4Type > 11
10810 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10811 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10812 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10813 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10814 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10815 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10816 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10817 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10818 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10819 }
10820 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10821 {
10822 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10823 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10824 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10825 || pCtx->fs.Attr.n.u4Type > 11
10826 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10827 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10828 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10829 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10830 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10831 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10832 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10833 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10834 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10835 }
10836 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10837 {
10838 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10839 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10840 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10841 || pCtx->gs.Attr.n.u4Type > 11
10842 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10843 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10844 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10845 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10846 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10847 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10848 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10849 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10850 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10851 }
10852 /* 64-bit capable CPUs. */
10853#if HC_ARCH_BITS == 64
10854 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10855 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10856 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10857 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10858 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10859 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10860 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10861 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10862 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10863 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10864 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10865#endif
10866 }
10867 else
10868 {
10869 /* V86 mode checks. */
10870 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10871 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10872 {
10873 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10874 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10875 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10876 }
10877 else
10878 {
10879 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10880 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10881 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10882 }
10883
10884 /* CS */
10885 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10886 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10887 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10888 /* SS */
10889 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10890 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10891 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10892 /* DS */
10893 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10894 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10895 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10896 /* ES */
10897 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10898 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10899 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10900 /* FS */
10901 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10902 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10903 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10904 /* GS */
10905 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10906 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10907 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10908 /* 64-bit capable CPUs. */
10909#if HC_ARCH_BITS == 64
10910 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10911 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10912 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10913 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10914 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10915 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10916 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10917 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10918 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10919 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10920 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10921#endif
10922 }
10923
10924 /*
10925 * TR.
10926 */
10927 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10928 /* 64-bit capable CPUs. */
10929#if HC_ARCH_BITS == 64
10930 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10931#endif
10932 if (fLongModeGuest)
10933 {
10934 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10935 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10936 }
10937 else
10938 {
10939 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10940 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10941 VMX_IGS_TR_ATTR_TYPE_INVALID);
10942 }
10943 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10944 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10945 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10946 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10947 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10948 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10949 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10950 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10951
10952 /*
10953 * GDTR and IDTR.
10954 */
10955#if HC_ARCH_BITS == 64
10956 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10957 AssertRCBreak(rc);
10958 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10959
10960 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10961 AssertRCBreak(rc);
10962 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10963#endif
10964
10965 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10966 AssertRCBreak(rc);
10967 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10968
10969 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10970 AssertRCBreak(rc);
10971 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10972
10973 /*
10974 * Guest Non-Register State.
10975 */
10976 /* Activity State. */
10977 uint32_t u32ActivityState;
10978 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10979 AssertRCBreak(rc);
10980 HMVMX_CHECK_BREAK( !u32ActivityState
10981 || (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
10982 VMX_IGS_ACTIVITY_STATE_INVALID);
10983 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10984 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10985 uint32_t u32IntrState;
10986 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
10987 AssertRCBreak(rc);
10988 if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
10989 || u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10990 {
10991 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10992 }
10993
10994 /** @todo Activity state and injecting interrupts. Left as a todo since we
10995 * currently don't use activity states but ACTIVE. */
10996
10997 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10998 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10999
11000 /* Guest interruptibility-state. */
11001 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11002 HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
11003 != (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
11004 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11005 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11006 || !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
11007 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11008 if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
11009 {
11010 if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
11011 {
11012 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
11013 && !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
11014 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11015 }
11016 else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
11017 {
11018 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
11019 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11020 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
11021 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11022 }
11023 }
11024 /** @todo Assumes the processor is not in SMM. */
11025 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
11026 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11027 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
11028 || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
11029 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11030 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
11031 && VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
11032 && VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
11033 {
11034 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
11035 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11036 }
11037
11038 /* Pending debug exceptions. */
11039#if HC_ARCH_BITS == 64
11040 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
11041 AssertRCBreak(rc);
11042 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11043 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11044 u32Val = u64Val; /* For pending debug exceptions checks below. */
11045#else
11046 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
11047 AssertRCBreak(rc);
11048 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11049 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11050#endif
11051
11052 if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
11053 || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
11054 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11055 {
11056 if ( (u32Eflags & X86_EFL_TF)
11057 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11058 {
11059 /* Bit 14 is PendingDebug.BS. */
11060 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11061 }
11062 if ( !(u32Eflags & X86_EFL_TF)
11063 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11064 {
11065 /* Bit 14 is PendingDebug.BS. */
11066 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11067 }
11068 }
11069
11070 /* VMCS link pointer. */
11071 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11072 AssertRCBreak(rc);
11073 if (u64Val != UINT64_C(0xffffffffffffffff))
11074 {
11075 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11076 /** @todo Bits beyond the processor's physical-address width MBZ. */
11077 /** @todo 32-bit located in memory referenced by value of this field (as a
11078 * physical address) must contain the processor's VMCS revision ID. */
11079 /** @todo SMM checks. */
11080 }
11081
11082 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11083 * not using Nested Paging? */
11084 if ( pVM->hm.s.fNestedPaging
11085 && !fLongModeGuest
11086 && CPUMIsGuestInPAEModeEx(pCtx))
11087 {
11088 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11089 AssertRCBreak(rc);
11090 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11091
11092 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11093 AssertRCBreak(rc);
11094 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11095
11096 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11097 AssertRCBreak(rc);
11098 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11099
11100 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11101 AssertRCBreak(rc);
11102 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11103 }
11104
11105 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11106 if (uError == VMX_IGS_ERROR)
11107 uError = VMX_IGS_REASON_NOT_FOUND;
11108 } while (0);
11109
11110 pVCpu->hm.s.u32HMError = uError;
11111 return uError;
11112
11113#undef HMVMX_ERROR_BREAK
11114#undef HMVMX_CHECK_BREAK
11115}
11116
11117
11118/** @name VM-exit handlers.
11119 * @{
11120 */
11121/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11122/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11123/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11124
11125/**
11126 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11127 */
11128HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11129{
11130 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11131 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11132 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11133 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11134 return VINF_SUCCESS;
11135 return VINF_EM_RAW_INTERRUPT;
11136}
11137
11138
11139/**
11140 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11141 */
11142HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11143{
11144 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11145 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11146
11147 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11148 AssertRCReturn(rc, rc);
11149
11150 uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
11151 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
11152 && uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
11153 Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11154
11155 if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
11156 {
11157 /*
11158 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
11159 * injected it ourselves and anything we inject is not going to cause a VM-exit directly
11160 * for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
11161 *
11162 * [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11163 * [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
11164 */
11165 VMXDispatchHostNmi();
11166 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11167 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11168 return VINF_SUCCESS;
11169 }
11170
11171 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11172 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
11173 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11174 { /* likely */ }
11175 else
11176 {
11177 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11178 rcStrictRc1 = VINF_SUCCESS;
11179 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11180 return rcStrictRc1;
11181 }
11182
11183 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11184 uint32_t uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
11185 switch (uIntType)
11186 {
11187 case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11188 Assert(uVector == X86_XCPT_DB);
11189 RT_FALL_THRU();
11190 case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11191 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
11192 RT_FALL_THRU();
11193 case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11194 {
11195 /*
11196 * If there's any exception caused as a result of event injection, the resulting
11197 * secondary/final execption will be pending, we shall continue guest execution
11198 * after injecting the event. The page-fault case is complicated and we manually
11199 * handle any currently pending event in hmR0VmxExitXcptPF.
11200 */
11201 if (!pVCpu->hm.s.Event.fPending)
11202 { /* likely */ }
11203 else if (uVector != X86_XCPT_PF)
11204 {
11205 rc = VINF_SUCCESS;
11206 break;
11207 }
11208
11209 switch (uVector)
11210 {
11211 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
11212 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
11213 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
11214 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
11215 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
11216 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
11217
11218 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11219 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11220 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11221 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11222 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11223 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11224 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11225 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11226 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11227 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11228 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11229 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11230 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11231 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11232 default:
11233 {
11234 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11235 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11236 {
11237 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11238 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11239 Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
11240
11241 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
11242 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11243 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11244 AssertRCReturn(rc, rc);
11245 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11246 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11247 0 /* GCPtrFaultAddress */);
11248 }
11249 else
11250 {
11251 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11252 pVCpu->hm.s.u32HMError = uVector;
11253 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11254 }
11255 break;
11256 }
11257 }
11258 break;
11259 }
11260
11261 default:
11262 {
11263 pVCpu->hm.s.u32HMError = uExitIntInfo;
11264 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11265 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
11266 break;
11267 }
11268 }
11269 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11270 return rc;
11271}
11272
11273
11274/**
11275 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11276 */
11277HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11278{
11279 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11280
11281 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11282 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11283
11284 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11285 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11286 return VINF_SUCCESS;
11287}
11288
11289
11290/**
11291 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11292 */
11293HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11294{
11295 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11296 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)))
11297 {
11298 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11299 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11300 }
11301
11302 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11303
11304 /*
11305 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11306 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11307 */
11308 uint32_t fIntrState = 0;
11309 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
11310 AssertRCReturn(rc, rc);
11311
11312 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
11313 if ( fBlockSti
11314 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11315 {
11316 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11317 }
11318
11319 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11320 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11321
11322 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11323 return VINF_SUCCESS;
11324}
11325
11326
11327/**
11328 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11329 */
11330HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11331{
11332 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11333 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11334}
11335
11336
11337/**
11338 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11339 */
11340HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11341{
11342 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11343 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11344}
11345
11346
11347/**
11348 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11349 */
11350HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11351{
11352 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11353
11354 /*
11355 * Get the state we need and update the exit history entry.
11356 */
11357 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11358 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX);
11359 AssertRCReturn(rc, rc);
11360
11361 VBOXSTRICTRC rcStrict;
11362 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11363 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
11364 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11365 if (!pExitRec)
11366 {
11367 /*
11368 * Regular CPUID instruction execution.
11369 */
11370 rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
11371 if (rcStrict == VINF_SUCCESS)
11372 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_RAX
11373 | HM_CHANGED_GUEST_RCX | HM_CHANGED_GUEST_RDX | HM_CHANGED_GUEST_RBX);
11374 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11375 {
11376 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11377 rcStrict = VINF_SUCCESS;
11378 }
11379 }
11380 else
11381 {
11382 /*
11383 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
11384 */
11385 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11386 AssertRCReturn(rc2, rc2);
11387
11388 Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11389 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11390
11391 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11392 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
11393
11394 Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11395 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11396 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11397 }
11398 return rcStrict;
11399}
11400
11401
11402/**
11403 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11404 */
11405HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11406{
11407 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11408 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4);
11409 AssertRCReturn(rc, rc);
11410
11411 if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
11412 return VINF_EM_RAW_EMULATE_INSTR;
11413
11414 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11415 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11416}
11417
11418
11419/**
11420 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11421 */
11422HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11423{
11424 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11425 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11426 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11427 AssertRCReturn(rc, rc);
11428
11429 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11430 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11431 {
11432 /* If we get a spurious VM-exit when offsetting is enabled,
11433 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11434 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
11435 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11436 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11437 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX);
11438 }
11439 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11440 {
11441 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11442 rcStrict = VINF_SUCCESS;
11443 }
11444 return rcStrict;
11445}
11446
11447
11448/**
11449 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11450 */
11451HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11452{
11453 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11454 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_TSC_AUX);
11455 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11456 AssertRCReturn(rc, rc);
11457
11458 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11459 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11460 {
11461 /* If we get a spurious VM-exit when offsetting is enabled,
11462 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11463 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
11464 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11465 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11466 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX | HM_CHANGED_GUEST_RCX);
11467 }
11468 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11469 {
11470 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11471 rcStrict = VINF_SUCCESS;
11472 }
11473 return rcStrict;
11474}
11475
11476
11477/**
11478 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11479 */
11480HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11481{
11482 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11483 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11484 AssertRCReturn(rc, rc);
11485
11486 PVM pVM = pVCpu->CTX_SUFF(pVM);
11487 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11488 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11489 if (RT_LIKELY(rc == VINF_SUCCESS))
11490 {
11491 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11492 Assert(pVmxTransient->cbInstr == 2);
11493 }
11494 else
11495 {
11496 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11497 rc = VERR_EM_INTERPRETER;
11498 }
11499 return rc;
11500}
11501
11502
11503/**
11504 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11505 */
11506HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11507{
11508 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11509
11510 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11511 if (EMAreHypercallInstructionsEnabled(pVCpu))
11512 {
11513 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_SS
11514 | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_EFER);
11515 AssertRCReturn(rc, rc);
11516
11517 /* Perform the hypercall. */
11518 rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
11519 if (rcStrict == VINF_SUCCESS)
11520 {
11521 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11522 AssertRCReturn(rc, rc);
11523 }
11524 else
11525 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11526 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11527 || RT_FAILURE(rcStrict));
11528
11529 /* If the hypercall changes anything other than guest's general-purpose registers,
11530 we would need to reload the guest changed bits here before VM-entry. */
11531 }
11532 else
11533 Log4Func(("Hypercalls not enabled\n"));
11534
11535 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11536 if (RT_FAILURE(rcStrict))
11537 {
11538 hmR0VmxSetPendingXcptUD(pVCpu);
11539 rcStrict = VINF_SUCCESS;
11540 }
11541
11542 return rcStrict;
11543}
11544
11545
11546/**
11547 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11548 */
11549HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11550{
11551 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11552 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11553
11554 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11555 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11556 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_DS);
11557 AssertRCReturn(rc, rc);
11558
11559 VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
11560
11561 if (rcStrict == VINF_SUCCESS || rcStrict == VINF_PGM_SYNC_CR3)
11562 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11563 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11564 {
11565 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11566 rcStrict = VINF_SUCCESS;
11567 }
11568 else
11569 AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) sttus: %Rrc\n", pVmxTransient->uExitQual,
11570 VBOXSTRICTRC_VAL(rcStrict)));
11571 return rcStrict;
11572}
11573
11574
11575/**
11576 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11577 */
11578HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11579{
11580 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11581 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11582 AssertRCReturn(rc, rc);
11583
11584 PVM pVM = pVCpu->CTX_SUFF(pVM);
11585 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11586 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11587 if (RT_LIKELY(rc == VINF_SUCCESS))
11588 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11589 else
11590 {
11591 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11592 rc = VERR_EM_INTERPRETER;
11593 }
11594 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11595 return rc;
11596}
11597
11598
11599/**
11600 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11601 */
11602HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11603{
11604 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11605 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11606 AssertRCReturn(rc, rc);
11607
11608 PVM pVM = pVCpu->CTX_SUFF(pVM);
11609 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11610 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11611 rc = VBOXSTRICTRC_VAL(rc2);
11612 if (RT_LIKELY( rc == VINF_SUCCESS
11613 || rc == VINF_EM_HALT))
11614 {
11615 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11616 AssertRCReturn(rc3, rc3);
11617
11618 if ( rc == VINF_EM_HALT
11619 && EMMonitorWaitShouldContinue(pVCpu, pCtx))
11620 rc = VINF_SUCCESS;
11621 }
11622 else
11623 {
11624 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11625 rc = VERR_EM_INTERPRETER;
11626 }
11627 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11628 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11629 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11630 return rc;
11631}
11632
11633
11634/**
11635 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11636 */
11637HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11638{
11639 /*
11640 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
11641 * mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
11642 * treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
11643 * VMX root operation. If we get here, something funny is going on.
11644 *
11645 * See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
11646 */
11647 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11648 AssertMsgFailed(("Unexpected RSM VM-exit\n"));
11649 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11650}
11651
11652
11653/**
11654 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11655 */
11656HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11657{
11658 /*
11659 * This can only happen if we support dual-monitor treatment of SMI, which can be activated
11660 * by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
11661 * this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
11662 * an SMI. If we get here, something funny is going on.
11663 *
11664 * See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11665 * See Intel spec. 25.3 "Other Causes of VM-Exits"
11666 */
11667 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11668 AssertMsgFailed(("Unexpected SMI VM-exit\n"));
11669 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11670}
11671
11672
11673/**
11674 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11675 */
11676HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11677{
11678 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11679 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11680 AssertMsgFailed(("Unexpected IO SMI VM-exit\n"));
11681 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11682}
11683
11684
11685/**
11686 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11687 */
11688HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11689{
11690 /*
11691 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
11692 * We don't make use of it as our guests don't have direct access to the host LAPIC.
11693 * See Intel spec. 25.3 "Other Causes of VM-exits".
11694 */
11695 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11696 AssertMsgFailed(("Unexpected SIPI VM-exit\n"));
11697 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11698}
11699
11700
11701/**
11702 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11703 * VM-exit.
11704 */
11705HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11706{
11707 /*
11708 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11709 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11710 *
11711 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11712 * See Intel spec. "23.8 Restrictions on VMX operation".
11713 */
11714 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11715 return VINF_SUCCESS;
11716}
11717
11718
11719/**
11720 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11721 * VM-exit.
11722 */
11723HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11724{
11725 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11726 return VINF_EM_RESET;
11727}
11728
11729
11730/**
11731 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11732 */
11733HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11734{
11735 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11736 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_HLT_EXIT);
11737
11738 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11739 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
11740 AssertRCReturn(rc, rc);
11741
11742 if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
11743 rc = VINF_SUCCESS;
11744 else
11745 rc = VINF_EM_HALT;
11746
11747 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11748 if (rc != VINF_SUCCESS)
11749 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11750 return rc;
11751}
11752
11753
11754/**
11755 * VM-exit handler for instructions that result in a \#UD exception delivered to
11756 * the guest.
11757 */
11758HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11759{
11760 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11761 hmR0VmxSetPendingXcptUD(pVCpu);
11762 return VINF_SUCCESS;
11763}
11764
11765
11766/**
11767 * VM-exit handler for expiry of the VMX preemption timer.
11768 */
11769HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11770{
11771 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11772
11773 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11774 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11775
11776 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11777 PVM pVM = pVCpu->CTX_SUFF(pVM);
11778 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11779 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11780 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11781}
11782
11783
11784/**
11785 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11786 */
11787HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11788{
11789 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11790
11791 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11792 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_CR4);
11793 AssertRCReturn(rc, rc);
11794
11795 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11796 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11797 : HM_CHANGED_RAISED_XCPT_MASK);
11798
11799 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11800 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
11801
11802 return rcStrict;
11803}
11804
11805
11806/**
11807 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11808 */
11809HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11810{
11811 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11812 /** @todo Use VM-exit instruction information. */
11813 return VERR_EM_INTERPRETER;
11814}
11815
11816
11817/**
11818 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11819 * Error VM-exit.
11820 */
11821HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11822{
11823 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11824 AssertRCReturn(rc, rc);
11825 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11826 if (RT_FAILURE(rc))
11827 return rc;
11828
11829 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
11830 NOREF(uInvalidReason);
11831
11832#ifdef VBOX_STRICT
11833 uint32_t fIntrState;
11834 RTHCUINTREG uHCReg;
11835 uint64_t u64Val;
11836 uint32_t u32Val;
11837
11838 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11839 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11840 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11841 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
11842 AssertRCReturn(rc, rc);
11843
11844 Log4(("uInvalidReason %u\n", uInvalidReason));
11845 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11846 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11847 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11848 Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
11849
11850 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11851 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11852 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11853 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11854 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11855 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11856 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11857 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11858 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11859 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11860 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11861 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11862
11863 hmR0DumpRegs(pVCpu);
11864#else
11865 NOREF(pVmxTransient);
11866#endif
11867
11868 return VERR_VMX_INVALID_GUEST_STATE;
11869}
11870
11871
11872/**
11873 * VM-exit handler for VM-entry failure due to an MSR-load
11874 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11875 */
11876HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11877{
11878 AssertMsgFailed(("Unexpected MSR-load exit\n"));
11879 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11880}
11881
11882
11883/**
11884 * VM-exit handler for VM-entry failure due to a machine-check event
11885 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11886 */
11887HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11888{
11889 AssertMsgFailed(("Unexpected machine-check event exit\n"));
11890 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11891}
11892
11893
11894/**
11895 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11896 * theory.
11897 */
11898HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11899{
11900 RT_NOREF2(pVCpu, pVmxTransient);
11901 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d\n", pVmxTransient->uExitReason));
11902 return VERR_VMX_UNDEFINED_EXIT_CODE;
11903}
11904
11905
11906/**
11907 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11908 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11909 * Conditional VM-exit.
11910 */
11911HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11912{
11913 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11914
11915 /* By default, we don't enable VMX_PROC_CTLS2_DESCRIPTOR_TABLE_EXIT. */
11916 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11917 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_DESC_TABLE_EXIT)
11918 return VERR_EM_INTERPRETER;
11919 AssertMsgFailed(("Unexpected XDTR access\n"));
11920 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11921}
11922
11923
11924/**
11925 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11926 */
11927HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11928{
11929 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11930
11931 /* By default, we don't enable VMX_PROC_CTLS2_RDRAND_EXIT. */
11932 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_RDRAND_EXIT)
11933 return VERR_EM_INTERPRETER;
11934 AssertMsgFailed(("Unexpected RDRAND exit\n"));
11935 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11936}
11937
11938
11939/**
11940 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11941 */
11942HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11943{
11944 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11945
11946 /** @todo Optimize this: We currently drag in in the whole MSR state
11947 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11948 * MSRs required. That would require changes to IEM and possibly CPUM too.
11949 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11950 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; NOREF(idMsr); /* Save it. */
11951 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11952 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
11953 AssertRCReturn(rc, rc);
11954
11955 Log4Func(("ecx=%#RX32\n", idMsr));
11956
11957#ifdef VBOX_STRICT
11958 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
11959 {
11960 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr)
11961 && idMsr != MSR_K6_EFER)
11962 {
11963 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
11964 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11965 }
11966 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11967 {
11968 VMXMSREXITREAD enmRead;
11969 VMXMSREXITWRITE enmWrite;
11970 int rc2 = hmR0VmxGetMsrPermission(pVCpu, idMsr, &enmRead, &enmWrite);
11971 AssertRCReturn(rc2, rc2);
11972 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11973 {
11974 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
11975 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11976 }
11977 }
11978 }
11979#endif
11980
11981 VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
11982 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11983 if (rcStrict == VINF_SUCCESS)
11984 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11985 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX);
11986 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11987 {
11988 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11989 rcStrict = VINF_SUCCESS;
11990 }
11991 else
11992 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11993
11994 return rcStrict;
11995}
11996
11997
11998/**
11999 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12000 */
12001HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12002{
12003 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12004
12005 /** @todo Optimize this: We currently drag in in the whole MSR state
12006 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
12007 * MSRs required. That would require changes to IEM and possibly CPUM too.
12008 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
12009 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; /* Save it. */
12010 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12011 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
12012 AssertRCReturn(rc, rc);
12013
12014 Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
12015
12016 VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
12017 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12018
12019 if (rcStrict == VINF_SUCCESS)
12020 {
12021 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12022
12023 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
12024 if ( idMsr == MSR_IA32_APICBASE
12025 || ( idMsr >= MSR_IA32_X2APIC_START
12026 && idMsr <= MSR_IA32_X2APIC_END))
12027 {
12028 /*
12029 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12030 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before IEM changes it.
12031 */
12032 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
12033 }
12034 else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12035 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12036 else if (idMsr == MSR_K6_EFER)
12037 {
12038 /*
12039 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12040 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12041 * the other bits as well, SCE and NXE. See @bugref{7368}.
12042 */
12043 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS
12044 | HM_CHANGED_VMX_EXIT_CTLS);
12045 }
12046
12047 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12048 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
12049 {
12050 switch (idMsr)
12051 {
12052 case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
12053 case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
12054 case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
12055 case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
12056 case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
12057 case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
12058 default:
12059 {
12060 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
12061 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12062 else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
12063 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
12064 break;
12065 }
12066 }
12067 }
12068#ifdef VBOX_STRICT
12069 else
12070 {
12071 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12072 switch (idMsr)
12073 {
12074 case MSR_IA32_SYSENTER_CS:
12075 case MSR_IA32_SYSENTER_EIP:
12076 case MSR_IA32_SYSENTER_ESP:
12077 case MSR_K8_FS_BASE:
12078 case MSR_K8_GS_BASE:
12079 {
12080 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
12081 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12082 }
12083
12084 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12085 default:
12086 {
12087 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
12088 {
12089 /* EFER writes are always intercepted, see hmR0VmxExportGuestMsrs(). */
12090 if (idMsr != MSR_K6_EFER)
12091 {
12092 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12093 idMsr));
12094 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12095 }
12096 }
12097
12098 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
12099 {
12100 VMXMSREXITREAD enmRead;
12101 VMXMSREXITWRITE enmWrite;
12102 int rc2 = hmR0VmxGetMsrPermission(pVCpu, idMsr, &enmRead, &enmWrite);
12103 AssertRCReturn(rc2, rc2);
12104 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12105 {
12106 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
12107 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12108 }
12109 }
12110 break;
12111 }
12112 }
12113 }
12114#endif /* VBOX_STRICT */
12115 }
12116 else if (rcStrict == VINF_IEM_RAISED_XCPT)
12117 {
12118 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
12119 rcStrict = VINF_SUCCESS;
12120 }
12121 else
12122 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12123
12124 return rcStrict;
12125}
12126
12127
12128/**
12129 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12130 */
12131HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12132{
12133 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12134 /** @todo The guest has likely hit a contended spinlock. We might want to
12135 * poke a schedule different guest VCPU. */
12136 return VINF_EM_RAW_INTERRUPT;
12137}
12138
12139
12140/**
12141 * VM-exit handler for when the TPR value is lowered below the specified
12142 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12143 */
12144HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12145{
12146 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12147 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
12148
12149 /*
12150 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12151 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12152 */
12153 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12154 return VINF_SUCCESS;
12155}
12156
12157
12158/**
12159 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12160 * VM-exit.
12161 *
12162 * @retval VINF_SUCCESS when guest execution can continue.
12163 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12164 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12165 * interpreter.
12166 */
12167HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12168{
12169 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12170 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12171
12172 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12173 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12174 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12175 AssertRCReturn(rc, rc);
12176
12177 VBOXSTRICTRC rcStrict;
12178 PVM pVM = pVCpu->CTX_SUFF(pVM);
12179 RTGCUINTPTR const uExitQual = pVmxTransient->uExitQual;
12180 uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
12181 switch (uAccessType)
12182 {
12183 case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
12184 {
12185 uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
12186 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
12187 VMX_EXIT_QUAL_CRX_GENREG(uExitQual));
12188 AssertMsg( rcStrict == VINF_SUCCESS
12189 || rcStrict == VINF_IEM_RAISED_XCPT
12190 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12191
12192 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
12193 {
12194 case 0:
12195 {
12196 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12197 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12198 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
12199 Log4Func(("CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
12200
12201 /*
12202 * This is a kludge for handling switches back to real mode when we try to use
12203 * V86 mode to run real mode code directly. Problem is that V86 mode cannot
12204 * deal with special selector values, so we have to return to ring-3 and run
12205 * there till the selector values are V86 mode compatible.
12206 *
12207 * Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
12208 * latter is an alias for VINF_IEM_RAISED_XCPT which is converted to VINF_SUCCESs
12209 * at the end of this function.
12210 */
12211 if ( rc == VINF_SUCCESS
12212 && !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
12213 && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
12214 && (uOldCr0 & X86_CR0_PE)
12215 && !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
12216 {
12217 /** @todo check selectors rather than returning all the time. */
12218 Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
12219 rcStrict = VINF_EM_RESCHEDULE_REM;
12220 }
12221 break;
12222 }
12223
12224 case 2:
12225 {
12226 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
12227 /* Nothing to do here, CR2 it's not part of the VMCS. */
12228 break;
12229 }
12230
12231 case 3:
12232 {
12233 Assert( !pVM->hm.s.fNestedPaging
12234 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
12235 || pVCpu->hm.s.fUsingDebugLoop);
12236 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
12237 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12238 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR3);
12239 Log4Func(("CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
12240 break;
12241 }
12242
12243 case 4:
12244 {
12245 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
12246 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12247 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR4);
12248 Log4Func(("CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
12249 pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12250 break;
12251 }
12252
12253 case 8:
12254 {
12255 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
12256 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
12257 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12258 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_APIC_TPR);
12259 break;
12260 }
12261 default:
12262 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual)));
12263 break;
12264 }
12265 break;
12266 }
12267
12268 case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
12269 {
12270 Assert( !pVM->hm.s.fNestedPaging
12271 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
12272 || pVCpu->hm.s.fUsingDebugLoop
12273 || VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 3);
12274 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12275 Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 8
12276 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
12277
12278 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_GENREG(uExitQual),
12279 VMX_EXIT_QUAL_CRX_REGISTER(uExitQual));
12280 AssertMsg( rcStrict == VINF_SUCCESS
12281 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12282#ifdef VBOX_WITH_STATISTICS
12283 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
12284 {
12285 case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
12286 case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
12287 case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
12288 case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
12289 case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
12290 }
12291#endif
12292 Log4Func(("CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
12293 VBOXSTRICTRC_VAL(rcStrict)));
12294 if (VMX_EXIT_QUAL_CRX_GENREG(uExitQual) == X86_GREG_xSP)
12295 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_RSP);
12296 else
12297 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12298 break;
12299 }
12300
12301 case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12302 {
12303 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12304 AssertMsg( rcStrict == VINF_SUCCESS
12305 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12306
12307 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12308 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12309 Log4Func(("CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12310 break;
12311 }
12312
12313 case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12314 {
12315 /* Note! LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here. */
12316 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual));
12317 AssertMsg( rcStrict == VINF_SUCCESS
12318 || rcStrict == VINF_IEM_RAISED_XCPT
12319 , ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12320
12321 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12322 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12323 Log4Func(("LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12324 break;
12325 }
12326
12327 default:
12328 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12329 VERR_VMX_UNEXPECTED_EXCEPTION);
12330 }
12331
12332 Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS))
12333 == (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS));
12334 if (rcStrict == VINF_IEM_RAISED_XCPT)
12335 {
12336 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
12337 rcStrict = VINF_SUCCESS;
12338 }
12339
12340 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12341 NOREF(pVM);
12342 return rcStrict;
12343}
12344
12345
12346/**
12347 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12348 * VM-exit.
12349 */
12350HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12351{
12352 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12353 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12354
12355 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12356 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12357 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12358 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_EFER);
12359 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12360 AssertRCReturn(rc, rc);
12361
12362 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12363 uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
12364 uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQual);
12365 bool fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12366 bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
12367 bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
12368 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12369 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12370
12371 /*
12372 * Update exit history to see if this exit can be optimized.
12373 */
12374 VBOXSTRICTRC rcStrict;
12375 PCEMEXITREC pExitRec = NULL;
12376 if ( !fGstStepping
12377 && !fDbgStepping)
12378 pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12379 !fIOString
12380 ? !fIOWrite
12381 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12382 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12383 : !fIOWrite
12384 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12385 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12386 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12387 if (!pExitRec)
12388 {
12389 /* I/O operation lookup arrays. */
12390 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12391 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12392 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12393 uint32_t const cbInstr = pVmxTransient->cbInstr;
12394 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12395 PVM pVM = pVCpu->CTX_SUFF(pVM);
12396 if (fIOString)
12397 {
12398 /*
12399 * INS/OUTS - I/O String instruction.
12400 *
12401 * Use instruction-information if available, otherwise fall back on
12402 * interpreting the instruction.
12403 */
12404 Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12405 AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
12406 bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
12407 if (fInsOutsInfo)
12408 {
12409 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12410 AssertRCReturn(rc2, rc2);
12411 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12412 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12413 IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12414 bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
12415 if (fIOWrite)
12416 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12417 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12418 else
12419 {
12420 /*
12421 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12422 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12423 * See Intel Instruction spec. for "INS".
12424 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12425 */
12426 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12427 }
12428 }
12429 else
12430 rcStrict = IEMExecOne(pVCpu);
12431
12432 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12433 fUpdateRipAlready = true;
12434 }
12435 else
12436 {
12437 /*
12438 * IN/OUT - I/O instruction.
12439 */
12440 Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12441 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12442 Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
12443 if (fIOWrite)
12444 {
12445 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
12446 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12447 if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
12448 && !pCtx->eflags.Bits.u1TF)
12449 rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
12450 }
12451 else
12452 {
12453 uint32_t u32Result = 0;
12454 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12455 if (IOM_SUCCESS(rcStrict))
12456 {
12457 /* Save result of I/O IN instr. in AL/AX/EAX. */
12458 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12459 }
12460 if ( rcStrict == VINF_IOM_R3_IOPORT_READ
12461 && !pCtx->eflags.Bits.u1TF)
12462 rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
12463 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12464 }
12465 }
12466
12467 if (IOM_SUCCESS(rcStrict))
12468 {
12469 if (!fUpdateRipAlready)
12470 {
12471 hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
12472 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12473 }
12474
12475 /*
12476 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12477 * while booting Fedora 17 64-bit guest.
12478 *
12479 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12480 */
12481 if (fIOString)
12482 {
12483 /** @todo Single-step for INS/OUTS with REP prefix? */
12484 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12485 }
12486 else if ( !fDbgStepping
12487 && fGstStepping)
12488 {
12489 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12490 AssertRCReturn(rc, rc);
12491 }
12492
12493 /*
12494 * If any I/O breakpoints are armed, we need to check if one triggered
12495 * and take appropriate action.
12496 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12497 */
12498 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12499 AssertRCReturn(rc, rc);
12500
12501 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12502 * execution engines about whether hyper BPs and such are pending. */
12503 uint32_t const uDr7 = pCtx->dr[7];
12504 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12505 && X86_DR7_ANY_RW_IO(uDr7)
12506 && (pCtx->cr4 & X86_CR4_DE))
12507 || DBGFBpIsHwIoArmed(pVM)))
12508 {
12509 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12510
12511 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12512 VMMRZCallRing3Disable(pVCpu);
12513 HM_DISABLE_PREEMPT(pVCpu);
12514
12515 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12516
12517 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
12518 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12519 {
12520 /* Raise #DB. */
12521 if (fIsGuestDbgActive)
12522 ASMSetDR6(pCtx->dr[6]);
12523 if (pCtx->dr[7] != uDr7)
12524 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_DR7;
12525
12526 hmR0VmxSetPendingXcptDB(pVCpu);
12527 }
12528 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12529 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12530 else if ( rcStrict2 != VINF_SUCCESS
12531 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12532 rcStrict = rcStrict2;
12533 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12534
12535 HM_RESTORE_PREEMPT();
12536 VMMRZCallRing3Enable(pVCpu);
12537 }
12538 }
12539
12540#ifdef VBOX_STRICT
12541 if ( rcStrict == VINF_IOM_R3_IOPORT_READ
12542 || rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
12543 Assert(!fIOWrite);
12544 else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
12545 || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
12546 || rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
12547 Assert(fIOWrite);
12548 else
12549 {
12550# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12551 * statuses, that the VMM device and some others may return. See
12552 * IOM_SUCCESS() for guidance. */
12553 AssertMsg( RT_FAILURE(rcStrict)
12554 || rcStrict == VINF_SUCCESS
12555 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12556 || rcStrict == VINF_EM_DBG_BREAKPOINT
12557 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12558 || rcStrict == VINF_EM_RAW_TO_R3
12559 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12560# endif
12561 }
12562#endif
12563 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12564 }
12565 else
12566 {
12567 /*
12568 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12569 */
12570 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12571 AssertRCReturn(rc2, rc2);
12572 STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
12573 : fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
12574 Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12575 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12576 VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
12577 fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
12578
12579 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12580 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12581
12582 Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12583 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12584 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12585 }
12586 return rcStrict;
12587}
12588
12589
12590/**
12591 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12592 * VM-exit.
12593 */
12594HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12595{
12596 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12597
12598 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12599 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12600 AssertRCReturn(rc, rc);
12601 if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12602 {
12603 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12604 AssertRCReturn(rc, rc);
12605 if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
12606 {
12607 uint32_t uErrCode;
12608 RTGCUINTPTR GCPtrFaultAddress;
12609 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12610 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12611 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
12612 if (fErrorCodeValid)
12613 {
12614 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12615 AssertRCReturn(rc, rc);
12616 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12617 }
12618 else
12619 uErrCode = 0;
12620
12621 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12622 && uVector == X86_XCPT_PF)
12623 GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
12624 else
12625 GCPtrFaultAddress = 0;
12626
12627 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12628 0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
12629
12630 Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
12631 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12632 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12633 }
12634 }
12635
12636 /* Fall back to the interpreter to emulate the task-switch. */
12637 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12638 return VERR_EM_INTERPRETER;
12639}
12640
12641
12642/**
12643 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12644 */
12645HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12646{
12647 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12648 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_MONITOR_TRAP_FLAG);
12649 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
12650 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12651 AssertRCReturn(rc, rc);
12652 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12653 return VINF_EM_DBG_STEPPED;
12654}
12655
12656
12657/**
12658 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12659 */
12660HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12661{
12662 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12663
12664 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12665
12666 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12667 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12668 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12669 {
12670 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12671 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12672 {
12673 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12674 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12675 }
12676 }
12677 else
12678 {
12679 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12680 rcStrict1 = VINF_SUCCESS;
12681 return rcStrict1;
12682 }
12683
12684 /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12685 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12686 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12687 AssertRCReturn(rc, rc);
12688
12689 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12690 uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
12691 VBOXSTRICTRC rcStrict2;
12692 switch (uAccessType)
12693 {
12694 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12695 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12696 {
12697 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
12698 || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
12699 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12700
12701 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12702 GCPhys &= PAGE_BASE_GC_MASK;
12703 GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
12704 PVM pVM = pVCpu->CTX_SUFF(pVM);
12705 Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12706 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
12707
12708 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12709 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12710 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12711 CPUMCTX2CORE(pCtx), GCPhys);
12712 Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12713 if ( rcStrict2 == VINF_SUCCESS
12714 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12715 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12716 {
12717 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12718 | HM_CHANGED_GUEST_APIC_TPR);
12719 rcStrict2 = VINF_SUCCESS;
12720 }
12721 break;
12722 }
12723
12724 default:
12725 Log4Func(("uAccessType=%#x\n", uAccessType));
12726 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12727 break;
12728 }
12729
12730 if (rcStrict2 != VINF_SUCCESS)
12731 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12732 return rcStrict2;
12733}
12734
12735
12736/**
12737 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12738 * VM-exit.
12739 */
12740HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12741{
12742 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12743
12744 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12745 if (pVmxTransient->fWasGuestDebugStateActive)
12746 {
12747 AssertMsgFailed(("Unexpected MOV DRx exit\n"));
12748 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12749 }
12750
12751 if ( !pVCpu->hm.s.fSingleInstruction
12752 && !pVmxTransient->fWasHyperDebugStateActive)
12753 {
12754 Assert(!DBGFIsStepping(pVCpu));
12755 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12756
12757 /* Don't intercept MOV DRx any more. */
12758 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
12759 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12760 AssertRCReturn(rc, rc);
12761
12762 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12763 VMMRZCallRing3Disable(pVCpu);
12764 HM_DISABLE_PREEMPT(pVCpu);
12765
12766 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12767 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12768 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12769
12770 HM_RESTORE_PREEMPT();
12771 VMMRZCallRing3Enable(pVCpu);
12772
12773#ifdef VBOX_WITH_STATISTICS
12774 rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12775 AssertRCReturn(rc, rc);
12776 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12777 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12778 else
12779 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12780#endif
12781 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12782 return VINF_SUCCESS;
12783 }
12784
12785 /*
12786 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12787 * Update the segment registers and DR7 from the CPU.
12788 */
12789 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12790 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12791 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_DR7);
12792 AssertRCReturn(rc, rc);
12793 Log4Func(("CS:RIP=%04x:%08RX64\n", pCtx->cs.Sel, pCtx->rip));
12794
12795 PVM pVM = pVCpu->CTX_SUFF(pVM);
12796 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12797 {
12798 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12799 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
12800 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
12801 if (RT_SUCCESS(rc))
12802 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12803 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12804 }
12805 else
12806 {
12807 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12808 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
12809 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
12810 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12811 }
12812
12813 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12814 if (RT_SUCCESS(rc))
12815 {
12816 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12817 AssertRCReturn(rc2, rc2);
12818 return VINF_SUCCESS;
12819 }
12820 return rc;
12821}
12822
12823
12824/**
12825 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12826 * Conditional VM-exit.
12827 */
12828HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12829{
12830 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12831 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12832
12833 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12834 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12835 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12836 {
12837 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12838 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12839 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12840 {
12841 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12842 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12843 }
12844 }
12845 else
12846 {
12847 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12848 rcStrict1 = VINF_SUCCESS;
12849 return rcStrict1;
12850 }
12851
12852 /*
12853 * Get sufficent state and update the exit history entry.
12854 */
12855 RTGCPHYS GCPhys;
12856 int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
12857 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12858 AssertRCReturn(rc, rc);
12859
12860 VBOXSTRICTRC rcStrict;
12861 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12862 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
12863 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12864 if (!pExitRec)
12865 {
12866 /*
12867 * If we succeed, resume guest execution.
12868 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12869 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12870 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12871 * weird case. See @bugref{6043}.
12872 */
12873 PVM pVM = pVCpu->CTX_SUFF(pVM);
12874 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12875 rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
12876 Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12877 if ( rcStrict == VINF_SUCCESS
12878 || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12879 || rcStrict == VERR_PAGE_NOT_PRESENT)
12880 {
12881 /* Successfully handled MMIO operation. */
12882 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12883 | HM_CHANGED_GUEST_APIC_TPR);
12884 rcStrict = VINF_SUCCESS;
12885 }
12886 }
12887 else
12888 {
12889 /*
12890 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12891 */
12892 int rc2 = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12893 AssertRCReturn(rc2, rc2);
12894
12895 Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12896 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12897
12898 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12899 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12900
12901 Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12902 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12903 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12904 }
12905 return VBOXSTRICTRC_TODO(rcStrict);
12906}
12907
12908
12909/**
12910 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12911 * VM-exit.
12912 */
12913HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12914{
12915 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12916 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12917
12918 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12919 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12920 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12921 {
12922 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12923 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12924 Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12925 }
12926 else
12927 {
12928 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12929 rcStrict1 = VINF_SUCCESS;
12930 return rcStrict1;
12931 }
12932
12933 RTGCPHYS GCPhys;
12934 int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
12935 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12936 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12937 AssertRCReturn(rc, rc);
12938
12939 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12940 AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
12941
12942 RTGCUINT uErrorCode = 0;
12943 if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
12944 uErrorCode |= X86_TRAP_PF_ID;
12945 if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
12946 uErrorCode |= X86_TRAP_PF_RW;
12947 if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
12948 uErrorCode |= X86_TRAP_PF_P;
12949
12950 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12951
12952
12953 /* Handle the pagefault trap for the nested shadow table. */
12954 PVM pVM = pVCpu->CTX_SUFF(pVM);
12955 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12956
12957 Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
12958 pCtx->cs.Sel, pCtx->rip));
12959
12960 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
12961 TRPMResetTrap(pVCpu);
12962
12963 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12964 if ( rcStrict2 == VINF_SUCCESS
12965 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12966 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12967 {
12968 /* Successfully synced our nested page tables. */
12969 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12970 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
12971 return VINF_SUCCESS;
12972 }
12973
12974 Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12975 return rcStrict2;
12976}
12977
12978/** @} */
12979
12980/** @name VM-exit exception handlers.
12981 * @{
12982 */
12983/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12984/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12985/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12986
12987/**
12988 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12989 */
12990static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12991{
12992 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12993 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12994
12995 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
12996 AssertRCReturn(rc, rc);
12997
12998 if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
12999 {
13000 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13001 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13002
13003 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13004 * provides VM-exit instruction length. If this causes problem later,
13005 * disassemble the instruction like it's done on AMD-V. */
13006 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13007 AssertRCReturn(rc2, rc2);
13008 return rc;
13009 }
13010
13011 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13012 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13013 return rc;
13014}
13015
13016
13017/**
13018 * VM-exit exception handler for \#BP (Breakpoint exception).
13019 */
13020static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13021{
13022 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13023 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13024
13025 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13026 AssertRCReturn(rc, rc);
13027
13028 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13029 rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13030 if (rc == VINF_EM_RAW_GUEST_TRAP)
13031 {
13032 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13033 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13034 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13035 AssertRCReturn(rc, rc);
13036
13037 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13038 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13039 }
13040
13041 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
13042 return rc;
13043}
13044
13045
13046/**
13047 * VM-exit exception handler for \#AC (alignment check exception).
13048 */
13049static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13050{
13051 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13052
13053 /*
13054 * Re-inject it. We'll detect any nesting before getting here.
13055 */
13056 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13057 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13058 AssertRCReturn(rc, rc);
13059 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13060
13061 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13062 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13063 return VINF_SUCCESS;
13064}
13065
13066
13067/**
13068 * VM-exit exception handler for \#DB (Debug exception).
13069 */
13070static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13071{
13072 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13073 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13074
13075 /*
13076 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13077 * for processing.
13078 */
13079 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13080
13081 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13082 uint64_t uDR6 = X86_DR6_INIT_VAL;
13083 uDR6 |= (pVmxTransient->uExitQual & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
13084
13085 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13086 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13087 Log6Func(("rc=%Rrc\n", rc));
13088 if (rc == VINF_EM_RAW_GUEST_TRAP)
13089 {
13090 /*
13091 * The exception was for the guest. Update DR6, DR7.GD and
13092 * IA32_DEBUGCTL.LBR before forwarding it.
13093 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13094 */
13095 VMMRZCallRing3Disable(pVCpu);
13096 HM_DISABLE_PREEMPT(pVCpu);
13097
13098 pCtx->dr[6] &= ~X86_DR6_B_MASK;
13099 pCtx->dr[6] |= uDR6;
13100 if (CPUMIsGuestDebugStateActive(pVCpu))
13101 ASMSetDR6(pCtx->dr[6]);
13102
13103 HM_RESTORE_PREEMPT();
13104 VMMRZCallRing3Enable(pVCpu);
13105
13106 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
13107 AssertRCReturn(rc, rc);
13108
13109 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13110 pCtx->dr[7] &= ~X86_DR7_GD;
13111
13112 /* Paranoia. */
13113 pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13114 pCtx->dr[7] |= X86_DR7_RA1_MASK;
13115
13116 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
13117 AssertRCReturn(rc, rc);
13118
13119 /*
13120 * Raise #DB in the guest.
13121 *
13122 * It is important to reflect exactly what the VM-exit gave us (preserving the
13123 * interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
13124 * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
13125 * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
13126 *
13127 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
13128 * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
13129 */
13130 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13131 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13132 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13133 AssertRCReturn(rc, rc);
13134 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13135 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13136 return VINF_SUCCESS;
13137 }
13138
13139 /*
13140 * Not a guest trap, must be a hypervisor related debug event then.
13141 * Update DR6 in case someone is interested in it.
13142 */
13143 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13144 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13145 CPUMSetHyperDR6(pVCpu, uDR6);
13146
13147 return rc;
13148}
13149
13150/**
13151 * VM-exit exception handler for \#GP (General-protection exception).
13152 *
13153 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13154 */
13155static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13156{
13157 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13158 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13159
13160 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13161 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13162 { /* likely */ }
13163 else
13164 {
13165#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13166 Assert(pVCpu->hm.s.fUsingDebugLoop);
13167#endif
13168 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13169 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13170 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13171 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13172 rc |= hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13173 AssertRCReturn(rc, rc);
13174 Log4Func(("Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pCtx->cs.Sel, pCtx->rip,
13175 pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
13176 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13177 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13178 return rc;
13179 }
13180
13181 Assert(CPUMIsGuestInRealModeEx(pCtx));
13182 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13183
13184 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13185 AssertRCReturn(rc, rc);
13186
13187 VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
13188 if (rcStrict == VINF_SUCCESS)
13189 {
13190 if (!CPUMIsGuestInRealModeEx(pCtx))
13191 {
13192 /*
13193 * The guest is no longer in real-mode, check if we can continue executing the
13194 * guest using hardware-assisted VMX. Otherwise, fall back to emulation.
13195 */
13196 if (HMVmxCanExecuteGuest(pVCpu, pCtx))
13197 {
13198 Log4Func(("Mode changed but guest still suitable for executing using VT-x\n"));
13199 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
13200 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13201 }
13202 else
13203 {
13204 Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
13205 rcStrict = VINF_EM_RESCHEDULE;
13206 }
13207 }
13208 else
13209 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13210 }
13211 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13212 {
13213 rcStrict = VINF_SUCCESS;
13214 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13215 }
13216 return VBOXSTRICTRC_VAL(rcStrict);
13217}
13218
13219
13220/**
13221 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13222 * the exception reported in the VMX transient structure back into the VM.
13223 *
13224 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13225 * up-to-date.
13226 */
13227static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13228{
13229 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13230#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13231 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13232 ("uVector=%#x u32XcptBitmap=%#X32\n",
13233 VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13234#endif
13235
13236 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13237 hmR0VmxCheckExitDueToEventDelivery(). */
13238 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13239 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13240 AssertRCReturn(rc, rc);
13241 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13242
13243#ifdef DEBUG_ramshankar
13244 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
13245 uint8_t uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13246 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13247#endif
13248
13249 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13250 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13251 return VINF_SUCCESS;
13252}
13253
13254
13255/**
13256 * VM-exit exception handler for \#PF (Page-fault exception).
13257 */
13258static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13259{
13260 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13261 PVM pVM = pVCpu->CTX_SUFF(pVM);
13262 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13263 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13264 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13265 AssertRCReturn(rc, rc);
13266
13267 if (!pVM->hm.s.fNestedPaging)
13268 { /* likely */ }
13269 else
13270 {
13271#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13272 Assert(pVCpu->hm.s.fUsingDebugLoop);
13273#endif
13274 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13275 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13276 {
13277 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13278 pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
13279 }
13280 else
13281 {
13282 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13283 hmR0VmxSetPendingXcptDF(pVCpu);
13284 Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13285 }
13286 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13287 return rc;
13288 }
13289
13290 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13291 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13292 if (pVmxTransient->fVectoringPF)
13293 {
13294 Assert(pVCpu->hm.s.Event.fPending);
13295 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13296 }
13297
13298 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13299 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13300 AssertRCReturn(rc, rc);
13301
13302 Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
13303 pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
13304
13305 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13306 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
13307
13308 Log4Func(("#PF: rc=%Rrc\n", rc));
13309 if (rc == VINF_SUCCESS)
13310 {
13311 /*
13312 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13313 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13314 */
13315 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13316 TRPMResetTrap(pVCpu);
13317 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13318 return rc;
13319 }
13320
13321 if (rc == VINF_EM_RAW_GUEST_TRAP)
13322 {
13323 if (!pVmxTransient->fVectoringDoublePF)
13324 {
13325 /* It's a guest page fault and needs to be reflected to the guest. */
13326 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13327 TRPMResetTrap(pVCpu);
13328 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13329 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13330 uGstErrorCode, pVmxTransient->uExitQual);
13331 }
13332 else
13333 {
13334 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13335 TRPMResetTrap(pVCpu);
13336 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13337 hmR0VmxSetPendingXcptDF(pVCpu);
13338 Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13339 }
13340
13341 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13342 return VINF_SUCCESS;
13343 }
13344
13345 TRPMResetTrap(pVCpu);
13346 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13347 return rc;
13348}
13349
13350/** @} */
13351
13352#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13353
13354/** @name Nested-guest VM-exit handlers.
13355 * @{
13356 */
13357/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13358/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= Nested-guest VM-exit handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13359/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13360
13361/**
13362 * VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
13363 */
13364HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13365{
13366 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13367
13368 /** @todo NSTVMX: Vmclear. */
13369 hmR0VmxSetPendingXcptUD(pVCpu);
13370 return VINF_SUCCESS;
13371}
13372
13373
13374/**
13375 * VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
13376 */
13377HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13378{
13379 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13380
13381 /** @todo NSTVMX: Vmlaunch. */
13382 hmR0VmxSetPendingXcptUD(pVCpu);
13383 return VINF_SUCCESS;
13384}
13385
13386
13387/**
13388 * VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
13389 */
13390HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13391{
13392 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13393
13394 /** @todo NSTVMX: Vmptrld. */
13395 hmR0VmxSetPendingXcptUD(pVCpu);
13396 return VINF_SUCCESS;
13397}
13398
13399
13400/**
13401 * VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
13402 */
13403HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13404{
13405 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13406
13407 /** @todo NSTVMX: Vmptrst. */
13408 hmR0VmxSetPendingXcptUD(pVCpu);
13409 return VINF_SUCCESS;
13410}
13411
13412
13413/**
13414 * VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Unconditional VM-exit.
13415 */
13416HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13417{
13418 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13419
13420 /** @todo NSTVMX: Vmread. */
13421 hmR0VmxSetPendingXcptUD(pVCpu);
13422 return VINF_SUCCESS;
13423}
13424
13425
13426/**
13427 * VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
13428 */
13429HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13430{
13431 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13432
13433 /** @todo NSTVMX: Vmresume. */
13434 hmR0VmxSetPendingXcptUD(pVCpu);
13435 return VINF_SUCCESS;
13436}
13437
13438
13439/**
13440 * VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Unconditional VM-exit.
13441 */
13442HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13443{
13444 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13445
13446 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13447 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13448 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13449 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13450 AssertRCReturn(rc, rc);
13451
13452 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13453
13454 VMXVEXITINFO ExitInfo;
13455 RT_ZERO(ExitInfo);
13456 ExitInfo.uReason = pVmxTransient->uExitReason;
13457 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13458 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13459 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13460 if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
13461 {
13462 RTGCPTR GCPtrVal;
13463 VBOXSTRICTRC rcStrict = hmR0VmxDecodeMemOperand(pVCpu, &ExitInfo.InstrInfo, ExitInfo.u64Qual, false /* fIsDstOperand */,
13464 &GCPtrVal);
13465 if (rcStrict == VINF_SUCCESS)
13466 { /* likely */ }
13467 else if (rcStrict == VINF_HM_PENDING_XCPT)
13468 {
13469 Assert(pVCpu->hm.s.Event.fPending);
13470 Log4Func(("Memory operand decoding failed, raising xcpt %#x\n",
13471 VMX_ENTRY_INT_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo)));
13472 return VINF_SUCCESS;
13473 }
13474 else
13475 {
13476 Log4Func(("hmR0VmxCheckExitDueToVmxInstr failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13477 return rcStrict;
13478 }
13479 ExitInfo.GCPtrEffAddr = GCPtrVal;
13480 }
13481
13482 VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
13483 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13484 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13485 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13486 {
13487 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13488 rcStrict = VINF_SUCCESS;
13489 }
13490 return rcStrict;
13491}
13492
13493
13494/**
13495 * VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
13496 */
13497HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13498{
13499 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13500
13501 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13502 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13503 AssertRCReturn(rc, rc);
13504
13505 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13506
13507 VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
13508 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13509 {
13510 /* VMXOFF on success changes the internal hwvirt state but not anything that's visible to the guest. */
13511 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
13512 }
13513 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13514 {
13515 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13516 rcStrict = VINF_SUCCESS;
13517 }
13518 return rcStrict;
13519}
13520
13521
13522/**
13523 * VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
13524 */
13525HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13526{
13527 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13528
13529 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13530 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13531 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13532 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13533 AssertRCReturn(rc, rc);
13534
13535 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13536
13537 VMXVEXITINFO ExitInfo;
13538 RT_ZERO(ExitInfo);
13539 ExitInfo.uReason = pVmxTransient->uExitReason;
13540 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13541 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13542 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13543
13544 RTGCPTR GCPtrVmxon;
13545 VBOXSTRICTRC rcStrict = hmR0VmxDecodeMemOperand(pVCpu, &ExitInfo.InstrInfo, ExitInfo.u64Qual, false /* fIsDstOperand */,
13546 &GCPtrVmxon);
13547 if (rcStrict == VINF_SUCCESS)
13548 { /* likely */ }
13549 else if (rcStrict == VINF_HM_PENDING_XCPT)
13550 {
13551 Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", VMX_ENTRY_INT_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo)));
13552 return VINF_SUCCESS;
13553 }
13554 else
13555 {
13556 Log4Func(("hmR0VmxCheckExitDueToVmxInstr failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13557 return rcStrict;
13558 }
13559 ExitInfo.GCPtrEffAddr = GCPtrVmxon;
13560
13561 rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
13562 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13563 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13564 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13565 {
13566 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13567 rcStrict = VINF_SUCCESS;
13568 }
13569 return rcStrict;
13570}
13571
13572/** @} */
13573#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13574
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette