VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 73311

Last change on this file since 73311 was 73311, checked in by vboxsync, 6 years ago

VMM/HMVMXR0: Nested VMX: bugref:9180 Cleanups, add new VMCS bits.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 563.6 KB
Line 
1/* $Id: HMVMXR0.cpp 73311 2018-07-23 07:29:34Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/x86.h>
25#include <iprt/asm-amd64-x86.h>
26#include <iprt/thread.h>
27
28#include <VBox/vmm/pdmapi.h>
29#include <VBox/vmm/dbgf.h>
30#include <VBox/vmm/iem.h>
31#include <VBox/vmm/iom.h>
32#include <VBox/vmm/selm.h>
33#include <VBox/vmm/tm.h>
34#include <VBox/vmm/gim.h>
35#include <VBox/vmm/apic.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#include "HMInternal.h"
40#include <VBox/vmm/vm.h>
41#include "HMVMXR0.h"
42#include "dtrace/VBoxVMM.h"
43
44#ifdef DEBUG_ramshankar
45# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
47# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_CHECK_GUEST_STATE
49# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
50# define HMVMX_ALWAYS_TRAP_PF
51# define HMVMX_ALWAYS_FLUSH_TLB
52# define HMVMX_ALWAYS_SWAP_EFER
53#endif
54
55
56/*********************************************************************************************************************************
57* Defined Constants And Macros *
58*********************************************************************************************************************************/
59/** Use the function table. */
60#define HMVMX_USE_FUNCTION_TABLE
61
62/** Determine which tagged-TLB flush handler to use. */
63#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68/** @name HMVMX_READ_XXX
69 * Flags to skip redundant reads of some common VMCS fields that are not part of
70 * the guest-CPU or VCPU state but are needed while handling VM-exits.
71 */
72#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
73#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
74#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
75#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
76#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
77#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
78#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
79/** @} */
80
81/**
82 * States of the VMCS.
83 *
84 * This does not reflect all possible VMCS states but currently only those
85 * needed for maintaining the VMCS consistently even when thread-context hooks
86 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
87 */
88#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
89#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
90#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
91
92/**
93 * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
94 * guest using hardware-assisted VMX.
95 *
96 * This excludes state like GPRs (other than RSP) which are always are
97 * swapped and restored across the world-switch and also registers like EFER,
98 * MSR which cannot be modified by the guest without causing a VM-exit.
99 */
100#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
101 | CPUMCTX_EXTRN_RFLAGS \
102 | CPUMCTX_EXTRN_RSP \
103 | CPUMCTX_EXTRN_SREG_MASK \
104 | CPUMCTX_EXTRN_TABLE_MASK \
105 | CPUMCTX_EXTRN_KERNEL_GS_BASE \
106 | CPUMCTX_EXTRN_SYSCALL_MSRS \
107 | CPUMCTX_EXTRN_SYSENTER_MSRS \
108 | CPUMCTX_EXTRN_TSC_AUX \
109 | CPUMCTX_EXTRN_OTHER_MSRS \
110 | CPUMCTX_EXTRN_CR0 \
111 | CPUMCTX_EXTRN_CR3 \
112 | CPUMCTX_EXTRN_CR4 \
113 | CPUMCTX_EXTRN_DR7 \
114 | CPUMCTX_EXTRN_HM_VMX_MASK)
115
116/**
117 * Exception bitmap mask for real-mode guests (real-on-v86).
118 *
119 * We need to intercept all exceptions manually except:
120 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
121 * due to bugs in Intel CPUs.
122 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
123 * support.
124 */
125#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
126 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
127 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
128 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
129 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
130 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
131 | RT_BIT(X86_XCPT_XF))
132
133/** Maximum VM-instruction error number. */
134#define HMVMX_INSTR_ERROR_MAX 28
135
136/** Profiling macro. */
137#ifdef HM_PROFILE_EXIT_DISPATCH
138# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
139# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
140#else
141# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
142# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
143#endif
144
145/** Assert that preemption is disabled or covered by thread-context hooks. */
146#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
147 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
148
149/** Assert that we haven't migrated CPUs when thread-context hooks are not
150 * used. */
151#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
152 || (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
153 ("Illegal migration! Entered on CPU %u Current %u\n", \
154 (a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
155
156/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
157 * context. */
158#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
159 ("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
160 (a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
161
162/** Helper macro for VM-exit handlers called unexpectedly. */
163#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
164 do { \
165 (a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
166 return VERR_VMX_UNEXPECTED_EXIT; \
167 } while (0)
168
169/** Macro for importing segment registers to the VMCS from the guest-CPU context. */
170#ifdef VMX_USE_CACHED_VMCS_ACCESSES
171# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
172 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
173 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
174#else
175# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
176 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
177 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
178#endif
179
180/** Macro for exporting segment registers to the VMCS from the guest-CPU context. */
181# define HMVMX_EXPORT_SREG(Sel, a_pCtxSelReg) \
182 hmR0VmxExportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
183 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
184
185
186/*********************************************************************************************************************************
187* Structures and Typedefs *
188*********************************************************************************************************************************/
189/**
190 * VMX transient state.
191 *
192 * A state structure for holding miscellaneous information across
193 * VMX non-root operation and restored after the transition.
194 */
195typedef struct VMXTRANSIENT
196{
197 /** The host's rflags/eflags. */
198 RTCCUINTREG fEFlags;
199#if HC_ARCH_BITS == 32
200 uint32_t u32Alignment0;
201#endif
202 /** The guest's TPR value used for TPR shadowing. */
203 uint8_t u8GuestTpr;
204 /** Alignment. */
205 uint8_t abAlignment0[7];
206
207 /** The basic VM-exit reason. */
208 uint16_t uExitReason;
209 /** Alignment. */
210 uint16_t u16Alignment0;
211 /** The VM-exit interruption error code. */
212 uint32_t uExitIntErrorCode;
213 /** The VM-exit exit code qualification. */
214 uint64_t uExitQualification;
215
216 /** The VM-exit interruption-information field. */
217 uint32_t uExitIntInfo;
218 /** The VM-exit instruction-length field. */
219 uint32_t cbInstr;
220 /** The VM-exit instruction-information field. */
221 union
222 {
223 /** Plain unsigned int representation. */
224 uint32_t u;
225 /** INS and OUTS information. */
226 struct
227 {
228 uint32_t u7Reserved0 : 7;
229 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
230 uint32_t u3AddrSize : 3;
231 uint32_t u5Reserved1 : 5;
232 /** The segment register (X86_SREG_XXX). */
233 uint32_t iSegReg : 3;
234 uint32_t uReserved2 : 14;
235 } StrIo;
236 /** INVEPT, INVVPID, INVPCID information. */
237 struct
238 {
239 /** Scaling; 0=no scaling, 1=scale-by-2, 2=scale-by-4, 3=scale-by-8. */
240 uint32_t u2Scaling : 2;
241 uint32_t u5Reserved0 : 5;
242 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
243 uint32_t u3AddrSize : 3;
244 uint32_t u1Reserved0 : 1;
245 uint32_t u4Reserved0 : 4;
246 /** The segment register (X86_SREG_XXX). */
247 uint32_t iSegReg : 3;
248 /** The index register (X86_GREG_XXX). */
249 uint32_t iIdxReg : 4;
250 /** Set if index register is invalid. */
251 uint32_t fIdxRegValid : 1;
252 /** The base register (X86_GREG_XXX). */
253 uint32_t iBaseReg : 4;
254 /** Set if base register is invalid. */
255 uint32_t fBaseRegValid : 1;
256 /** Register 2 (X86_GREG_XXX). */
257 uint32_t iReg2 : 4;
258 } Inv;
259 } ExitInstrInfo;
260 /** Whether the VM-entry failed or not. */
261 bool fVMEntryFailed;
262 /** Alignment. */
263 uint8_t abAlignment1[3];
264
265 /** The VM-entry interruption-information field. */
266 uint32_t uEntryIntInfo;
267 /** The VM-entry exception error code field. */
268 uint32_t uEntryXcptErrorCode;
269 /** The VM-entry instruction length field. */
270 uint32_t cbEntryInstr;
271
272 /** IDT-vectoring information field. */
273 uint32_t uIdtVectoringInfo;
274 /** IDT-vectoring error code. */
275 uint32_t uIdtVectoringErrorCode;
276
277 /** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
278 uint32_t fVmcsFieldsRead;
279
280 /** Whether the guest debug state was active at the time of VM-exit. */
281 bool fWasGuestDebugStateActive;
282 /** Whether the hyper debug state was active at the time of VM-exit. */
283 bool fWasHyperDebugStateActive;
284 /** Whether TSC-offsetting should be setup before VM-entry. */
285 bool fUpdateTscOffsettingAndPreemptTimer;
286 /** Whether the VM-exit was caused by a page-fault during delivery of a
287 * contributory exception or a page-fault. */
288 bool fVectoringDoublePF;
289 /** Whether the VM-exit was caused by a page-fault during delivery of an
290 * external interrupt or NMI. */
291 bool fVectoringPF;
292} VMXTRANSIENT;
293AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
294AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
295AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
296AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
297AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
298/** Pointer to VMX transient state. */
299typedef VMXTRANSIENT *PVMXTRANSIENT;
300
301
302/**
303 * MSR-bitmap read permissions.
304 */
305typedef enum VMXMSREXITREAD
306{
307 /** Reading this MSR causes a VM-exit. */
308 VMXMSREXIT_INTERCEPT_READ = 0xb,
309 /** Reading this MSR does not cause a VM-exit. */
310 VMXMSREXIT_PASSTHRU_READ
311} VMXMSREXITREAD;
312/** Pointer to MSR-bitmap read permissions. */
313typedef VMXMSREXITREAD* PVMXMSREXITREAD;
314
315/**
316 * MSR-bitmap write permissions.
317 */
318typedef enum VMXMSREXITWRITE
319{
320 /** Writing to this MSR causes a VM-exit. */
321 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
322 /** Writing to this MSR does not cause a VM-exit. */
323 VMXMSREXIT_PASSTHRU_WRITE
324} VMXMSREXITWRITE;
325/** Pointer to MSR-bitmap write permissions. */
326typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
327
328
329/**
330 * VMX VM-exit handler.
331 *
332 * @returns Strict VBox status code (i.e. informational status codes too).
333 * @param pVCpu The cross context virtual CPU structure.
334 * @param pVmxTransient Pointer to the VMX-transient structure.
335 */
336#ifndef HMVMX_USE_FUNCTION_TABLE
337typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
338#else
339typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
340/** Pointer to VM-exit handler. */
341typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
342#endif
343
344/**
345 * VMX VM-exit handler, non-strict status code.
346 *
347 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
348 *
349 * @returns VBox status code, no informational status code returned.
350 * @param pVCpu The cross context virtual CPU structure.
351 * @param pVmxTransient Pointer to the VMX-transient structure.
352 *
353 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
354 * use of that status code will be replaced with VINF_EM_SOMETHING
355 * later when switching over to IEM.
356 */
357#ifndef HMVMX_USE_FUNCTION_TABLE
358typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
359#else
360typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
361#endif
362
363
364/*********************************************************************************************************************************
365* Internal Functions *
366*********************************************************************************************************************************/
367static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush);
368static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr);
369static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
370static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat);
371static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
372 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState);
373#if HC_ARCH_BITS == 32
374static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
375#endif
376#ifndef HMVMX_USE_FUNCTION_TABLE
377DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
378# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
379# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
380#else
381# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
382# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
383#endif
384
385
386/** @name VM-exit handlers.
387 * @{
388 */
389static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
390static FNVMXEXITHANDLER hmR0VmxExitExtInt;
391static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
392static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
393static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
394static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
395static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
396static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
397static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
398static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
399static FNVMXEXITHANDLER hmR0VmxExitCpuid;
400static FNVMXEXITHANDLER hmR0VmxExitGetsec;
401static FNVMXEXITHANDLER hmR0VmxExitHlt;
402static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
403static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
404static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
405static FNVMXEXITHANDLER hmR0VmxExitVmcall;
406static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
407static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
408static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
409static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
410static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
411static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
412static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
413static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
414static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
415static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
416static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
417static FNVMXEXITHANDLER hmR0VmxExitMwait;
418static FNVMXEXITHANDLER hmR0VmxExitMtf;
419static FNVMXEXITHANDLER hmR0VmxExitMonitor;
420static FNVMXEXITHANDLER hmR0VmxExitPause;
421static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
422static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
423static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
424static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
425static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
426static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
427static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
428static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
429static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
430static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
431static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
432static FNVMXEXITHANDLER hmR0VmxExitRdrand;
433static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
434/** @} */
435
436static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
437static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
438static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
439static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
440static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
441static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
442static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
443static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu);
444
445
446/*********************************************************************************************************************************
447* Global Variables *
448*********************************************************************************************************************************/
449#ifdef HMVMX_USE_FUNCTION_TABLE
450
451/**
452 * VMX_EXIT dispatch table.
453 */
454static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
455{
456 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
457 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
458 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
459 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
460 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
461 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
462 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
463 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
464 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
465 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
466 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
467 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
468 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
469 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
470 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
471 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
472 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
473 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
474 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
475 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
476 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
477 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
478 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
479 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
480 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
481 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
482 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
483 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
484 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
485 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
486 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
487 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
488 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
489 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
490 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
491 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
492 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
493 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
494 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
495 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
496 /* 40 UNDEFINED */ hmR0VmxExitPause,
497 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
498 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
499 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
500 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
501 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
502 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
503 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
504 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
505 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
506 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
507 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
508 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
509 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
510 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
511 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
512 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
513 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
514 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
515 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
516 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
517 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
518 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
519 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
520 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
521};
522#endif /* HMVMX_USE_FUNCTION_TABLE */
523
524#ifdef VBOX_STRICT
525static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
526{
527 /* 0 */ "(Not Used)",
528 /* 1 */ "VMCALL executed in VMX root operation.",
529 /* 2 */ "VMCLEAR with invalid physical address.",
530 /* 3 */ "VMCLEAR with VMXON pointer.",
531 /* 4 */ "VMLAUNCH with non-clear VMCS.",
532 /* 5 */ "VMRESUME with non-launched VMCS.",
533 /* 6 */ "VMRESUME after VMXOFF",
534 /* 7 */ "VM-entry with invalid control fields.",
535 /* 8 */ "VM-entry with invalid host state fields.",
536 /* 9 */ "VMPTRLD with invalid physical address.",
537 /* 10 */ "VMPTRLD with VMXON pointer.",
538 /* 11 */ "VMPTRLD with incorrect revision identifier.",
539 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
540 /* 13 */ "VMWRITE to read-only VMCS component.",
541 /* 14 */ "(Not Used)",
542 /* 15 */ "VMXON executed in VMX root operation.",
543 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
544 /* 17 */ "VM-entry with non-launched executing VMCS.",
545 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
546 /* 19 */ "VMCALL with non-clear VMCS.",
547 /* 20 */ "VMCALL with invalid VM-exit control fields.",
548 /* 21 */ "(Not Used)",
549 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
550 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
551 /* 24 */ "VMCALL with invalid SMM-monitor features.",
552 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
553 /* 26 */ "VM-entry with events blocked by MOV SS.",
554 /* 27 */ "(Not Used)",
555 /* 28 */ "Invalid operand to INVEPT/INVVPID."
556};
557#endif /* VBOX_STRICT */
558
559
560
561/**
562 * Updates the VM's last error record.
563 *
564 * If there was a VMX instruction error, reads the error data from the VMCS and
565 * updates VCPU's last error record as well.
566 *
567 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
568 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
569 * VERR_VMX_INVALID_VMCS_FIELD.
570 * @param rc The error code.
571 */
572static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
573{
574 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
575 || rc == VERR_VMX_UNABLE_TO_START_VM)
576 {
577 AssertPtrReturnVoid(pVCpu);
578 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
579 }
580 pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
581}
582
583
584/**
585 * Reads the VM-entry interruption-information field from the VMCS into the VMX
586 * transient structure.
587 *
588 * @returns VBox status code.
589 * @param pVmxTransient Pointer to the VMX transient structure.
590 *
591 * @remarks No-long-jump zone!!!
592 */
593DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
594{
595 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
596 AssertRCReturn(rc, rc);
597 return VINF_SUCCESS;
598}
599
600#ifdef VBOX_STRICT
601/**
602 * Reads the VM-entry exception error code field from the VMCS into
603 * the VMX transient structure.
604 *
605 * @returns VBox status code.
606 * @param pVmxTransient Pointer to the VMX transient structure.
607 *
608 * @remarks No-long-jump zone!!!
609 */
610DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
611{
612 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
613 AssertRCReturn(rc, rc);
614 return VINF_SUCCESS;
615}
616
617
618/**
619 * Reads the VM-entry exception error code field from the VMCS into
620 * the VMX transient structure.
621 *
622 * @returns VBox status code.
623 * @param pVmxTransient Pointer to the VMX transient structure.
624 *
625 * @remarks No-long-jump zone!!!
626 */
627DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
628{
629 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
630 AssertRCReturn(rc, rc);
631 return VINF_SUCCESS;
632}
633#endif /* VBOX_STRICT */
634
635
636/**
637 * Reads the VM-exit interruption-information field from the VMCS into the VMX
638 * transient structure.
639 *
640 * @returns VBox status code.
641 * @param pVmxTransient Pointer to the VMX transient structure.
642 */
643DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
644{
645 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
646 {
647 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
648 AssertRCReturn(rc,rc);
649 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_INFO;
650 }
651 return VINF_SUCCESS;
652}
653
654
655/**
656 * Reads the VM-exit interruption error code from the VMCS into the VMX
657 * transient structure.
658 *
659 * @returns VBox status code.
660 * @param pVmxTransient Pointer to the VMX transient structure.
661 */
662DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
663{
664 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
665 {
666 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
667 AssertRCReturn(rc, rc);
668 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
669 }
670 return VINF_SUCCESS;
671}
672
673
674/**
675 * Reads the VM-exit instruction length field from the VMCS into the VMX
676 * transient structure.
677 *
678 * @returns VBox status code.
679 * @param pVmxTransient Pointer to the VMX transient structure.
680 */
681DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
682{
683 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
684 {
685 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
686 AssertRCReturn(rc, rc);
687 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_LEN;
688 }
689 return VINF_SUCCESS;
690}
691
692
693/**
694 * Reads the VM-exit instruction-information field from the VMCS into
695 * the VMX transient structure.
696 *
697 * @returns VBox status code.
698 * @param pVmxTransient Pointer to the VMX transient structure.
699 */
700DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
701{
702 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
703 {
704 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
705 AssertRCReturn(rc, rc);
706 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_INFO;
707 }
708 return VINF_SUCCESS;
709}
710
711
712/**
713 * Reads the exit code qualification from the VMCS into the VMX transient
714 * structure.
715 *
716 * @returns VBox status code.
717 * @param pVCpu The cross context virtual CPU structure of the
718 * calling EMT. (Required for the VMCS cache case.)
719 * @param pVmxTransient Pointer to the VMX transient structure.
720 */
721DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
722{
723 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
724 {
725 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
726 AssertRCReturn(rc, rc);
727 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION;
728 }
729 return VINF_SUCCESS;
730}
731
732
733/**
734 * Reads the IDT-vectoring information field from the VMCS into the VMX
735 * transient structure.
736 *
737 * @returns VBox status code.
738 * @param pVmxTransient Pointer to the VMX transient structure.
739 *
740 * @remarks No-long-jump zone!!!
741 */
742DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
743{
744 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
745 {
746 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
747 AssertRCReturn(rc, rc);
748 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_INFO;
749 }
750 return VINF_SUCCESS;
751}
752
753
754/**
755 * Reads the IDT-vectoring error code from the VMCS into the VMX
756 * transient structure.
757 *
758 * @returns VBox status code.
759 * @param pVmxTransient Pointer to the VMX transient structure.
760 */
761DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
762{
763 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
764 {
765 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
766 AssertRCReturn(rc, rc);
767 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
768 }
769 return VINF_SUCCESS;
770}
771
772
773/**
774 * Enters VMX root mode operation on the current CPU.
775 *
776 * @returns VBox status code.
777 * @param pVM The cross context VM structure. Can be
778 * NULL, after a resume.
779 * @param HCPhysCpuPage Physical address of the VMXON region.
780 * @param pvCpuPage Pointer to the VMXON region.
781 */
782static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
783{
784 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
785 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
786 Assert(pvCpuPage);
787 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
788
789 if (pVM)
790 {
791 /* Write the VMCS revision dword to the VMXON region. */
792 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_VMCS_ID(pVM->hm.s.vmx.Msrs.u64Basic);
793 }
794
795 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
796 RTCCUINTREG fEFlags = ASMIntDisableFlags();
797
798 /* Enable the VMX bit in CR4 if necessary. */
799 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
800
801 /* Enter VMX root mode. */
802 int rc = VMXEnable(HCPhysCpuPage);
803 if (RT_FAILURE(rc))
804 {
805 if (!(uOldCr4 & X86_CR4_VMXE))
806 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
807
808 if (pVM)
809 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
810 }
811
812 /* Restore interrupts. */
813 ASMSetFlags(fEFlags);
814 return rc;
815}
816
817
818/**
819 * Exits VMX root mode operation on the current CPU.
820 *
821 * @returns VBox status code.
822 */
823static int hmR0VmxLeaveRootMode(void)
824{
825 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
826
827 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
828 RTCCUINTREG fEFlags = ASMIntDisableFlags();
829
830 /* If we're for some reason not in VMX root mode, then don't leave it. */
831 RTCCUINTREG uHostCR4 = ASMGetCR4();
832
833 int rc;
834 if (uHostCR4 & X86_CR4_VMXE)
835 {
836 /* Exit VMX root mode and clear the VMX bit in CR4. */
837 VMXDisable();
838 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
839 rc = VINF_SUCCESS;
840 }
841 else
842 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
843
844 /* Restore interrupts. */
845 ASMSetFlags(fEFlags);
846 return rc;
847}
848
849
850/**
851 * Allocates and maps one physically contiguous page. The allocated page is
852 * zero'd out. (Used by various VT-x structures).
853 *
854 * @returns IPRT status code.
855 * @param pMemObj Pointer to the ring-0 memory object.
856 * @param ppVirt Where to store the virtual address of the
857 * allocation.
858 * @param pHCPhys Where to store the physical address of the
859 * allocation.
860 */
861static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
862{
863 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
864 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
865 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
866
867 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
868 if (RT_FAILURE(rc))
869 return rc;
870 *ppVirt = RTR0MemObjAddress(*pMemObj);
871 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
872 ASMMemZero32(*ppVirt, PAGE_SIZE);
873 return VINF_SUCCESS;
874}
875
876
877/**
878 * Frees and unmaps an allocated physical page.
879 *
880 * @param pMemObj Pointer to the ring-0 memory object.
881 * @param ppVirt Where to re-initialize the virtual address of
882 * allocation as 0.
883 * @param pHCPhys Where to re-initialize the physical address of the
884 * allocation as 0.
885 */
886static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
887{
888 AssertPtr(pMemObj);
889 AssertPtr(ppVirt);
890 AssertPtr(pHCPhys);
891 if (*pMemObj != NIL_RTR0MEMOBJ)
892 {
893 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
894 AssertRC(rc);
895 *pMemObj = NIL_RTR0MEMOBJ;
896 *ppVirt = 0;
897 *pHCPhys = 0;
898 }
899}
900
901
902/**
903 * Worker function to free VT-x related structures.
904 *
905 * @returns IPRT status code.
906 * @param pVM The cross context VM structure.
907 */
908static void hmR0VmxStructsFree(PVM pVM)
909{
910 for (VMCPUID i = 0; i < pVM->cCpus; i++)
911 {
912 PVMCPU pVCpu = &pVM->aCpus[i];
913 AssertPtr(pVCpu);
914
915 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
916 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
917
918 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
919 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
920
921 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
922 }
923
924 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
925#ifdef VBOX_WITH_CRASHDUMP_MAGIC
926 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
927#endif
928}
929
930
931/**
932 * Worker function to allocate VT-x related VM structures.
933 *
934 * @returns IPRT status code.
935 * @param pVM The cross context VM structure.
936 */
937static int hmR0VmxStructsAlloc(PVM pVM)
938{
939 /*
940 * Initialize members up-front so we can cleanup properly on allocation failure.
941 */
942#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
943 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
944 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
945 pVM->hm.s.vmx.HCPhys##a_Name = 0;
946
947#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
948 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
949 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
950 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
951
952#ifdef VBOX_WITH_CRASHDUMP_MAGIC
953 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
954#endif
955 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
956
957 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
958 for (VMCPUID i = 0; i < pVM->cCpus; i++)
959 {
960 PVMCPU pVCpu = &pVM->aCpus[i];
961 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
962 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
963 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
964 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
965 }
966#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
967#undef VMXLOCAL_INIT_VM_MEMOBJ
968
969 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
970 AssertReturnStmt(MSR_IA32_VMX_BASIC_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64Basic) <= PAGE_SIZE,
971 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
972 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
973
974 /*
975 * Allocate all the VT-x structures.
976 */
977 int rc = VINF_SUCCESS;
978#ifdef VBOX_WITH_CRASHDUMP_MAGIC
979 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
980 if (RT_FAILURE(rc))
981 goto cleanup;
982 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
983 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
984#endif
985
986 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
987 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
988 {
989 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
990 &pVM->hm.s.vmx.HCPhysApicAccess);
991 if (RT_FAILURE(rc))
992 goto cleanup;
993 }
994
995 /*
996 * Initialize per-VCPU VT-x structures.
997 */
998 for (VMCPUID i = 0; i < pVM->cCpus; i++)
999 {
1000 PVMCPU pVCpu = &pVM->aCpus[i];
1001 AssertPtr(pVCpu);
1002
1003 /* Allocate the VM control structure (VMCS). */
1004 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1005 if (RT_FAILURE(rc))
1006 goto cleanup;
1007
1008 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1009 if ( PDMHasApic(pVM)
1010 && (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW))
1011 {
1012 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1013 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1014 if (RT_FAILURE(rc))
1015 goto cleanup;
1016 }
1017
1018 /*
1019 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1020 * transparent accesses of specific MSRs.
1021 *
1022 * If the condition for enabling MSR bitmaps changes here, don't forget to
1023 * update HMAreMsrBitmapsAvailable().
1024 */
1025 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1026 {
1027 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1028 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1029 if (RT_FAILURE(rc))
1030 goto cleanup;
1031 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1032 }
1033
1034 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1035 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1036 if (RT_FAILURE(rc))
1037 goto cleanup;
1038
1039 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1040 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1041 if (RT_FAILURE(rc))
1042 goto cleanup;
1043 }
1044
1045 return VINF_SUCCESS;
1046
1047cleanup:
1048 hmR0VmxStructsFree(pVM);
1049 return rc;
1050}
1051
1052
1053/**
1054 * Does global VT-x initialization (called during module initialization).
1055 *
1056 * @returns VBox status code.
1057 */
1058VMMR0DECL(int) VMXR0GlobalInit(void)
1059{
1060#ifdef HMVMX_USE_FUNCTION_TABLE
1061 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1062# ifdef VBOX_STRICT
1063 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1064 Assert(g_apfnVMExitHandlers[i]);
1065# endif
1066#endif
1067 return VINF_SUCCESS;
1068}
1069
1070
1071/**
1072 * Does global VT-x termination (called during module termination).
1073 */
1074VMMR0DECL(void) VMXR0GlobalTerm()
1075{
1076 /* Nothing to do currently. */
1077}
1078
1079
1080/**
1081 * Sets up and activates VT-x on the current CPU.
1082 *
1083 * @returns VBox status code.
1084 * @param pHostCpu Pointer to the global CPU info struct.
1085 * @param pVM The cross context VM structure. Can be
1086 * NULL after a host resume operation.
1087 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1088 * fEnabledByHost is @c true).
1089 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1090 * @a fEnabledByHost is @c true).
1091 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1092 * enable VT-x on the host.
1093 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1094 */
1095VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1096 void *pvMsrs)
1097{
1098 Assert(pHostCpu);
1099 Assert(pvMsrs);
1100 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1101
1102 /* Enable VT-x if it's not already enabled by the host. */
1103 if (!fEnabledByHost)
1104 {
1105 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1106 if (RT_FAILURE(rc))
1107 return rc;
1108 }
1109
1110 /*
1111 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
1112 * using EPTPs) so we don't retain any stale guest-physical mappings which won't get
1113 * invalidated when flushing by VPID.
1114 */
1115 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1116 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1117 {
1118 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
1119 pHostCpu->fFlushAsidBeforeUse = false;
1120 }
1121 else
1122 pHostCpu->fFlushAsidBeforeUse = true;
1123
1124 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1125 ++pHostCpu->cTlbFlushes;
1126
1127 return VINF_SUCCESS;
1128}
1129
1130
1131/**
1132 * Deactivates VT-x on the current CPU.
1133 *
1134 * @returns VBox status code.
1135 * @param pHostCpu Pointer to the global CPU info struct.
1136 * @param pvCpuPage Pointer to the VMXON region.
1137 * @param HCPhysCpuPage Physical address of the VMXON region.
1138 *
1139 * @remarks This function should never be called when SUPR0EnableVTx() or
1140 * similar was used to enable VT-x on the host.
1141 */
1142VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1143{
1144 RT_NOREF3(pHostCpu, pvCpuPage, HCPhysCpuPage);
1145
1146 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1147 return hmR0VmxLeaveRootMode();
1148}
1149
1150
1151/**
1152 * Sets the permission bits for the specified MSR in the MSR bitmap.
1153 *
1154 * @param pVCpu The cross context virtual CPU structure.
1155 * @param uMsr The MSR value.
1156 * @param enmRead Whether reading this MSR causes a VM-exit.
1157 * @param enmWrite Whether writing this MSR causes a VM-exit.
1158 */
1159static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1160{
1161 int32_t iBit;
1162 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1163
1164 /*
1165 * Layout:
1166 * 0x000 - 0x3ff - Low MSR read bits
1167 * 0x400 - 0x7ff - High MSR read bits
1168 * 0x800 - 0xbff - Low MSR write bits
1169 * 0xc00 - 0xfff - High MSR write bits
1170 */
1171 if (uMsr <= 0x00001fff)
1172 iBit = uMsr;
1173 else if (uMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1174 {
1175 iBit = uMsr - UINT32_C(0xc0000000);
1176 pbMsrBitmap += 0x400;
1177 }
1178 else
1179 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1180
1181 Assert(iBit <= 0x1fff);
1182 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1183 ASMBitSet(pbMsrBitmap, iBit);
1184 else
1185 ASMBitClear(pbMsrBitmap, iBit);
1186
1187 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1188 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1189 else
1190 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1191}
1192
1193
1194#ifdef VBOX_STRICT
1195/**
1196 * Gets the permission bits for the specified MSR in the MSR bitmap.
1197 *
1198 * @returns VBox status code.
1199 * @retval VINF_SUCCESS if the specified MSR is found.
1200 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1201 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1202 *
1203 * @param pVCpu The cross context virtual CPU structure.
1204 * @param uMsr The MSR.
1205 * @param penmRead Where to store the read permissions.
1206 * @param penmWrite Where to store the write permissions.
1207 */
1208static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1209{
1210 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1211 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1212 int32_t iBit;
1213 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1214
1215 /* See hmR0VmxSetMsrPermission() for the layout. */
1216 if (uMsr <= 0x00001fff)
1217 iBit = uMsr;
1218 else if ( uMsr >= 0xc0000000
1219 && uMsr <= 0xc0001fff)
1220 {
1221 iBit = (uMsr - 0xc0000000);
1222 pbMsrBitmap += 0x400;
1223 }
1224 else
1225 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1226
1227 Assert(iBit <= 0x1fff);
1228 if (ASMBitTest(pbMsrBitmap, iBit))
1229 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1230 else
1231 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1232
1233 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1234 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1235 else
1236 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1237 return VINF_SUCCESS;
1238}
1239#endif /* VBOX_STRICT */
1240
1241
1242/**
1243 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1244 * area.
1245 *
1246 * @returns VBox status code.
1247 * @param pVCpu The cross context virtual CPU structure.
1248 * @param cMsrs The number of MSRs.
1249 */
1250static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1251{
1252 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1253 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1254 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1255 {
1256 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1257 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1258 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1259 }
1260
1261 /* Update number of guest MSRs to load/store across the world-switch. */
1262 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1263 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1264
1265 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1266 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1267 AssertRCReturn(rc, rc);
1268
1269 /* Update the VCPU's copy of the MSR count. */
1270 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1271
1272 return VINF_SUCCESS;
1273}
1274
1275
1276/**
1277 * Adds a new (or updates the value of an existing) guest/host MSR
1278 * pair to be swapped during the world-switch as part of the
1279 * auto-load/store MSR area in the VMCS.
1280 *
1281 * @returns VBox status code.
1282 * @param pVCpu The cross context virtual CPU structure.
1283 * @param uMsr The MSR.
1284 * @param uGuestMsrValue Value of the guest MSR.
1285 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1286 * necessary.
1287 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1288 * its value was updated. Optional, can be NULL.
1289 */
1290static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1291 bool *pfAddedAndUpdated)
1292{
1293 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1294 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1295 uint32_t i;
1296 for (i = 0; i < cMsrs; i++)
1297 {
1298 if (pGuestMsr->u32Msr == uMsr)
1299 break;
1300 pGuestMsr++;
1301 }
1302
1303 bool fAdded = false;
1304 if (i == cMsrs)
1305 {
1306 ++cMsrs;
1307 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1308 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1309
1310 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1311 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1312 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1313
1314 fAdded = true;
1315 }
1316
1317 /* Update the MSR values in the auto-load/store MSR area. */
1318 pGuestMsr->u32Msr = uMsr;
1319 pGuestMsr->u64Value = uGuestMsrValue;
1320
1321 /* Create/update the MSR slot in the host MSR area. */
1322 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1323 pHostMsr += i;
1324 pHostMsr->u32Msr = uMsr;
1325
1326 /*
1327 * Update the host MSR only when requested by the caller AND when we're
1328 * adding it to the auto-load/store area. Otherwise, it would have been
1329 * updated by hmR0VmxExportHostMsrs(). We do this for performance reasons.
1330 */
1331 bool fUpdatedMsrValue = false;
1332 if ( fAdded
1333 && fUpdateHostMsr)
1334 {
1335 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1336 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1337 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1338 fUpdatedMsrValue = true;
1339 }
1340
1341 if (pfAddedAndUpdated)
1342 *pfAddedAndUpdated = fUpdatedMsrValue;
1343 return VINF_SUCCESS;
1344}
1345
1346
1347/**
1348 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1349 * auto-load/store MSR area in the VMCS.
1350 *
1351 * @returns VBox status code.
1352 * @param pVCpu The cross context virtual CPU structure.
1353 * @param uMsr The MSR.
1354 */
1355static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1356{
1357 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1358 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1359 for (uint32_t i = 0; i < cMsrs; i++)
1360 {
1361 /* Find the MSR. */
1362 if (pGuestMsr->u32Msr == uMsr)
1363 {
1364 /* If it's the last MSR, simply reduce the count. */
1365 if (i == cMsrs - 1)
1366 {
1367 --cMsrs;
1368 break;
1369 }
1370
1371 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1372 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1373 pLastGuestMsr += cMsrs - 1;
1374 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1375 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1376
1377 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1378 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1379 pLastHostMsr += cMsrs - 1;
1380 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1381 pHostMsr->u64Value = pLastHostMsr->u64Value;
1382 --cMsrs;
1383 break;
1384 }
1385 pGuestMsr++;
1386 }
1387
1388 /* Update the VMCS if the count changed (meaning the MSR was found). */
1389 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1390 {
1391 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1392 AssertRCReturn(rc, rc);
1393
1394 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1395 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1396 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1397
1398 Log4Func(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1399 return VINF_SUCCESS;
1400 }
1401
1402 return VERR_NOT_FOUND;
1403}
1404
1405
1406/**
1407 * Checks if the specified guest MSR is part of the auto-load/store area in
1408 * the VMCS.
1409 *
1410 * @returns true if found, false otherwise.
1411 * @param pVCpu The cross context virtual CPU structure.
1412 * @param uMsr The MSR to find.
1413 */
1414static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1415{
1416 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1417 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1418
1419 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1420 {
1421 if (pGuestMsr->u32Msr == uMsr)
1422 return true;
1423 }
1424 return false;
1425}
1426
1427
1428/**
1429 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1430 *
1431 * @param pVCpu The cross context virtual CPU structure.
1432 *
1433 * @remarks No-long-jump zone!!!
1434 */
1435static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1436{
1437 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1438 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1439 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1440 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1441
1442 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1443 {
1444 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1445
1446 /*
1447 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1448 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1449 */
1450 if (pHostMsr->u32Msr == MSR_K6_EFER)
1451 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1452 else
1453 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1454 }
1455
1456 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1457}
1458
1459
1460/**
1461 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1462 * perform lazy restoration of the host MSRs while leaving VT-x.
1463 *
1464 * @param pVCpu The cross context virtual CPU structure.
1465 *
1466 * @remarks No-long-jump zone!!!
1467 */
1468static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1469{
1470 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1471
1472 /*
1473 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1474 */
1475 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1476 {
1477 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1478#if HC_ARCH_BITS == 64
1479 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1480 {
1481 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1482 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1483 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1484 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1485 }
1486#endif
1487 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1488 }
1489}
1490
1491
1492/**
1493 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1494 * lazily while leaving VT-x.
1495 *
1496 * @returns true if it does, false otherwise.
1497 * @param pVCpu The cross context virtual CPU structure.
1498 * @param uMsr The MSR to check.
1499 */
1500static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1501{
1502 NOREF(pVCpu);
1503#if HC_ARCH_BITS == 64
1504 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1505 {
1506 switch (uMsr)
1507 {
1508 case MSR_K8_LSTAR:
1509 case MSR_K6_STAR:
1510 case MSR_K8_SF_MASK:
1511 case MSR_K8_KERNEL_GS_BASE:
1512 return true;
1513 }
1514 }
1515#else
1516 RT_NOREF(pVCpu, uMsr);
1517#endif
1518 return false;
1519}
1520
1521
1522/**
1523 * Loads a set of guests MSRs to allow read/passthru to the guest.
1524 *
1525 * The name of this function is slightly confusing. This function does NOT
1526 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1527 * common prefix for functions dealing with "lazy restoration" of the shared
1528 * MSRs.
1529 *
1530 * @param pVCpu The cross context virtual CPU structure.
1531 *
1532 * @remarks No-long-jump zone!!!
1533 */
1534static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1535{
1536 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1537 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1538
1539 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1540#if HC_ARCH_BITS == 64
1541 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1542 {
1543 /*
1544 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1545 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1546 * we can skip a few MSR writes.
1547 *
1548 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1549 * guest MSR values in the guest-CPU context might be different to what's currently
1550 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1551 * CPU, see @bugref{8728}.
1552 */
1553 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1554 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1555 && pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1556 && pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1557 && pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1558 && pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1559 {
1560#ifdef VBOX_STRICT
1561 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1562 Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1563 Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
1564 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
1565#endif
1566 }
1567 else
1568 {
1569 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1570 ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
1571 ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
1572 ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
1573 }
1574 }
1575#endif
1576 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1577}
1578
1579
1580/**
1581 * Performs lazy restoration of the set of host MSRs if they were previously
1582 * loaded with guest MSR values.
1583 *
1584 * @param pVCpu The cross context virtual CPU structure.
1585 *
1586 * @remarks No-long-jump zone!!!
1587 * @remarks The guest MSRs should have been saved back into the guest-CPU
1588 * context by hmR0VmxImportGuestState()!!!
1589 */
1590static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1591{
1592 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1593 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1594
1595 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1596 {
1597 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1598#if HC_ARCH_BITS == 64
1599 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1600 {
1601 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1602 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1603 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1604 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1605 }
1606#endif
1607 }
1608 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1609}
1610
1611
1612/**
1613 * Verifies that our cached values of the VMCS fields are all consistent with
1614 * what's actually present in the VMCS.
1615 *
1616 * @returns VBox status code.
1617 * @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
1618 * @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
1619 * VMCS content. HMCPU error-field is
1620 * updated, see VMX_VCI_XXX.
1621 * @param pVCpu The cross context virtual CPU structure.
1622 */
1623static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1624{
1625 uint32_t u32Val;
1626 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1627 AssertRCReturn(rc, rc);
1628 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32EntryCtls == u32Val,
1629 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1630 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
1631 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1632
1633 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1634 AssertRCReturn(rc, rc);
1635 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ExitCtls == u32Val,
1636 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1637 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
1638 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1639
1640 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1641 AssertRCReturn(rc, rc);
1642 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32PinCtls == u32Val,
1643 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1644 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
1645 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1646
1647 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1648 AssertRCReturn(rc, rc);
1649 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls == u32Val,
1650 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1651 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
1652 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1653
1654 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1655 {
1656 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1657 AssertRCReturn(rc, rc);
1658 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1659 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1660 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
1661 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1662 }
1663
1664 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
1665 AssertRCReturn(rc, rc);
1666 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32XcptBitmap == u32Val,
1667 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap, u32Val),
1668 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
1669 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1670
1671 uint64_t u64Val;
1672 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
1673 AssertRCReturn(rc, rc);
1674 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u64TscOffset == u64Val,
1675 ("Cache=%#RX64 VMCS=%#RX64\n", pVCpu->hm.s.vmx.u64TscOffset, u64Val),
1676 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
1677 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1678
1679 return VINF_SUCCESS;
1680}
1681
1682
1683#ifdef VBOX_STRICT
1684/**
1685 * Verifies that our cached host EFER value has not changed
1686 * since we cached it.
1687 *
1688 * @param pVCpu The cross context virtual CPU structure.
1689 */
1690static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1691{
1692 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1693
1694 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1695 {
1696 uint64_t u64Val;
1697 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1698 AssertRC(rc);
1699
1700 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1701 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1702 }
1703}
1704
1705
1706/**
1707 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1708 * VMCS are correct.
1709 *
1710 * @param pVCpu The cross context virtual CPU structure.
1711 */
1712static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1713{
1714 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1715
1716 /* Verify MSR counts in the VMCS are what we think it should be. */
1717 uint32_t cMsrs;
1718 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1719 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1720
1721 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1722 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1723
1724 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1725 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1726
1727 PCVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1728 PCVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1729 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1730 {
1731 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1732 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1733 pGuestMsr->u32Msr, cMsrs));
1734
1735 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1736 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1737 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1738
1739 /* Verify that the permissions are as expected in the MSR bitmap. */
1740 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1741 {
1742 VMXMSREXITREAD enmRead;
1743 VMXMSREXITWRITE enmWrite;
1744 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1745 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1746 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1747 {
1748 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1749 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1750 }
1751 else
1752 {
1753 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1754 pGuestMsr->u32Msr, cMsrs));
1755 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1756 pGuestMsr->u32Msr, cMsrs));
1757 }
1758 }
1759 }
1760}
1761#endif /* VBOX_STRICT */
1762
1763
1764/**
1765 * Flushes the TLB using EPT.
1766 *
1767 * @returns VBox status code.
1768 * @param pVCpu The cross context virtual CPU structure of the calling
1769 * EMT. Can be NULL depending on @a enmTlbFlush.
1770 * @param enmTlbFlush Type of flush.
1771 *
1772 * @remarks Caller is responsible for making sure this function is called only
1773 * when NestedPaging is supported and providing @a enmTlbFlush that is
1774 * supported by the CPU.
1775 * @remarks Can be called with interrupts disabled.
1776 */
1777static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush)
1778{
1779 uint64_t au64Descriptor[2];
1780 if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
1781 au64Descriptor[0] = 0;
1782 else
1783 {
1784 Assert(pVCpu);
1785 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1786 }
1787 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1788
1789 int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
1790 AssertMsg(rc == VINF_SUCCESS,
1791 ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0, rc));
1792
1793 if ( RT_SUCCESS(rc)
1794 && pVCpu)
1795 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1796}
1797
1798
1799/**
1800 * Flushes the TLB using VPID.
1801 *
1802 * @returns VBox status code.
1803 * @param pVCpu The cross context virtual CPU structure of the calling
1804 * EMT. Can be NULL depending on @a enmTlbFlush.
1805 * @param enmTlbFlush Type of flush.
1806 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1807 * on @a enmTlbFlush).
1808 *
1809 * @remarks Can be called with interrupts disabled.
1810 */
1811static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
1812{
1813 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
1814
1815 uint64_t au64Descriptor[2];
1816 if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
1817 {
1818 au64Descriptor[0] = 0;
1819 au64Descriptor[1] = 0;
1820 }
1821 else
1822 {
1823 AssertPtr(pVCpu);
1824 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1825 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1826 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1827 au64Descriptor[1] = GCPtr;
1828 }
1829
1830 int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
1831 AssertMsg(rc == VINF_SUCCESS,
1832 ("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1833
1834 if ( RT_SUCCESS(rc)
1835 && pVCpu)
1836 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1837 NOREF(rc);
1838}
1839
1840
1841/**
1842 * Invalidates a guest page by guest virtual address. Only relevant for
1843 * EPT/VPID, otherwise there is nothing really to invalidate.
1844 *
1845 * @returns VBox status code.
1846 * @param pVCpu The cross context virtual CPU structure.
1847 * @param GCVirt Guest virtual address of the page to invalidate.
1848 */
1849VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
1850{
1851 AssertPtr(pVCpu);
1852 LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
1853
1854 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1855 if (!fFlushPending)
1856 {
1857 /*
1858 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1859 * the EPT case. See @bugref{6043} and @bugref{6177}.
1860 *
1861 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1862 * as this function maybe called in a loop with individual addresses.
1863 */
1864 PVM pVM = pVCpu->CTX_SUFF(pVM);
1865 if (pVM->hm.s.vmx.fVpid)
1866 {
1867 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1868
1869#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1870 /*
1871 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1872 * where executing INVVPID outside 64-bit mode does not flush translations of
1873 * 64-bit linear addresses, see @bugref{6208#c72}.
1874 */
1875 if (RT_HI_U32(GCVirt))
1876 fVpidFlush = false;
1877#endif
1878
1879 if (fVpidFlush)
1880 {
1881 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
1882 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1883 }
1884 else
1885 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1886 }
1887 else if (pVM->hm.s.fNestedPaging)
1888 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1889 }
1890
1891 return VINF_SUCCESS;
1892}
1893
1894
1895/**
1896 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1897 * case where neither EPT nor VPID is supported by the CPU.
1898 *
1899 * @param pVCpu The cross context virtual CPU structure.
1900 * @param pCpu Pointer to the global HM struct.
1901 *
1902 * @remarks Called with interrupts disabled.
1903 */
1904static void hmR0VmxFlushTaggedTlbNone(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1905{
1906 AssertPtr(pVCpu);
1907 AssertPtr(pCpu);
1908
1909 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1910
1911 Assert(pCpu->idCpu != NIL_RTCPUID);
1912 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1913 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1914 pVCpu->hm.s.fForceTLBFlush = false;
1915 return;
1916}
1917
1918
1919/**
1920 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1921 *
1922 * @param pVCpu The cross context virtual CPU structure.
1923 * @param pCpu Pointer to the global HM CPU struct.
1924 *
1925 * @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
1926 * nomenclature. The reason is, to avoid confusion in compare statements
1927 * since the host-CPU copies are named "ASID".
1928 *
1929 * @remarks Called with interrupts disabled.
1930 */
1931static void hmR0VmxFlushTaggedTlbBoth(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1932{
1933#ifdef VBOX_WITH_STATISTICS
1934 bool fTlbFlushed = false;
1935# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1936# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1937 if (!fTlbFlushed) \
1938 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1939 } while (0)
1940#else
1941# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1942# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1943#endif
1944
1945 AssertPtr(pCpu);
1946 AssertPtr(pVCpu);
1947 Assert(pCpu->idCpu != NIL_RTCPUID);
1948
1949 PVM pVM = pVCpu->CTX_SUFF(pVM);
1950 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1951 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1952 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1953
1954 /*
1955 * Force a TLB flush for the first world-switch if the current CPU differs from the one we
1956 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1957 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1958 * cannot reuse the current ASID anymore.
1959 */
1960 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1961 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1962 {
1963 ++pCpu->uCurrentAsid;
1964 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1965 {
1966 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1967 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1968 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1969 }
1970
1971 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1972 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1973 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1974
1975 /*
1976 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1977 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1978 */
1979 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
1980 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1981 HMVMX_SET_TAGGED_TLB_FLUSHED();
1982 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1983 }
1984 else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
1985 {
1986 /*
1987 * Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
1988 * creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
1989 * tables when EPT is in use. Flushing-by-VPID will only flush linear (only
1990 * VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
1991 * mappings, see @bugref{6568}.
1992 *
1993 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
1994 */
1995 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
1996 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1997 HMVMX_SET_TAGGED_TLB_FLUSHED();
1998 }
1999
2000 pVCpu->hm.s.fForceTLBFlush = false;
2001 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2002
2003 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2004 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2005 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2006 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2007 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2008 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2009 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2010 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2011 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2012
2013 /* Update VMCS with the VPID. */
2014 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2015 AssertRC(rc);
2016
2017#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2018}
2019
2020
2021/**
2022 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2023 *
2024 * @returns VBox status code.
2025 * @param pVCpu The cross context virtual CPU structure.
2026 * @param pCpu Pointer to the global HM CPU struct.
2027 *
2028 * @remarks Called with interrupts disabled.
2029 */
2030static void hmR0VmxFlushTaggedTlbEpt(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2031{
2032 AssertPtr(pVCpu);
2033 AssertPtr(pCpu);
2034 Assert(pCpu->idCpu != NIL_RTCPUID);
2035 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2036 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2037
2038 /*
2039 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2040 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2041 */
2042 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2043 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2044 {
2045 pVCpu->hm.s.fForceTLBFlush = true;
2046 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2047 }
2048
2049 /* Check for explicit TLB flushes. */
2050 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2051 {
2052 pVCpu->hm.s.fForceTLBFlush = true;
2053 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2054 }
2055
2056 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2057 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2058
2059 if (pVCpu->hm.s.fForceTLBFlush)
2060 {
2061 hmR0VmxFlushEpt(pVCpu, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2062 pVCpu->hm.s.fForceTLBFlush = false;
2063 }
2064}
2065
2066
2067/**
2068 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2069 *
2070 * @returns VBox status code.
2071 * @param pVCpu The cross context virtual CPU structure.
2072 * @param pCpu Pointer to the global HM CPU struct.
2073 *
2074 * @remarks Called with interrupts disabled.
2075 */
2076static void hmR0VmxFlushTaggedTlbVpid(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2077{
2078 AssertPtr(pVCpu);
2079 AssertPtr(pCpu);
2080 Assert(pCpu->idCpu != NIL_RTCPUID);
2081 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2082 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2083
2084 /*
2085 * Force a TLB flush for the first world switch if the current CPU differs from the one we
2086 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2087 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2088 * cannot reuse the current ASID anymore.
2089 */
2090 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2091 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2092 {
2093 pVCpu->hm.s.fForceTLBFlush = true;
2094 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2095 }
2096
2097 /* Check for explicit TLB flushes. */
2098 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2099 {
2100 /*
2101 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2102 * hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2103 * fExplicitFlush = true here and change the pCpu->fFlushAsidBeforeUse check below to
2104 * include fExplicitFlush's too) - an obscure corner case.
2105 */
2106 pVCpu->hm.s.fForceTLBFlush = true;
2107 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2108 }
2109
2110 PVM pVM = pVCpu->CTX_SUFF(pVM);
2111 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2112 if (pVCpu->hm.s.fForceTLBFlush)
2113 {
2114 ++pCpu->uCurrentAsid;
2115 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2116 {
2117 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2118 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2119 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2120 }
2121
2122 pVCpu->hm.s.fForceTLBFlush = false;
2123 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2124 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2125 if (pCpu->fFlushAsidBeforeUse)
2126 {
2127 if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2128 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2129 else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2130 {
2131 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2132 pCpu->fFlushAsidBeforeUse = false;
2133 }
2134 else
2135 {
2136 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2137 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2138 }
2139 }
2140 }
2141
2142 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2143 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2144 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2145 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2146 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2147 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2148 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2149
2150 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2151 AssertRC(rc);
2152}
2153
2154
2155/**
2156 * Flushes the guest TLB entry based on CPU capabilities.
2157 *
2158 * @param pVCpu The cross context virtual CPU structure.
2159 * @param pCpu Pointer to the global HM CPU struct.
2160 */
2161DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2162{
2163#ifdef HMVMX_ALWAYS_FLUSH_TLB
2164 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2165#endif
2166 PVM pVM = pVCpu->CTX_SUFF(pVM);
2167 switch (pVM->hm.s.vmx.enmTlbFlushType)
2168 {
2169 case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVCpu, pCpu); break;
2170 case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pVCpu, pCpu); break;
2171 case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pVCpu, pCpu); break;
2172 case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pVCpu, pCpu); break;
2173 default:
2174 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2175 break;
2176 }
2177 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2178}
2179
2180
2181/**
2182 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2183 * TLB entries from the host TLB before VM-entry.
2184 *
2185 * @returns VBox status code.
2186 * @param pVM The cross context VM structure.
2187 */
2188static int hmR0VmxSetupTaggedTlb(PVM pVM)
2189{
2190 /*
2191 * Determine optimal flush type for Nested Paging.
2192 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2193 * guest execution (see hmR3InitFinalizeR0()).
2194 */
2195 if (pVM->hm.s.fNestedPaging)
2196 {
2197 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2198 {
2199 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2200 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2201 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2202 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2203 else
2204 {
2205 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2206 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2207 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2208 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2209 }
2210
2211 /* Make sure the write-back cacheable memory type for EPT is supported. */
2212 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2213 {
2214 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2215 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2216 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2217 }
2218
2219 /* EPT requires a page-walk length of 4. */
2220 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2221 {
2222 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2223 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2224 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2225 }
2226 }
2227 else
2228 {
2229 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2230 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2231 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2232 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2233 }
2234 }
2235
2236 /*
2237 * Determine optimal flush type for VPID.
2238 */
2239 if (pVM->hm.s.vmx.fVpid)
2240 {
2241 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2242 {
2243 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2244 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2245 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2246 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2247 else
2248 {
2249 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2250 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2251 LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2252 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2253 LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2254 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2255 pVM->hm.s.vmx.fVpid = false;
2256 }
2257 }
2258 else
2259 {
2260 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2261 Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2262 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2263 pVM->hm.s.vmx.fVpid = false;
2264 }
2265 }
2266
2267 /*
2268 * Setup the handler for flushing tagged-TLBs.
2269 */
2270 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2271 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2272 else if (pVM->hm.s.fNestedPaging)
2273 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2274 else if (pVM->hm.s.vmx.fVpid)
2275 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2276 else
2277 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2278 return VINF_SUCCESS;
2279}
2280
2281
2282/**
2283 * Sets up pin-based VM-execution controls in the VMCS.
2284 *
2285 * @returns VBox status code.
2286 * @param pVCpu The cross context virtual CPU structure.
2287 *
2288 * @remarks We don't really care about optimizing vmwrites here as it's done only
2289 * once per VM and hence we don't care about VMCS-field cache comparisons.
2290 */
2291static int hmR0VmxSetupPinCtls(PVMCPU pVCpu)
2292{
2293 PVM pVM = pVCpu->CTX_SUFF(pVM);
2294 uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.disallowed0; /* Bits set here must always be set. */
2295 uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2296
2297 fVal |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2298 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2299
2300 if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2301 fVal |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2302
2303 /* Enable the VMX preemption timer. */
2304 if (pVM->hm.s.vmx.fUsePreemptTimer)
2305 {
2306 Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2307 fVal |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2308 }
2309
2310#if 0
2311 /* Enable posted-interrupt processing. */
2312 if (pVM->hm.s.fPostedIntrs)
2313 {
2314 Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2315 Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2316 fVal |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2317 }
2318#endif
2319
2320 if ((fVal & fZap) != fVal)
2321 {
2322 LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2323 pVM->hm.s.vmx.Msrs.PinCtls.n.disallowed0, fVal, fZap));
2324 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2325 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2326 }
2327
2328 /* Commit it to the VMCS and update our cache. */
2329 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2330 AssertRCReturn(rc, rc);
2331 pVCpu->hm.s.vmx.u32PinCtls = fVal;
2332
2333 return VINF_SUCCESS;
2334}
2335
2336
2337/**
2338 * Sets up secondary processor-based VM-execution controls in the VMCS.
2339 *
2340 * @returns VBox status code.
2341 * @param pVCpu The cross context virtual CPU structure.
2342 *
2343 * @remarks We don't really care about optimizing vmwrites here as it's done only
2344 * once per VM and hence we don't care about VMCS-field cache comparisons.
2345 */
2346static int hmR0VmxSetupProcCtls2(PVMCPU pVCpu)
2347{
2348 PVM pVM = pVCpu->CTX_SUFF(pVM);
2349 uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2350 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2351
2352 /* WBINVD causes a VM-exit. */
2353 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2354 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
2355
2356 /* Enable EPT (aka nested-paging). */
2357 if (pVM->hm.s.fNestedPaging)
2358 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
2359
2360 /*
2361 * Enable the INVPCID instruction if supported by the hardware and we expose
2362 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2363 */
2364 if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2365 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2366 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2367
2368 /* Enable VPID. */
2369 if (pVM->hm.s.vmx.fVpid)
2370 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
2371
2372 /* Enable Unrestricted guest execution. */
2373 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2374 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST;
2375
2376#if 0
2377 if (pVM->hm.s.fVirtApicRegs)
2378 {
2379 /* Enable APIC-register virtualization. */
2380 Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2381 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT;
2382
2383 /* Enable virtual-interrupt delivery. */
2384 Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2385 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY;
2386 }
2387#endif
2388
2389 /* Enable Virtual-APIC page accesses if supported by the CPU. This is where the TPR shadow resides. */
2390 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2391 * done dynamically. */
2392 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2393 {
2394 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2395 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2396 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2397 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2398 AssertRCReturn(rc, rc);
2399 }
2400
2401 /* Enable RDTSCP. */
2402 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2403 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP;
2404
2405 /* Enable Pause-Loop exiting. */
2406 if ( pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2407 && pVM->hm.s.vmx.cPleGapTicks
2408 && pVM->hm.s.vmx.cPleWindowTicks)
2409 {
2410 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT;
2411
2412 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2413 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2414 AssertRCReturn(rc, rc);
2415 }
2416
2417 if ((fVal & fZap) != fVal)
2418 {
2419 LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2420 pVM->hm.s.vmx.Msrs.ProcCtls2.n.disallowed0, fVal, fZap));
2421 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2422 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2423 }
2424
2425 /* Commit it to the VMCS and update our cache. */
2426 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2427 AssertRCReturn(rc, rc);
2428 pVCpu->hm.s.vmx.u32ProcCtls2 = fVal;
2429
2430 return VINF_SUCCESS;
2431}
2432
2433
2434/**
2435 * Sets up processor-based VM-execution controls in the VMCS.
2436 *
2437 * @returns VBox status code.
2438 * @param pVCpu The cross context virtual CPU structure.
2439 *
2440 * @remarks We don't really care about optimizing vmwrites here as it's done only
2441 * once per VM and hence we don't care about VMCS-field cache comparisons.
2442 */
2443static int hmR0VmxSetupProcCtls(PVMCPU pVCpu)
2444{
2445 PVM pVM = pVCpu->CTX_SUFF(pVM);
2446 uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2447 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2448
2449 fVal |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2450 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2451 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2452 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2453 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2454 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2455 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2456
2457 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2458 if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2459 || (pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2460 {
2461 LogRelFunc(("Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2462 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2463 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2464 }
2465
2466 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2467 if (!pVM->hm.s.fNestedPaging)
2468 {
2469 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2470 fVal |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2471 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2472 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2473 }
2474
2475 /* Use TPR shadowing if supported by the CPU. */
2476 if ( PDMHasApic(pVM)
2477 && pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2478 {
2479 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2480 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2481 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2482 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2483 AssertRCReturn(rc, rc);
2484
2485 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2486 /* CR8 writes cause a VM-exit based on TPR threshold. */
2487 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2488 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2489 }
2490 else
2491 {
2492 /*
2493 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2494 * Set this control only for 64-bit guests.
2495 */
2496 if (pVM->hm.s.fAllow64BitGuests)
2497 {
2498 fVal |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2499 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2500 }
2501 }
2502
2503 /* Use MSR-bitmaps if supported by the CPU. */
2504 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2505 {
2506 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2507
2508 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2509 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2510 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2511 AssertRCReturn(rc, rc);
2512
2513 /*
2514 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2515 * automatically using dedicated fields in the VMCS.
2516 */
2517 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2518 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2519 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2520 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2521 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2522#if HC_ARCH_BITS == 64
2523 /*
2524 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2525 */
2526 if (pVM->hm.s.fAllow64BitGuests)
2527 {
2528 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2529 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2530 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2531 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2532 }
2533#endif
2534 /*
2535 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2536 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2537 */
2538 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2539 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2540
2541 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2542 }
2543
2544 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2545 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2546 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2547
2548 if ((fVal & fZap) != fVal)
2549 {
2550 LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2551 pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0, fVal, fZap));
2552 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2553 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2554 }
2555
2556 /* Commit it to the VMCS and update our cache. */
2557 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2558 AssertRCReturn(rc, rc);
2559 pVCpu->hm.s.vmx.u32ProcCtls = fVal;
2560
2561 /* Set up secondary processor-based VM-execution controls if the CPU supports it. */
2562 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2563 return hmR0VmxSetupProcCtls2(pVCpu);
2564
2565 /* Sanity check, should not really happen. */
2566 if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2567 {
2568 LogRelFunc(("Unrestricted Guest enabled when secondary processor-based VM-execution controls not available\n"));
2569 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2570 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2571 }
2572
2573 /* Old CPUs without secondary processor-based VM-execution controls would end up here. */
2574 return VINF_SUCCESS;
2575}
2576
2577
2578/**
2579 * Sets up miscellaneous (everything other than Pin & Processor-based
2580 * VM-execution) control fields in the VMCS.
2581 *
2582 * @returns VBox status code.
2583 * @param pVCpu The cross context virtual CPU structure.
2584 */
2585static int hmR0VmxSetupMiscCtls(PVMCPU pVCpu)
2586{
2587 AssertPtr(pVCpu);
2588
2589 int rc = VERR_GENERAL_FAILURE;
2590
2591 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2592#if 0
2593 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxExportGuestCR3AndCR4())*/
2594 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2595 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2596
2597 /*
2598 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2599 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2600 * We thus use the exception bitmap to control it rather than use both.
2601 */
2602 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2603 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2604
2605 /* All IO & IOIO instructions cause VM-exits. */
2606 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2607 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2608
2609 /* Initialize the MSR-bitmap area. */
2610 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2611 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2612 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2613 AssertRCReturn(rc, rc);
2614#endif
2615
2616 /* Setup MSR auto-load/store area. */
2617 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2618 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2619 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2620 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2621 AssertRCReturn(rc, rc);
2622
2623 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2624 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2625 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2626 AssertRCReturn(rc, rc);
2627
2628 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2629 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2630 AssertRCReturn(rc, rc);
2631
2632 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2633#if 0
2634 /* Setup debug controls */
2635 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2636 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2637 AssertRCReturn(rc, rc);
2638#endif
2639
2640 return rc;
2641}
2642
2643
2644/**
2645 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2646 *
2647 * We shall setup those exception intercepts that don't change during the
2648 * lifetime of the VM here. The rest are done dynamically while loading the
2649 * guest state.
2650 *
2651 * @returns VBox status code.
2652 * @param pVCpu The cross context virtual CPU structure.
2653 */
2654static int hmR0VmxInitXcptBitmap(PVMCPU pVCpu)
2655{
2656 AssertPtr(pVCpu);
2657
2658 uint32_t uXcptBitmap;
2659
2660 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2661 uXcptBitmap = RT_BIT_32(X86_XCPT_AC);
2662
2663 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2664 and writes, and because recursive #DBs can cause the CPU hang, we must always
2665 intercept #DB. */
2666 uXcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2667
2668 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2669 if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
2670 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
2671
2672 /* Commit it to the VMCS. */
2673 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
2674 AssertRCReturn(rc, rc);
2675
2676 /* Update our cache of the exception bitmap. */
2677 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
2678 return VINF_SUCCESS;
2679}
2680
2681
2682/**
2683 * Does per-VM VT-x initialization.
2684 *
2685 * @returns VBox status code.
2686 * @param pVM The cross context VM structure.
2687 */
2688VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2689{
2690 LogFlowFunc(("pVM=%p\n", pVM));
2691
2692 int rc = hmR0VmxStructsAlloc(pVM);
2693 if (RT_FAILURE(rc))
2694 {
2695 LogRelFunc(("hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2696 return rc;
2697 }
2698
2699 return VINF_SUCCESS;
2700}
2701
2702
2703/**
2704 * Does per-VM VT-x termination.
2705 *
2706 * @returns VBox status code.
2707 * @param pVM The cross context VM structure.
2708 */
2709VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2710{
2711 LogFlowFunc(("pVM=%p\n", pVM));
2712
2713#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2714 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2715 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2716#endif
2717 hmR0VmxStructsFree(pVM);
2718 return VINF_SUCCESS;
2719}
2720
2721
2722/**
2723 * Sets up the VM for execution under VT-x.
2724 * This function is only called once per-VM during initialization.
2725 *
2726 * @returns VBox status code.
2727 * @param pVM The cross context VM structure.
2728 */
2729VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2730{
2731 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2732 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2733
2734 LogFlowFunc(("pVM=%p\n", pVM));
2735
2736 /*
2737 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be
2738 * allocated. We no longer support the highly unlikely case of UnrestrictedGuest without
2739 * pRealModeTSS, see hmR3InitFinalizeR0Intel().
2740 */
2741 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2742 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2743 || !pVM->hm.s.vmx.pRealModeTSS))
2744 {
2745 LogRelFunc(("Invalid real-on-v86 state.\n"));
2746 return VERR_INTERNAL_ERROR;
2747 }
2748
2749 /* Initialize these always, see hmR3InitFinalizeR0().*/
2750 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
2751 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
2752
2753 /* Setup the tagged-TLB flush handlers. */
2754 int rc = hmR0VmxSetupTaggedTlb(pVM);
2755 if (RT_FAILURE(rc))
2756 {
2757 LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2758 return rc;
2759 }
2760
2761 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2762 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2763#if HC_ARCH_BITS == 64
2764 if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2765 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2766 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2767 {
2768 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2769 }
2770#endif
2771
2772 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2773 RTCCUINTREG uHostCR4 = ASMGetCR4();
2774 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2775 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2776
2777 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2778 {
2779 PVMCPU pVCpu = &pVM->aCpus[i];
2780 AssertPtr(pVCpu);
2781 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2782
2783 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2784 Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2785
2786 /* Set revision dword at the beginning of the VMCS structure. */
2787 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_VMCS_ID(pVM->hm.s.vmx.Msrs.u64Basic);
2788
2789 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2790 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2791 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc\n", rc),
2792 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2793
2794 /* Load this VMCS as the current VMCS. */
2795 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2796 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc\n", rc),
2797 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2798
2799 rc = hmR0VmxSetupPinCtls(pVCpu);
2800 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc\n", rc),
2801 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2802
2803 rc = hmR0VmxSetupProcCtls(pVCpu);
2804 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc\n", rc),
2805 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2806
2807 rc = hmR0VmxSetupMiscCtls(pVCpu);
2808 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc\n", rc),
2809 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2810
2811 rc = hmR0VmxInitXcptBitmap(pVCpu);
2812 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc\n", rc),
2813 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2814
2815#if HC_ARCH_BITS == 32
2816 rc = hmR0VmxInitVmcsReadCache(pVCpu);
2817 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc\n", rc),
2818 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2819#endif
2820
2821 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2822 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2823 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc\n", rc),
2824 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2825
2826 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2827
2828 hmR0VmxUpdateErrorRecord(pVCpu, rc);
2829 }
2830
2831 return VINF_SUCCESS;
2832}
2833
2834
2835/**
2836 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2837 * the VMCS.
2838 *
2839 * @returns VBox status code.
2840 */
2841static int hmR0VmxExportHostControlRegs(void)
2842{
2843 RTCCUINTREG uReg = ASMGetCR0();
2844 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2845 AssertRCReturn(rc, rc);
2846
2847 uReg = ASMGetCR3();
2848 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2849 AssertRCReturn(rc, rc);
2850
2851 uReg = ASMGetCR4();
2852 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2853 AssertRCReturn(rc, rc);
2854 return rc;
2855}
2856
2857
2858/**
2859 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2860 * the host-state area in the VMCS.
2861 *
2862 * @returns VBox status code.
2863 * @param pVCpu The cross context virtual CPU structure.
2864 */
2865static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
2866{
2867#if HC_ARCH_BITS == 64
2868/**
2869 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2870 * requirements. See hmR0VmxExportHostSegmentRegs().
2871 */
2872# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2873 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2874 { \
2875 bool fValidSelector = true; \
2876 if ((selValue) & X86_SEL_LDT) \
2877 { \
2878 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2879 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2880 } \
2881 if (fValidSelector) \
2882 { \
2883 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2884 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2885 } \
2886 (selValue) = 0; \
2887 }
2888
2889 /*
2890 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2891 * should -not- save the messed up state without restoring the original host-state,
2892 * see @bugref{7240}.
2893 *
2894 * This apparently can happen (most likely the FPU changes), deal with it rather than
2895 * asserting. Was observed booting Solaris 10u10 32-bit guest.
2896 */
2897 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2898 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2899 {
2900 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2901 pVCpu->idCpu));
2902 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2903 }
2904 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2905#else
2906 RT_NOREF(pVCpu);
2907#endif
2908
2909 /*
2910 * Host DS, ES, FS and GS segment registers.
2911 */
2912#if HC_ARCH_BITS == 64
2913 RTSEL uSelDS = ASMGetDS();
2914 RTSEL uSelES = ASMGetES();
2915 RTSEL uSelFS = ASMGetFS();
2916 RTSEL uSelGS = ASMGetGS();
2917#else
2918 RTSEL uSelDS = 0;
2919 RTSEL uSelES = 0;
2920 RTSEL uSelFS = 0;
2921 RTSEL uSelGS = 0;
2922#endif
2923
2924 /*
2925 * Host CS and SS segment registers.
2926 */
2927 RTSEL uSelCS = ASMGetCS();
2928 RTSEL uSelSS = ASMGetSS();
2929
2930 /*
2931 * Host TR segment register.
2932 */
2933 RTSEL uSelTR = ASMGetTR();
2934
2935#if HC_ARCH_BITS == 64
2936 /*
2937 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
2938 * gain VM-entry and restore them before we get preempted.
2939 *
2940 * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2941 */
2942 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2943 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2944 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2945 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2946# undef VMXLOCAL_ADJUST_HOST_SEG
2947#endif
2948
2949 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2950 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2951 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2952 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2953 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2954 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2955 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2956 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2957 Assert(uSelCS);
2958 Assert(uSelTR);
2959
2960 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2961#if 0
2962 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2963 Assert(uSelSS != 0);
2964#endif
2965
2966 /* Write these host selector fields into the host-state area in the VMCS. */
2967 int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2968 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2969#if HC_ARCH_BITS == 64
2970 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2971 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2972 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2973 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2974#else
2975 NOREF(uSelDS);
2976 NOREF(uSelES);
2977 NOREF(uSelFS);
2978 NOREF(uSelGS);
2979#endif
2980 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2981 AssertRCReturn(rc, rc);
2982
2983 /*
2984 * Host GDTR and IDTR.
2985 */
2986 RTGDTR Gdtr;
2987 RTIDTR Idtr;
2988 RT_ZERO(Gdtr);
2989 RT_ZERO(Idtr);
2990 ASMGetGDTR(&Gdtr);
2991 ASMGetIDTR(&Idtr);
2992 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2993 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2994 AssertRCReturn(rc, rc);
2995
2996#if HC_ARCH_BITS == 64
2997 /*
2998 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
2999 * them to the maximum limit (0xffff) on every VM-exit.
3000 */
3001 if (Gdtr.cbGdt != 0xffff)
3002 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3003
3004 /*
3005 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
3006 * Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
3007 * as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
3008 * However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
3009 * uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
3010 * alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
3011 * at 0xffff on hosts where we are sure it won't cause trouble.
3012 */
3013# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3014 if (Idtr.cbIdt < 0x0fff)
3015# else
3016 if (Idtr.cbIdt != 0xffff)
3017# endif
3018 {
3019 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3020 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3021 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3022 }
3023#endif
3024
3025 /*
3026 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3027 * and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3028 * RPL should be too in most cases.
3029 */
3030 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3031 ("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3032
3033 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3034#if HC_ARCH_BITS == 64
3035 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3036
3037 /*
3038 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3039 * all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3040 * restoration if the host has something else. Task switching is not supported in 64-bit
3041 * mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3042 * limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3043 *
3044 * [1] See Intel spec. 3.5 "System Descriptor Types".
3045 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3046 */
3047 PVM pVM = pVCpu->CTX_SUFF(pVM);
3048 Assert(pDesc->System.u4Type == 11);
3049 if ( pDesc->System.u16LimitLow != 0x67
3050 || pDesc->System.u4LimitHigh)
3051 {
3052 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3053 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3054 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3055 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3056 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3057 }
3058
3059 /*
3060 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3061 */
3062 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3063 {
3064 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3065 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3066 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3067 {
3068 /* The GDT is read-only but the writable GDT is available. */
3069 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3070 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3071 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3072 AssertRCReturn(rc, rc);
3073 }
3074 }
3075#else
3076 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3077#endif
3078 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3079 AssertRCReturn(rc, rc);
3080
3081 /*
3082 * Host FS base and GS base.
3083 */
3084#if HC_ARCH_BITS == 64
3085 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3086 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3087 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3088 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3089 AssertRCReturn(rc, rc);
3090
3091 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3092 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3093 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3094 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3095 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3096#endif
3097 return VINF_SUCCESS;
3098}
3099
3100
3101/**
3102 * Exports certain host MSRs in the VM-exit MSR-load area and some in the
3103 * host-state area of the VMCS.
3104 *
3105 * Theses MSRs will be automatically restored on the host after every successful
3106 * VM-exit.
3107 *
3108 * @returns VBox status code.
3109 * @param pVCpu The cross context virtual CPU structure.
3110 *
3111 * @remarks No-long-jump zone!!!
3112 */
3113static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
3114{
3115 AssertPtr(pVCpu);
3116 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3117
3118 /*
3119 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3120 * rather than swapping them on every VM-entry.
3121 */
3122 hmR0VmxLazySaveHostMsrs(pVCpu);
3123
3124 /*
3125 * Host Sysenter MSRs.
3126 */
3127 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3128#if HC_ARCH_BITS == 32
3129 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3130 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3131#else
3132 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3133 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3134#endif
3135 AssertRCReturn(rc, rc);
3136
3137 /*
3138 * Host EFER MSR.
3139 *
3140 * If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3141 * done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3142 */
3143 PVM pVM = pVCpu->CTX_SUFF(pVM);
3144 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3145 {
3146 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3147 AssertRCReturn(rc, rc);
3148 }
3149
3150 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see hmR0VmxExportGuestExitCtls(). */
3151
3152 return VINF_SUCCESS;
3153}
3154
3155
3156/**
3157 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3158 *
3159 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3160 * these two bits are handled by VM-entry, see hmR0VmxExportGuestExitCtls() and
3161 * hmR0VMxExportGuestEntryCtls().
3162 *
3163 * @returns true if we need to load guest EFER, false otherwise.
3164 * @param pVCpu The cross context virtual CPU structure.
3165 *
3166 * @remarks Requires EFER, CR4.
3167 * @remarks No-long-jump zone!!!
3168 */
3169static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu)
3170{
3171#ifdef HMVMX_ALWAYS_SWAP_EFER
3172 return true;
3173#endif
3174
3175 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3176#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3177 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3178 if (CPUMIsGuestInLongModeEx(pCtx))
3179 return false;
3180#endif
3181
3182 PVM pVM = pVCpu->CTX_SUFF(pVM);
3183 uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3184 uint64_t const u64GuestEfer = pCtx->msrEFER;
3185
3186 /*
3187 * For 64-bit guests, if EFER.SCE bit differs, we need to swap EFER to ensure that the
3188 * guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3189 */
3190 if ( CPUMIsGuestInLongModeEx(pCtx)
3191 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3192 {
3193 return true;
3194 }
3195
3196 /*
3197 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3198 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3199 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3200 */
3201 if ( (pCtx->cr4 & X86_CR4_PAE)
3202 && (pCtx->cr0 & X86_CR0_PG)
3203 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3204 {
3205 /* Assert that host is NX capable. */
3206 Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
3207 return true;
3208 }
3209
3210 return false;
3211}
3212
3213
3214/**
3215 * Exports the guest state with appropriate VM-entry controls in the VMCS.
3216 *
3217 * These controls can affect things done on VM-exit; e.g. "load debug controls",
3218 * see Intel spec. 24.8.1 "VM-entry controls".
3219 *
3220 * @returns VBox status code.
3221 * @param pVCpu The cross context virtual CPU structure.
3222 *
3223 * @remarks Requires EFER.
3224 * @remarks No-long-jump zone!!!
3225 */
3226static int hmR0VmxExportGuestEntryCtls(PVMCPU pVCpu)
3227{
3228 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_CTLS)
3229 {
3230 PVM pVM = pVCpu->CTX_SUFF(pVM);
3231 uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
3232 uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3233
3234 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3235 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3236
3237 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3238 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
3239 {
3240 fVal |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3241 Log4Func(("VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n"));
3242 }
3243 else
3244 Assert(!(fVal & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3245
3246 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3247 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3248 && hmR0VmxShouldSwapEferMsr(pVCpu))
3249 {
3250 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3251 Log4Func(("VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n"));
3252 }
3253
3254 /*
3255 * The following should -not- be set (since we're not in SMM mode):
3256 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3257 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3258 */
3259
3260 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3261 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3262
3263 if ((fVal & fZap) != fVal)
3264 {
3265 Log4Func(("Invalid VM-entry controls combo! Cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3266 pVM->hm.s.vmx.Msrs.EntryCtls.n.disallowed0, fVal, fZap));
3267 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3268 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3269 }
3270
3271 /* Commit it to the VMCS and update our cache. */
3272 if (pVCpu->hm.s.vmx.u32EntryCtls != fVal)
3273 {
3274 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
3275 AssertRCReturn(rc, rc);
3276 pVCpu->hm.s.vmx.u32EntryCtls = fVal;
3277 }
3278
3279 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_CTLS);
3280 }
3281 return VINF_SUCCESS;
3282}
3283
3284
3285/**
3286 * Exports the guest state with appropriate VM-exit controls in the VMCS.
3287 *
3288 * @returns VBox status code.
3289 * @param pVCpu The cross context virtual CPU structure.
3290 *
3291 * @remarks Requires EFER.
3292 */
3293static int hmR0VmxExportGuestExitCtls(PVMCPU pVCpu)
3294{
3295 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_EXIT_CTLS)
3296 {
3297 PVM pVM = pVCpu->CTX_SUFF(pVM);
3298 uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
3299 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3300
3301 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3302 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3303
3304 /*
3305 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3306 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in
3307 * hmR0VmxExportHostMsrs().
3308 */
3309#if HC_ARCH_BITS == 64
3310 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3311 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3312#else
3313 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3314 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3315 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3316 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3317 {
3318 /* The switcher returns to long mode, EFER is managed by the switcher. */
3319 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3320 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3321 }
3322 else
3323 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3324#endif
3325
3326 /* If the newer VMCS fields for managing EFER exists, use it. */
3327 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3328 && hmR0VmxShouldSwapEferMsr(pVCpu))
3329 {
3330 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3331 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3332 Log4Func(("VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR and VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n"));
3333 }
3334
3335 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3336 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3337
3338 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3339 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3340 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3341
3342 /* Enable saving of the VMX preemption timer value on VM-exit. */
3343 if ( pVM->hm.s.vmx.fUsePreemptTimer
3344 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3345 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3346
3347 if ((fVal & fZap) != fVal)
3348 {
3349 LogRelFunc(("Invalid VM-exit controls combo! cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3350 pVM->hm.s.vmx.Msrs.ExitCtls.n.disallowed0, fVal, fZap));
3351 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3352 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3353 }
3354
3355 /* Commit it to the VMCS and update our cache. */
3356 if (pVCpu->hm.s.vmx.u32ExitCtls != fVal)
3357 {
3358 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
3359 AssertRCReturn(rc, rc);
3360 pVCpu->hm.s.vmx.u32ExitCtls = fVal;
3361 }
3362
3363 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_EXIT_CTLS);
3364 }
3365 return VINF_SUCCESS;
3366}
3367
3368
3369/**
3370 * Sets the TPR threshold in the VMCS.
3371 *
3372 * @returns VBox status code.
3373 * @param pVCpu The cross context virtual CPU structure.
3374 * @param u32TprThreshold The TPR threshold (task-priority class only).
3375 */
3376DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3377{
3378 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3379 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3380 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3381}
3382
3383
3384/**
3385 * Exports the guest APIC TPR state into the VMCS.
3386 *
3387 * @returns VBox status code.
3388 * @param pVCpu The cross context virtual CPU structure.
3389 *
3390 * @remarks No-long-jump zone!!!
3391 */
3392static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu)
3393{
3394 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
3395 {
3396 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
3397
3398 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3399 && APICIsEnabled(pVCpu))
3400 {
3401 /*
3402 * Setup TPR shadowing.
3403 */
3404 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3405 {
3406 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3407
3408 bool fPendingIntr = false;
3409 uint8_t u8Tpr = 0;
3410 uint8_t u8PendingIntr = 0;
3411 int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3412 AssertRCReturn(rc, rc);
3413
3414 /*
3415 * If there are interrupts pending but masked by the TPR, instruct VT-x to
3416 * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
3417 * priority of the pending interrupt so we can deliver the interrupt. If there
3418 * are no interrupts pending, set threshold to 0 to not cause any
3419 * TPR-below-threshold VM-exits.
3420 */
3421 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3422 uint32_t u32TprThreshold = 0;
3423 if (fPendingIntr)
3424 {
3425 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3426 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3427 const uint8_t u8TprPriority = u8Tpr >> 4;
3428 if (u8PendingPriority <= u8TprPriority)
3429 u32TprThreshold = u8PendingPriority;
3430 }
3431
3432 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3433 AssertRCReturn(rc, rc);
3434 }
3435 }
3436 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
3437 }
3438 return VINF_SUCCESS;
3439}
3440
3441
3442/**
3443 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3444 *
3445 * @returns Guest's interruptibility-state.
3446 * @param pVCpu The cross context virtual CPU structure.
3447 *
3448 * @remarks No-long-jump zone!!!
3449 */
3450static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu)
3451{
3452 /*
3453 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3454 */
3455 uint32_t fIntrState = 0;
3456 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3457 {
3458 /* If inhibition is active, RIP & RFLAGS should've been accessed
3459 (i.e. read previously from the VMCS or from ring-3). */
3460 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3461#ifdef VBOX_STRICT
3462 uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
3463 AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
3464#endif
3465 if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3466 {
3467 if (pCtx->eflags.Bits.u1IF)
3468 fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
3469 else
3470 fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
3471 }
3472 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3473 {
3474 /*
3475 * We can clear the inhibit force flag as even if we go back to the recompiler
3476 * without executing guest code in VT-x, the flag's condition to be cleared is
3477 * met and thus the cleared state is correct.
3478 */
3479 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3480 }
3481 }
3482
3483 /*
3484 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3485 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3486 * setting this would block host-NMIs and IRET will not clear the blocking.
3487 *
3488 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3489 */
3490 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3491 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3492 {
3493 fIntrState |= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
3494 }
3495
3496 return fIntrState;
3497}
3498
3499
3500/**
3501 * Exports the guest's interruptibility-state into the guest-state area in the
3502 * VMCS.
3503 *
3504 * @returns VBox status code.
3505 * @param pVCpu The cross context virtual CPU structure.
3506 * @param fIntrState The interruptibility-state to set.
3507 */
3508static int hmR0VmxExportGuestIntrState(PVMCPU pVCpu, uint32_t fIntrState)
3509{
3510 NOREF(pVCpu);
3511 AssertMsg(!(fIntrState & 0xfffffff0), ("%#x\n", fIntrState)); /* Bits 31:4 MBZ. */
3512 Assert((fIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3513 return VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
3514}
3515
3516
3517/**
3518 * Exports the exception intercepts required for guest execution in the VMCS.
3519 *
3520 * @returns VBox status code.
3521 * @param pVCpu The cross context virtual CPU structure.
3522 *
3523 * @remarks No-long-jump zone!!!
3524 */
3525static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu)
3526{
3527 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
3528 {
3529 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3530
3531 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxExportSharedCR0(). */
3532 if (pVCpu->hm.s.fGIMTrapXcptUD)
3533 uXcptBitmap |= RT_BIT(X86_XCPT_UD);
3534#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3535 else
3536 uXcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3537#endif
3538
3539 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_AC));
3540 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_DB));
3541
3542 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3543 {
3544 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3545 AssertRCReturn(rc, rc);
3546 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3547 }
3548
3549 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3550 Log4Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64\n", uXcptBitmap));
3551 }
3552 return VINF_SUCCESS;
3553}
3554
3555
3556/**
3557 * Exports the guest's RIP into the guest-state area in the VMCS.
3558 *
3559 * @returns VBox status code.
3560 * @param pVCpu The cross context virtual CPU structure.
3561 *
3562 * @remarks No-long-jump zone!!!
3563 */
3564static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
3565{
3566 int rc = VINF_SUCCESS;
3567 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
3568 {
3569 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
3570
3571 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
3572 AssertRCReturn(rc, rc);
3573
3574 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3575 Log4Func(("RIP=%#RX64\n", pVCpu->cpum.GstCtx.rip));
3576 }
3577 return rc;
3578}
3579
3580
3581/**
3582 * Exports the guest's RSP into the guest-state area in the VMCS.
3583 *
3584 * @returns VBox status code.
3585 * @param pVCpu The cross context virtual CPU structure.
3586 *
3587 * @remarks No-long-jump zone!!!
3588 */
3589static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
3590{
3591 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3592 {
3593 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
3594
3595 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
3596 AssertRCReturn(rc, rc);
3597
3598 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3599 }
3600 return VINF_SUCCESS;
3601}
3602
3603
3604/**
3605 * Exports the guest's RFLAGS into the guest-state area in the VMCS.
3606 *
3607 * @returns VBox status code.
3608 * @param pVCpu The cross context virtual CPU structure.
3609 *
3610 * @remarks No-long-jump zone!!!
3611 */
3612static int hmR0VmxExportGuestRflags(PVMCPU pVCpu)
3613{
3614 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3615 {
3616 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
3617
3618 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3619 Let us assert it as such and use 32-bit VMWRITE. */
3620 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
3621 X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
3622 Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3623 Assert(!(fEFlags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3624
3625 /*
3626 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3627 * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3628 * can run the real-mode guest code under Virtual 8086 mode.
3629 */
3630 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3631 {
3632 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3633 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3634 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3635 fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3636 fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3637 }
3638
3639 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3640 AssertRCReturn(rc, rc);
3641
3642 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3643 Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
3644 }
3645 return VINF_SUCCESS;
3646}
3647
3648
3649/**
3650 * Exports the guest CR0 control register into the guest-state area in the VMCS.
3651 *
3652 * The guest FPU state is always pre-loaded hence we don't need to bother about
3653 * sharing FPU related CR0 bits between the guest and host.
3654 *
3655 * @returns VBox status code.
3656 * @param pVCpu The cross context virtual CPU structure.
3657 *
3658 * @remarks No-long-jump zone!!!
3659 */
3660static int hmR0VmxExportGuestCR0(PVMCPU pVCpu)
3661{
3662 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
3663 {
3664 PVM pVM = pVCpu->CTX_SUFF(pVM);
3665 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
3666 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.cr0));
3667
3668 uint32_t const u32ShadowCr0 = pVCpu->cpum.GstCtx.cr0;
3669 uint32_t u32GuestCr0 = pVCpu->cpum.GstCtx.cr0;
3670
3671 /*
3672 * Setup VT-x's view of the guest CR0.
3673 * Minimize VM-exits due to CR3 changes when we have NestedPaging.
3674 */
3675 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
3676 if (pVM->hm.s.fNestedPaging)
3677 {
3678 if (CPUMIsGuestPagingEnabled(pVCpu))
3679 {
3680 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3681 uProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3682 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3683 }
3684 else
3685 {
3686 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3687 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3688 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3689 }
3690
3691 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3692 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3693 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3694 }
3695 else
3696 {
3697 /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3698 u32GuestCr0 |= X86_CR0_WP;
3699 }
3700
3701 /*
3702 * Guest FPU bits.
3703 *
3704 * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3705 * using CR0.TS.
3706 *
3707 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3708 * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3709 */
3710 u32GuestCr0 |= X86_CR0_NE;
3711
3712 /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3713 bool const fInterceptMF = !(u32ShadowCr0 & X86_CR0_NE);
3714
3715 /*
3716 * Update exception intercepts.
3717 */
3718 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3719 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3720 {
3721 Assert(PDMVmmDevHeapIsEnabled(pVM));
3722 Assert(pVM->hm.s.vmx.pRealModeTSS);
3723 uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3724 }
3725 else
3726 {
3727 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3728 uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3729 if (fInterceptMF)
3730 uXcptBitmap |= RT_BIT(X86_XCPT_MF);
3731 }
3732
3733 /* Additional intercepts for debugging, define these yourself explicitly. */
3734#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3735 uXcptBitmap |= 0
3736 | RT_BIT(X86_XCPT_BP)
3737 | RT_BIT(X86_XCPT_DE)
3738 | RT_BIT(X86_XCPT_NM)
3739 | RT_BIT(X86_XCPT_TS)
3740 | RT_BIT(X86_XCPT_UD)
3741 | RT_BIT(X86_XCPT_NP)
3742 | RT_BIT(X86_XCPT_SS)
3743 | RT_BIT(X86_XCPT_GP)
3744 | RT_BIT(X86_XCPT_PF)
3745 | RT_BIT(X86_XCPT_MF)
3746 ;
3747#elif defined(HMVMX_ALWAYS_TRAP_PF)
3748 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
3749#endif
3750 Assert(pVM->hm.s.fNestedPaging || (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
3751
3752 /*
3753 * Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW).
3754 */
3755 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3756 uint32_t fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3757 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3758 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
3759 else
3760 Assert((fSetCr0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3761
3762 u32GuestCr0 |= fSetCr0;
3763 u32GuestCr0 &= fZapCr0;
3764 u32GuestCr0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3765
3766 /*
3767 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3768 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3769 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3770 */
3771 uint32_t u32Cr0Mask = X86_CR0_PE
3772 | X86_CR0_NE
3773 | (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
3774 | X86_CR0_PG
3775 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3776 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3777 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3778
3779 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3780 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3781 * and @bugref{6944}. */
3782#if 0
3783 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3784 u32Cr0Mask &= ~X86_CR0_PE;
3785#endif
3786 /*
3787 * Finally, update VMCS fields with the CR0 values and the exception bitmap.
3788 */
3789 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCr0);
3790 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32ShadowCr0);
3791 if (u32Cr0Mask != pVCpu->hm.s.vmx.u32Cr0Mask)
3792 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32Cr0Mask);
3793 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
3794 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3795 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3796 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3797 AssertRCReturn(rc, rc);
3798
3799 /* Update our caches. */
3800 pVCpu->hm.s.vmx.u32Cr0Mask = u32Cr0Mask;
3801 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
3802 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3803
3804 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3805
3806 Log4Func(("u32Cr0Mask=%#RX32 u32ShadowCr0=%#RX32 u32GuestCr0=%#RX32 (fSetCr0=%#RX32 fZapCr0=%#RX32\n", u32Cr0Mask,
3807 u32ShadowCr0, u32GuestCr0, fSetCr0, fZapCr0));
3808 }
3809
3810 return VINF_SUCCESS;
3811}
3812
3813
3814/**
3815 * Exports the guest control registers (CR3, CR4) into the guest-state area
3816 * in the VMCS.
3817 *
3818 * @returns VBox strict status code.
3819 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3820 * without unrestricted guest access and the VMMDev is not presently
3821 * mapped (e.g. EFI32).
3822 *
3823 * @param pVCpu The cross context virtual CPU structure.
3824 *
3825 * @remarks No-long-jump zone!!!
3826 */
3827static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu)
3828{
3829 int rc = VINF_SUCCESS;
3830 PVM pVM = pVCpu->CTX_SUFF(pVM);
3831
3832 /*
3833 * Guest CR2.
3834 * It's always loaded in the assembler code. Nothing to do here.
3835 */
3836
3837 /*
3838 * Guest CR3.
3839 */
3840 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
3841 {
3842 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
3843
3844 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3845 if (pVM->hm.s.fNestedPaging)
3846 {
3847 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3848
3849 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3850 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3851 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3852 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3853
3854 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3855 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3856 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3857
3858 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3859 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3860 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3861 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3862 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3863 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3864 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3865
3866 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3867 AssertRCReturn(rc, rc);
3868
3869 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3870 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3871 || CPUMIsGuestPagingEnabledEx(pCtx))
3872 {
3873 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3874 if (CPUMIsGuestInPAEModeEx(pCtx))
3875 {
3876 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3877 AssertRCReturn(rc, rc);
3878 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3879 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3880 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3881 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3882 AssertRCReturn(rc, rc);
3883 }
3884
3885 /*
3886 * The guest's view of its CR3 is unblemished with Nested Paging when the
3887 * guest is using paging or we have unrestricted guest execution to handle
3888 * the guest when it's not using paging.
3889 */
3890 GCPhysGuestCR3 = pCtx->cr3;
3891 }
3892 else
3893 {
3894 /*
3895 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
3896 * thinks it accesses physical memory directly, we use our identity-mapped
3897 * page table to map guest-linear to guest-physical addresses. EPT takes care
3898 * of translating it to host-physical addresses.
3899 */
3900 RTGCPHYS GCPhys;
3901 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3902
3903 /* We obtain it here every time as the guest could have relocated this PCI region. */
3904 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3905 if (RT_SUCCESS(rc))
3906 { /* likely */ }
3907 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3908 {
3909 Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3910 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3911 }
3912 else
3913 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3914
3915 GCPhysGuestCR3 = GCPhys;
3916 }
3917
3918 Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
3919 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3920 AssertRCReturn(rc, rc);
3921 }
3922 else
3923 {
3924 /* Non-nested paging case, just use the hypervisor's CR3. */
3925 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3926
3927 Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
3928 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3929 AssertRCReturn(rc, rc);
3930 }
3931
3932 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3933 }
3934
3935 /*
3936 * Guest CR4.
3937 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3938 */
3939 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
3940 {
3941 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3942 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
3943 Assert(!RT_HI_U32(pCtx->cr4));
3944
3945 uint32_t u32GuestCr4 = pCtx->cr4;
3946 uint32_t const u32ShadowCr4 = pCtx->cr4;
3947
3948 /*
3949 * Setup VT-x's view of the guest CR4.
3950 *
3951 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3952 * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3953 * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3954 *
3955 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3956 */
3957 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3958 {
3959 Assert(pVM->hm.s.vmx.pRealModeTSS);
3960 Assert(PDMVmmDevHeapIsEnabled(pVM));
3961 u32GuestCr4 &= ~X86_CR4_VME;
3962 }
3963
3964 if (pVM->hm.s.fNestedPaging)
3965 {
3966 if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3967 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3968 {
3969 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3970 u32GuestCr4 |= X86_CR4_PSE;
3971 /* Our identity mapping is a 32-bit page directory. */
3972 u32GuestCr4 &= ~X86_CR4_PAE;
3973 }
3974 /* else use guest CR4.*/
3975 }
3976 else
3977 {
3978 /*
3979 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3980 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3981 */
3982 switch (pVCpu->hm.s.enmShadowMode)
3983 {
3984 case PGMMODE_REAL: /* Real-mode. */
3985 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3986 case PGMMODE_32_BIT: /* 32-bit paging. */
3987 {
3988 u32GuestCr4 &= ~X86_CR4_PAE;
3989 break;
3990 }
3991
3992 case PGMMODE_PAE: /* PAE paging. */
3993 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3994 {
3995 u32GuestCr4 |= X86_CR4_PAE;
3996 break;
3997 }
3998
3999 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4000 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4001#ifdef VBOX_ENABLE_64_BITS_GUESTS
4002 break;
4003#endif
4004 default:
4005 AssertFailed();
4006 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4007 }
4008 }
4009
4010 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4011 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4012 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4013 u32GuestCr4 |= fSetCr4;
4014 u32GuestCr4 &= fZapCr4;
4015
4016 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them,
4017 that would cause a VM-exit. */
4018 uint32_t u32Cr4Mask = X86_CR4_VME
4019 | X86_CR4_PAE
4020 | X86_CR4_PGE
4021 | X86_CR4_PSE
4022 | X86_CR4_VMXE;
4023 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4024 u32Cr4Mask |= X86_CR4_OSXSAVE;
4025 if (pVM->cpum.ro.GuestFeatures.fPcid)
4026 u32Cr4Mask |= X86_CR4_PCIDE;
4027
4028 /* Write VT-x's view of the guest CR4, the CR4 modify mask and the read-only CR4 shadow
4029 into the VMCS and update our cache. */
4030 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCr4);
4031 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32ShadowCr4);
4032 if (pVCpu->hm.s.vmx.u32Cr4Mask != u32Cr4Mask)
4033 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32Cr4Mask);
4034 AssertRCReturn(rc, rc);
4035 pVCpu->hm.s.vmx.u32Cr4Mask = u32Cr4Mask;
4036
4037 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4038 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
4039
4040 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
4041
4042 Log4Func(("u32GuestCr4=%#RX32 u32ShadowCr4=%#RX32 (fSetCr4=%#RX32 fZapCr4=%#RX32)\n", u32GuestCr4, u32ShadowCr4, fSetCr4,
4043 fZapCr4));
4044 }
4045 return rc;
4046}
4047
4048
4049/**
4050 * Exports the guest debug registers into the guest-state area in the VMCS.
4051 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4052 *
4053 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4054 *
4055 * @returns VBox status code.
4056 * @param pVCpu The cross context virtual CPU structure.
4057 *
4058 * @remarks No-long-jump zone!!!
4059 */
4060static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu)
4061{
4062 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4063
4064#ifdef VBOX_STRICT
4065 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4066 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4067 {
4068 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4069 Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0);
4070 Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
4071 }
4072#endif
4073
4074 bool fSteppingDB = false;
4075 bool fInterceptMovDRx = false;
4076 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
4077 if (pVCpu->hm.s.fSingleInstruction)
4078 {
4079 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4080 PVM pVM = pVCpu->CTX_SUFF(pVM);
4081 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4082 {
4083 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4084 Assert(fSteppingDB == false);
4085 }
4086 else
4087 {
4088 pVCpu->cpum.GstCtx.eflags.u32 |= X86_EFL_TF;
4089 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_RFLAGS;
4090 pVCpu->hm.s.fClearTrapFlag = true;
4091 fSteppingDB = true;
4092 }
4093 }
4094
4095 uint32_t u32GuestDr7;
4096 if ( fSteppingDB
4097 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4098 {
4099 /*
4100 * Use the combined guest and host DRx values found in the hypervisor register set
4101 * because the debugger has breakpoints active or someone is single stepping on the
4102 * host side without a monitor trap flag.
4103 *
4104 * Note! DBGF expects a clean DR6 state before executing guest code.
4105 */
4106#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4107 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4108 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4109 {
4110 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4111 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4112 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4113 }
4114 else
4115#endif
4116 if (!CPUMIsHyperDebugStateActive(pVCpu))
4117 {
4118 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4119 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4120 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4121 }
4122
4123 /* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
4124 u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
4125 pVCpu->hm.s.fUsingHyperDR7 = true;
4126 fInterceptMovDRx = true;
4127 }
4128 else
4129 {
4130 /*
4131 * If the guest has enabled debug registers, we need to load them prior to
4132 * executing guest code so they'll trigger at the right time.
4133 */
4134 if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD))
4135 {
4136#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4137 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4138 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4139 {
4140 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4141 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4142 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4143 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4144 }
4145 else
4146#endif
4147 if (!CPUMIsGuestDebugStateActive(pVCpu))
4148 {
4149 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4150 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4151 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4152 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4153 }
4154 Assert(!fInterceptMovDRx);
4155 }
4156 /*
4157 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4158 * must intercept #DB in order to maintain a correct DR6 guest value, and
4159 * because we need to intercept it to prevent nested #DBs from hanging the
4160 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4161 */
4162#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4163 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4164 && !CPUMIsGuestDebugStateActive(pVCpu))
4165#else
4166 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4167#endif
4168 {
4169 fInterceptMovDRx = true;
4170 }
4171
4172 /* Update DR7 with the actual guest value. */
4173 u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
4174 pVCpu->hm.s.fUsingHyperDR7 = false;
4175 }
4176
4177 if (fInterceptMovDRx)
4178 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4179 else
4180 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4181
4182 /*
4183 * Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
4184 * monitor-trap flag and update our cache.
4185 */
4186 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
4187 {
4188 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4189 AssertRCReturn(rc2, rc2);
4190 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
4191 }
4192
4193 /*
4194 * Update guest DR7.
4195 */
4196 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
4197 AssertRCReturn(rc, rc);
4198
4199 return VINF_SUCCESS;
4200}
4201
4202
4203#ifdef VBOX_STRICT
4204/**
4205 * Strict function to validate segment registers.
4206 *
4207 * @param pVCpu The cross context virtual CPU structure.
4208 *
4209 * @remarks Will import guest CR0 on strict builds during validation of
4210 * segments.
4211 */
4212static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu)
4213{
4214 /*
4215 * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
4216 *
4217 * The reason we check for attribute value 0 in this function and not just the unusable bit is
4218 * because hmR0VmxExportGuestSegmentReg() only updates the VMCS' copy of the value with the unusable bit
4219 * and doesn't change the guest-context value.
4220 */
4221 PVM pVM = pVCpu->CTX_SUFF(pVM);
4222 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4223 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
4224 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4225 && ( !CPUMIsGuestInRealModeEx(pCtx)
4226 && !CPUMIsGuestInV86ModeEx(pCtx)))
4227 {
4228 /* Protected mode checks */
4229 /* CS */
4230 Assert(pCtx->cs.Attr.n.u1Present);
4231 Assert(!(pCtx->cs.Attr.u & 0xf00));
4232 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4233 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4234 || !(pCtx->cs.Attr.n.u1Granularity));
4235 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4236 || (pCtx->cs.Attr.n.u1Granularity));
4237 /* CS cannot be loaded with NULL in protected mode. */
4238 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4239 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4240 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4241 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4242 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4243 else
4244 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4245 /* SS */
4246 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4247 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4248 if ( !(pCtx->cr0 & X86_CR0_PE)
4249 || pCtx->cs.Attr.n.u4Type == 3)
4250 {
4251 Assert(!pCtx->ss.Attr.n.u2Dpl);
4252 }
4253 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4254 {
4255 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4256 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4257 Assert(pCtx->ss.Attr.n.u1Present);
4258 Assert(!(pCtx->ss.Attr.u & 0xf00));
4259 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4260 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4261 || !(pCtx->ss.Attr.n.u1Granularity));
4262 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4263 || (pCtx->ss.Attr.n.u1Granularity));
4264 }
4265 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmentReg(). */
4266 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4267 {
4268 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4269 Assert(pCtx->ds.Attr.n.u1Present);
4270 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4271 Assert(!(pCtx->ds.Attr.u & 0xf00));
4272 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4273 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4274 || !(pCtx->ds.Attr.n.u1Granularity));
4275 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4276 || (pCtx->ds.Attr.n.u1Granularity));
4277 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4278 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4279 }
4280 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4281 {
4282 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4283 Assert(pCtx->es.Attr.n.u1Present);
4284 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4285 Assert(!(pCtx->es.Attr.u & 0xf00));
4286 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4287 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4288 || !(pCtx->es.Attr.n.u1Granularity));
4289 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4290 || (pCtx->es.Attr.n.u1Granularity));
4291 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4292 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4293 }
4294 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4295 {
4296 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4297 Assert(pCtx->fs.Attr.n.u1Present);
4298 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4299 Assert(!(pCtx->fs.Attr.u & 0xf00));
4300 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4301 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4302 || !(pCtx->fs.Attr.n.u1Granularity));
4303 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4304 || (pCtx->fs.Attr.n.u1Granularity));
4305 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4306 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4307 }
4308 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4309 {
4310 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4311 Assert(pCtx->gs.Attr.n.u1Present);
4312 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4313 Assert(!(pCtx->gs.Attr.u & 0xf00));
4314 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4315 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4316 || !(pCtx->gs.Attr.n.u1Granularity));
4317 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4318 || (pCtx->gs.Attr.n.u1Granularity));
4319 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4320 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4321 }
4322 /* 64-bit capable CPUs. */
4323# if HC_ARCH_BITS == 64
4324 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4325 Assert(!pCtx->ss.Attr.u || !RT_HI_U32(pCtx->ss.u64Base));
4326 Assert(!pCtx->ds.Attr.u || !RT_HI_U32(pCtx->ds.u64Base));
4327 Assert(!pCtx->es.Attr.u || !RT_HI_U32(pCtx->es.u64Base));
4328# endif
4329 }
4330 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4331 || ( CPUMIsGuestInRealModeEx(pCtx)
4332 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4333 {
4334 /* Real and v86 mode checks. */
4335 /* hmR0VmxExportGuestSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4336 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4337 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4338 {
4339 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4340 }
4341 else
4342 {
4343 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4344 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4345 }
4346
4347 /* CS */
4348 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4349 Assert(pCtx->cs.u32Limit == 0xffff);
4350 Assert(u32CSAttr == 0xf3);
4351 /* SS */
4352 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4353 Assert(pCtx->ss.u32Limit == 0xffff);
4354 Assert(u32SSAttr == 0xf3);
4355 /* DS */
4356 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4357 Assert(pCtx->ds.u32Limit == 0xffff);
4358 Assert(u32DSAttr == 0xf3);
4359 /* ES */
4360 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4361 Assert(pCtx->es.u32Limit == 0xffff);
4362 Assert(u32ESAttr == 0xf3);
4363 /* FS */
4364 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4365 Assert(pCtx->fs.u32Limit == 0xffff);
4366 Assert(u32FSAttr == 0xf3);
4367 /* GS */
4368 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4369 Assert(pCtx->gs.u32Limit == 0xffff);
4370 Assert(u32GSAttr == 0xf3);
4371 /* 64-bit capable CPUs. */
4372# if HC_ARCH_BITS == 64
4373 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4374 Assert(!u32SSAttr || !RT_HI_U32(pCtx->ss.u64Base));
4375 Assert(!u32DSAttr || !RT_HI_U32(pCtx->ds.u64Base));
4376 Assert(!u32ESAttr || !RT_HI_U32(pCtx->es.u64Base));
4377# endif
4378 }
4379}
4380#endif /* VBOX_STRICT */
4381
4382
4383/**
4384 * Exports a guest segment register into the guest-state area in the VMCS.
4385 *
4386 * @returns VBox status code.
4387 * @param pVCpu The cross context virtual CPU structure.
4388 * @param idxSel Index of the selector in the VMCS.
4389 * @param idxLimit Index of the segment limit in the VMCS.
4390 * @param idxBase Index of the segment base in the VMCS.
4391 * @param idxAccess Index of the access rights of the segment in the VMCS.
4392 * @param pSelReg Pointer to the segment selector.
4393 *
4394 * @remarks No-long-jump zone!!!
4395 */
4396static int hmR0VmxExportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
4397 PCCPUMSELREG pSelReg)
4398{
4399 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4400 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4401 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4402 AssertRCReturn(rc, rc);
4403
4404 uint32_t u32Access = pSelReg->Attr.u;
4405 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4406 {
4407 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4408 u32Access = 0xf3;
4409 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4410 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4411 }
4412 else
4413 {
4414 /*
4415 * The way to differentiate between whether this is really a null selector or was just
4416 * a selector loaded with 0 in real-mode is using the segment attributes. A selector
4417 * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4418 * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4419 * NULL selectors loaded in protected-mode have their attribute as 0.
4420 */
4421 if (!u32Access)
4422 u32Access = X86DESCATTR_UNUSABLE;
4423 }
4424
4425 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4426 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4427 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4428
4429 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4430 AssertRCReturn(rc, rc);
4431 return rc;
4432}
4433
4434
4435/**
4436 * Exports the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4437 * into the guest-state area in the VMCS.
4438 *
4439 * @returns VBox status code.
4440 * @param pVCpu The cross context virtual CPU structure.
4441 *
4442 * @remarks Will import guest CR0 on strict builds during validation of
4443 * segments.
4444 * @remarks No-long-jump zone!!!
4445 */
4446static int hmR0VmxExportGuestSegmentRegs(PVMCPU pVCpu)
4447{
4448 int rc = VERR_INTERNAL_ERROR_5;
4449 PVM pVM = pVCpu->CTX_SUFF(pVM);
4450 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4451
4452 /*
4453 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4454 */
4455 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4456 {
4457#ifdef VBOX_WITH_REM
4458 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4459 {
4460 Assert(pVM->hm.s.vmx.pRealModeTSS);
4461 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4462 if ( pVCpu->hm.s.vmx.fWasInRealMode
4463 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4464 {
4465 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4466 in real-mode (e.g. OpenBSD 4.0) */
4467 REMFlushTBs(pVM);
4468 Log4Func(("Switch to protected mode detected!\n"));
4469 pVCpu->hm.s.vmx.fWasInRealMode = false;
4470 }
4471 }
4472#endif
4473 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
4474 {
4475 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
4476 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4477 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pCtx->cs.Attr.u;
4478 rc = HMVMX_EXPORT_SREG(CS, &pCtx->cs);
4479 AssertRCReturn(rc, rc);
4480 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
4481 }
4482
4483 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
4484 {
4485 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
4486 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4487 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pCtx->ss.Attr.u;
4488 rc = HMVMX_EXPORT_SREG(SS, &pCtx->ss);
4489 AssertRCReturn(rc, rc);
4490 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
4491 }
4492
4493 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
4494 {
4495 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
4496 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4497 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pCtx->ds.Attr.u;
4498 rc = HMVMX_EXPORT_SREG(DS, &pCtx->ds);
4499 AssertRCReturn(rc, rc);
4500 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
4501 }
4502
4503 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
4504 {
4505 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
4506 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4507 pVCpu->hm.s.vmx.RealMode.AttrES.u = pCtx->es.Attr.u;
4508 rc = HMVMX_EXPORT_SREG(ES, &pCtx->es);
4509 AssertRCReturn(rc, rc);
4510 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
4511 }
4512
4513 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
4514 {
4515 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
4516 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4517 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pCtx->fs.Attr.u;
4518 rc = HMVMX_EXPORT_SREG(FS, &pCtx->fs);
4519 AssertRCReturn(rc, rc);
4520 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
4521 }
4522
4523 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
4524 {
4525 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
4526 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4527 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pCtx->gs.Attr.u;
4528 rc = HMVMX_EXPORT_SREG(GS, &pCtx->gs);
4529 AssertRCReturn(rc, rc);
4530 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
4531 }
4532
4533#ifdef VBOX_STRICT
4534 hmR0VmxValidateSegmentRegs(pVCpu);
4535#endif
4536
4537 Log4Func(("CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pCtx->cs.Sel, pCtx->cs.u64Base,
4538 pCtx->cs.u32Limit, pCtx->cs.Attr.u));
4539 }
4540
4541 /*
4542 * Guest TR.
4543 */
4544 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
4545 {
4546 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
4547
4548 /*
4549 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
4550 * achieved using the interrupt redirection bitmap (all bits cleared to let the guest
4551 * handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4552 */
4553 uint16_t u16Sel = 0;
4554 uint32_t u32Limit = 0;
4555 uint64_t u64Base = 0;
4556 uint32_t u32AccessRights = 0;
4557
4558 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4559 {
4560 u16Sel = pCtx->tr.Sel;
4561 u32Limit = pCtx->tr.u32Limit;
4562 u64Base = pCtx->tr.u64Base;
4563 u32AccessRights = pCtx->tr.Attr.u;
4564 }
4565 else
4566 {
4567 Assert(pVM->hm.s.vmx.pRealModeTSS);
4568 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4569
4570 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4571 RTGCPHYS GCPhys;
4572 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4573 AssertRCReturn(rc, rc);
4574
4575 X86DESCATTR DescAttr;
4576 DescAttr.u = 0;
4577 DescAttr.n.u1Present = 1;
4578 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4579
4580 u16Sel = 0;
4581 u32Limit = HM_VTX_TSS_SIZE;
4582 u64Base = GCPhys; /* in real-mode phys = virt. */
4583 u32AccessRights = DescAttr.u;
4584 }
4585
4586 /* Validate. */
4587 Assert(!(u16Sel & RT_BIT(2)));
4588 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4589 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4590 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4591 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4592 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4593 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4594 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4595 Assert( (u32Limit & 0xfff) == 0xfff
4596 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4597 Assert( !(pCtx->tr.u32Limit & 0xfff00000)
4598 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4599
4600 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4601 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4602 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4603 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4604 AssertRCReturn(rc, rc);
4605
4606 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
4607 Log4Func(("TR base=%#RX64\n", pCtx->tr.u64Base));
4608 }
4609
4610 /*
4611 * Guest GDTR.
4612 */
4613 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4614 {
4615 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
4616
4617 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
4618 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
4619 AssertRCReturn(rc, rc);
4620
4621 /* Validate. */
4622 Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4623
4624 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4625 Log4Func(("GDTR base=%#RX64\n", pCtx->gdtr.pGdt));
4626 }
4627
4628 /*
4629 * Guest LDTR.
4630 */
4631 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4632 {
4633 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
4634
4635 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4636 uint32_t u32Access = 0;
4637 if (!pCtx->ldtr.Attr.u)
4638 u32Access = X86DESCATTR_UNUSABLE;
4639 else
4640 u32Access = pCtx->ldtr.Attr.u;
4641
4642 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
4643 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
4644 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4645 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
4646 AssertRCReturn(rc, rc);
4647
4648 /* Validate. */
4649 if (!(u32Access & X86DESCATTR_UNUSABLE))
4650 {
4651 Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4652 Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4653 Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4654 Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4655 Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4656 Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4657 Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
4658 || !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4659 Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
4660 || pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4661 }
4662
4663 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4664 Log4Func(("LDTR base=%#RX64\n", pCtx->ldtr.u64Base));
4665 }
4666
4667 /*
4668 * Guest IDTR.
4669 */
4670 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4671 {
4672 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
4673
4674 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
4675 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
4676 AssertRCReturn(rc, rc);
4677
4678 /* Validate. */
4679 Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4680
4681 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4682 Log4Func(("IDTR base=%#RX64\n", pCtx->idtr.pIdt));
4683 }
4684
4685 return VINF_SUCCESS;
4686}
4687
4688
4689/**
4690 * Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4691 * areas.
4692 *
4693 * These MSRs will automatically be loaded to the host CPU on every successful
4694 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4695 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4696 * -not- updated here for performance reasons. See hmR0VmxExportHostMsrs().
4697 *
4698 * Also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4699 *
4700 * @returns VBox status code.
4701 * @param pVCpu The cross context virtual CPU structure.
4702 *
4703 * @remarks No-long-jump zone!!!
4704 */
4705static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu)
4706{
4707 AssertPtr(pVCpu);
4708 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4709
4710 /*
4711 * MSRs that we use the auto-load/store MSR area in the VMCS.
4712 * For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
4713 */
4714 PVM pVM = pVCpu->CTX_SUFF(pVM);
4715 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4716 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4717 {
4718 if (pVM->hm.s.fAllow64BitGuests)
4719 {
4720#if HC_ARCH_BITS == 32
4721 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS | CPUMCTX_EXTRN_KERNEL_GS_BASE);
4722
4723 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pCtx->msrLSTAR, false, NULL);
4724 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pCtx->msrSTAR, false, NULL);
4725 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pCtx->msrSFMASK, false, NULL);
4726 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, false, NULL);
4727 AssertRCReturn(rc, rc);
4728# ifdef LOG_ENABLED
4729 PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4730 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4731 Log4Func(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4732# endif
4733#endif
4734 }
4735 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4736 }
4737
4738 /*
4739 * Guest Sysenter MSRs.
4740 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4741 * VM-exits on WRMSRs for these MSRs.
4742 */
4743 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4744 {
4745 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
4746
4747 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4748 {
4749 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
4750 AssertRCReturn(rc, rc);
4751 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4752 }
4753
4754 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4755 {
4756 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
4757 AssertRCReturn(rc, rc);
4758 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4759 }
4760
4761 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4762 {
4763 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
4764 AssertRCReturn(rc, rc);
4765 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4766 }
4767 }
4768
4769 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4770 {
4771 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
4772
4773 if (hmR0VmxShouldSwapEferMsr(pVCpu))
4774 {
4775 /*
4776 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4777 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4778 */
4779 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4780 {
4781 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
4782 AssertRCReturn(rc,rc);
4783 Log4Func(("EFER=%#RX64\n", pCtx->msrEFER));
4784 }
4785 else
4786 {
4787 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pCtx->msrEFER, false /* fUpdateHostMsr */,
4788 NULL /* pfAddedAndUpdated */);
4789 AssertRCReturn(rc, rc);
4790
4791 /* We need to intercept reads too, see @bugref{7386#c16}. */
4792 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4793 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4794 Log4Func(("MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pCtx->msrEFER,
4795 pVCpu->hm.s.vmx.cMsrs));
4796 }
4797 }
4798 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4799 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4800 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4801 }
4802
4803 return VINF_SUCCESS;
4804}
4805
4806
4807#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4808/**
4809 * Check if guest state allows safe use of 32-bit switcher again.
4810 *
4811 * Segment bases and protected mode structures must be 32-bit addressable
4812 * because the 32-bit switcher will ignore high dword when writing these VMCS
4813 * fields. See @bugref{8432} for details.
4814 *
4815 * @returns true if safe, false if must continue to use the 64-bit switcher.
4816 * @param pCtx Pointer to the guest-CPU context.
4817 *
4818 * @remarks No-long-jump zone!!!
4819 */
4820static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
4821{
4822 if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4823 if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4824 if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4825 if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4826 if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4827 if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4828 if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4829 if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4830 if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4831 if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4832
4833 /* All good, bases are 32-bit. */
4834 return true;
4835}
4836#endif
4837
4838
4839/**
4840 * Selects up the appropriate function to run guest code.
4841 *
4842 * @returns VBox status code.
4843 * @param pVCpu The cross context virtual CPU structure.
4844 *
4845 * @remarks No-long-jump zone!!!
4846 */
4847static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu)
4848{
4849 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4850 if (CPUMIsGuestInLongModeEx(pCtx))
4851 {
4852#ifndef VBOX_ENABLE_64_BITS_GUESTS
4853 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4854#endif
4855 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4856#if HC_ARCH_BITS == 32
4857 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4858 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4859 {
4860#ifdef VBOX_STRICT
4861 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4862 {
4863 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4864 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4865 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4866 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4867 | HM_CHANGED_VMX_ENTRY_CTLS
4868 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4869 }
4870#endif
4871 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4872
4873 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4874 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4875 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4876 Log4Func(("Selected 64-bit switcher\n"));
4877 }
4878#else
4879 /* 64-bit host. */
4880 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4881#endif
4882 }
4883 else
4884 {
4885 /* Guest is not in long mode, use the 32-bit handler. */
4886#if HC_ARCH_BITS == 32
4887 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4888 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4889 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4890 {
4891# ifdef VBOX_STRICT
4892 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4893 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4894 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4895 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4896 | HM_CHANGED_VMX_ENTRY_CTLS
4897 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4898# endif
4899 }
4900# ifdef VBOX_ENABLE_64_BITS_GUESTS
4901 /*
4902 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
4903 * design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
4904 * switcher flag because now we know the guest is in a sane state where it's safe
4905 * to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4906 * the much faster 32-bit switcher again.
4907 */
4908 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4909 {
4910 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4911 Log4Func(("Selected 32-bit switcher\n"));
4912 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4913 }
4914 else
4915 {
4916 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4917 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4918 || hmR0VmxIs32BitSwitcherSafe(pCtx))
4919 {
4920 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4921 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4922 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
4923 | HM_CHANGED_VMX_ENTRY_CTLS
4924 | HM_CHANGED_VMX_EXIT_CTLS
4925 | HM_CHANGED_HOST_CONTEXT);
4926 Log4Func(("Selected 32-bit switcher (safe)\n"));
4927 }
4928 }
4929# else
4930 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4931# endif
4932#else
4933 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4934#endif
4935 }
4936 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4937 return VINF_SUCCESS;
4938}
4939
4940
4941/**
4942 * Wrapper for running the guest code in VT-x.
4943 *
4944 * @returns VBox status code, no informational status codes.
4945 * @param pVCpu The cross context virtual CPU structure.
4946 *
4947 * @remarks No-long-jump zone!!!
4948 */
4949DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu)
4950{
4951 /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4952 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4953 pCtx->fExtrn |= HMVMX_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
4954
4955 /*
4956 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4957 * floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
4958 * callee-saved and thus the need for this XMM wrapper.
4959 *
4960 * See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
4961 */
4962 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4963 /** @todo Add stats for resume vs launch. */
4964 PVM pVM = pVCpu->CTX_SUFF(pVM);
4965#ifdef VBOX_WITH_KERNEL_USING_XMM
4966 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4967#else
4968 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4969#endif
4970 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4971 return rc;
4972}
4973
4974
4975/**
4976 * Reports world-switch error and dumps some useful debug info.
4977 *
4978 * @param pVCpu The cross context virtual CPU structure.
4979 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4980 * @param pVmxTransient Pointer to the VMX transient structure (only
4981 * exitReason updated).
4982 */
4983static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
4984{
4985 Assert(pVCpu);
4986 Assert(pVmxTransient);
4987 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
4988
4989 Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
4990 switch (rcVMRun)
4991 {
4992 case VERR_VMX_INVALID_VMXON_PTR:
4993 AssertFailed();
4994 break;
4995 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4996 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4997 {
4998 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4999 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5000 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5001 AssertRC(rc);
5002
5003 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5004 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5005 Cannot do it here as we may have been long preempted. */
5006
5007#ifdef VBOX_STRICT
5008 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5009 pVmxTransient->uExitReason));
5010 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5011 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5012 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5013 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5014 else
5015 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5016 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5017 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5018
5019 /* VMX control bits. */
5020 uint32_t u32Val;
5021 uint64_t u64Val;
5022 RTHCUINTREG uHCReg;
5023 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5024 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5025 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5026 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5027 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5028 {
5029 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5030 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5031 }
5032 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5033 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5034 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5035 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5036 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5037 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5038 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5039 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5040 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5041 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5042 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5043 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5044 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5045 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5046 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5047 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5048 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5049 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5050 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5051 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5052 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5053 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5054 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5055 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5056 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5057 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5058 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5059 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5060 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5061 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5062 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5063 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5064 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5065 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5066 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5067 {
5068 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5069 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5070 }
5071
5072 /* Guest bits. */
5073 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5074 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
5075 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5076 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
5077 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5078 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
5079 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
5080 {
5081 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5082 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5083 }
5084
5085 /* Host bits. */
5086 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5087 Log4(("Host CR0 %#RHr\n", uHCReg));
5088 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5089 Log4(("Host CR3 %#RHr\n", uHCReg));
5090 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5091 Log4(("Host CR4 %#RHr\n", uHCReg));
5092
5093 RTGDTR HostGdtr;
5094 PCX86DESCHC pDesc;
5095 ASMGetGDTR(&HostGdtr);
5096 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5097 Log4(("Host CS %#08x\n", u32Val));
5098 if (u32Val < HostGdtr.cbGdt)
5099 {
5100 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5101 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5102 }
5103
5104 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5105 Log4(("Host DS %#08x\n", u32Val));
5106 if (u32Val < HostGdtr.cbGdt)
5107 {
5108 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5109 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5110 }
5111
5112 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5113 Log4(("Host ES %#08x\n", u32Val));
5114 if (u32Val < HostGdtr.cbGdt)
5115 {
5116 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5117 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5118 }
5119
5120 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5121 Log4(("Host FS %#08x\n", u32Val));
5122 if (u32Val < HostGdtr.cbGdt)
5123 {
5124 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5125 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5126 }
5127
5128 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5129 Log4(("Host GS %#08x\n", u32Val));
5130 if (u32Val < HostGdtr.cbGdt)
5131 {
5132 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5133 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5134 }
5135
5136 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5137 Log4(("Host SS %#08x\n", u32Val));
5138 if (u32Val < HostGdtr.cbGdt)
5139 {
5140 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5141 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5142 }
5143
5144 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5145 Log4(("Host TR %#08x\n", u32Val));
5146 if (u32Val < HostGdtr.cbGdt)
5147 {
5148 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5149 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5150 }
5151
5152 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5153 Log4(("Host TR Base %#RHv\n", uHCReg));
5154 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5155 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5156 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5157 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5158 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5159 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5160 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5161 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5162 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5163 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5164 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5165 Log4(("Host RSP %#RHv\n", uHCReg));
5166 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5167 Log4(("Host RIP %#RHv\n", uHCReg));
5168# if HC_ARCH_BITS == 64
5169 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5170 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5171 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5172 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5173 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5174 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5175# endif
5176#endif /* VBOX_STRICT */
5177 break;
5178 }
5179
5180 default:
5181 /* Impossible */
5182 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5183 break;
5184 }
5185}
5186
5187
5188#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5189#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5190# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5191#endif
5192#ifdef VBOX_STRICT
5193static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5194{
5195 switch (idxField)
5196 {
5197 case VMX_VMCS_GUEST_RIP:
5198 case VMX_VMCS_GUEST_RSP:
5199 case VMX_VMCS_GUEST_SYSENTER_EIP:
5200 case VMX_VMCS_GUEST_SYSENTER_ESP:
5201 case VMX_VMCS_GUEST_GDTR_BASE:
5202 case VMX_VMCS_GUEST_IDTR_BASE:
5203 case VMX_VMCS_GUEST_CS_BASE:
5204 case VMX_VMCS_GUEST_DS_BASE:
5205 case VMX_VMCS_GUEST_ES_BASE:
5206 case VMX_VMCS_GUEST_FS_BASE:
5207 case VMX_VMCS_GUEST_GS_BASE:
5208 case VMX_VMCS_GUEST_SS_BASE:
5209 case VMX_VMCS_GUEST_LDTR_BASE:
5210 case VMX_VMCS_GUEST_TR_BASE:
5211 case VMX_VMCS_GUEST_CR3:
5212 return true;
5213 }
5214 return false;
5215}
5216
5217static bool hmR0VmxIsValidReadField(uint32_t idxField)
5218{
5219 switch (idxField)
5220 {
5221 /* Read-only fields. */
5222 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5223 return true;
5224 }
5225 /* Remaining readable fields should also be writable. */
5226 return hmR0VmxIsValidWriteField(idxField);
5227}
5228#endif /* VBOX_STRICT */
5229
5230
5231/**
5232 * Executes the specified handler in 64-bit mode.
5233 *
5234 * @returns VBox status code (no informational status codes).
5235 * @param pVCpu The cross context virtual CPU structure.
5236 * @param enmOp The operation to perform.
5237 * @param cParams Number of parameters.
5238 * @param paParam Array of 32-bit parameters.
5239 */
5240VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
5241{
5242 PVM pVM = pVCpu->CTX_SUFF(pVM);
5243 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5244 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5245 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5246 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5247
5248#ifdef VBOX_STRICT
5249 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5250 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5251
5252 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5253 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5254#endif
5255
5256 /* Disable interrupts. */
5257 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5258
5259#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5260 RTCPUID idHostCpu = RTMpCpuId();
5261 CPUMR0SetLApic(pVCpu, idHostCpu);
5262#endif
5263
5264 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5265 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5266
5267 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5268 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5269 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5270
5271 /* Leave VMX Root Mode. */
5272 VMXDisable();
5273
5274 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5275
5276 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5277 CPUMSetHyperEIP(pVCpu, enmOp);
5278 for (int i = (int)cParams - 1; i >= 0; i--)
5279 CPUMPushHyper(pVCpu, paParam[i]);
5280
5281 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5282
5283 /* Call the switcher. */
5284 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
5285 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5286
5287 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5288 /* Make sure the VMX instructions don't cause #UD faults. */
5289 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5290
5291 /* Re-enter VMX Root Mode */
5292 int rc2 = VMXEnable(HCPhysCpuPage);
5293 if (RT_FAILURE(rc2))
5294 {
5295 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5296 ASMSetFlags(fOldEFlags);
5297 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5298 return rc2;
5299 }
5300
5301 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5302 AssertRC(rc2);
5303 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5304 Assert(!(ASMGetFlags() & X86_EFL_IF));
5305 ASMSetFlags(fOldEFlags);
5306 return rc;
5307}
5308
5309
5310/**
5311 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5312 * supporting 64-bit guests.
5313 *
5314 * @returns VBox status code.
5315 * @param fResume Whether to VMLAUNCH or VMRESUME.
5316 * @param pCtx Pointer to the guest-CPU context.
5317 * @param pCache Pointer to the VMCS cache.
5318 * @param pVM The cross context VM structure.
5319 * @param pVCpu The cross context virtual CPU structure.
5320 */
5321DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5322{
5323 NOREF(fResume);
5324
5325 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5326 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5327
5328#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5329 pCache->uPos = 1;
5330 pCache->interPD = PGMGetInterPaeCR3(pVM);
5331 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5332#endif
5333
5334#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5335 pCache->TestIn.HCPhysCpuPage = 0;
5336 pCache->TestIn.HCPhysVmcs = 0;
5337 pCache->TestIn.pCache = 0;
5338 pCache->TestOut.HCPhysVmcs = 0;
5339 pCache->TestOut.pCache = 0;
5340 pCache->TestOut.pCtx = 0;
5341 pCache->TestOut.eflags = 0;
5342#else
5343 NOREF(pCache);
5344#endif
5345
5346 uint32_t aParam[10];
5347 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5348 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5349 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5350 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5351 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5352 aParam[5] = 0;
5353 aParam[6] = VM_RC_ADDR(pVM, pVM);
5354 aParam[7] = 0;
5355 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5356 aParam[9] = 0;
5357
5358#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5359 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5360 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5361#endif
5362 int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5363
5364#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5365 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5366 Assert(pCtx->dr[4] == 10);
5367 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5368#endif
5369
5370#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5371 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5372 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5373 pVCpu->hm.s.vmx.HCPhysVmcs));
5374 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5375 pCache->TestOut.HCPhysVmcs));
5376 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5377 pCache->TestOut.pCache));
5378 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5379 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5380 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5381 pCache->TestOut.pCtx));
5382 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5383#endif
5384 NOREF(pCtx);
5385 return rc;
5386}
5387
5388
5389/**
5390 * Initialize the VMCS-Read cache.
5391 *
5392 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5393 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5394 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5395 * (those that have a 32-bit FULL & HIGH part).
5396 *
5397 * @returns VBox status code.
5398 * @param pVCpu The cross context virtual CPU structure.
5399 */
5400static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
5401{
5402#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5403 do { \
5404 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5405 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5406 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5407 ++cReadFields; \
5408 } while (0)
5409
5410 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5411 uint32_t cReadFields = 0;
5412
5413 /*
5414 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5415 * and serve to indicate exceptions to the rules.
5416 */
5417
5418 /* Guest-natural selector base fields. */
5419#if 0
5420 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5421 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5422 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5423#endif
5424 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5425 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5426 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5427 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5428 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5429 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5430 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5431 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5432 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5433 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5434 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5435 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5436#if 0
5437 /* Unused natural width guest-state fields. */
5438 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5439 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5440#endif
5441 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5442 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5443
5444 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
5445 these 64-bit fields (using "FULL" and "HIGH" fields). */
5446#if 0
5447 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5448 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5449 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5450 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5451 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5452 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5453 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5454 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5455 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5456#endif
5457
5458 /* Natural width guest-state fields. */
5459 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5460#if 0
5461 /* Currently unused field. */
5462 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5463#endif
5464
5465 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5466 {
5467 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5468 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5469 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5470 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5471 }
5472 else
5473 {
5474 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5475 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5476 }
5477
5478#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5479 return VINF_SUCCESS;
5480}
5481
5482
5483/**
5484 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5485 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5486 * darwin, running 64-bit guests).
5487 *
5488 * @returns VBox status code.
5489 * @param pVCpu The cross context virtual CPU structure.
5490 * @param idxField The VMCS field encoding.
5491 * @param u64Val 16, 32 or 64-bit value.
5492 */
5493VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5494{
5495 int rc;
5496 switch (idxField)
5497 {
5498 /*
5499 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5500 */
5501 /* 64-bit Control fields. */
5502 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5503 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5504 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5505 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5506 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5507 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5508 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5509 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5510 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5511 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5512 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5513 case VMX_VMCS64_CTRL_EPTP_FULL:
5514 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5515 /* 64-bit Guest-state fields. */
5516 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5517 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5518 case VMX_VMCS64_GUEST_PAT_FULL:
5519 case VMX_VMCS64_GUEST_EFER_FULL:
5520 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5521 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5522 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5523 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5524 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5525 /* 64-bit Host-state fields. */
5526 case VMX_VMCS64_HOST_PAT_FULL:
5527 case VMX_VMCS64_HOST_EFER_FULL:
5528 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5529 {
5530 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5531 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5532 break;
5533 }
5534
5535 /*
5536 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5537 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5538 */
5539 /* Natural-width Guest-state fields. */
5540 case VMX_VMCS_GUEST_CR3:
5541 case VMX_VMCS_GUEST_ES_BASE:
5542 case VMX_VMCS_GUEST_CS_BASE:
5543 case VMX_VMCS_GUEST_SS_BASE:
5544 case VMX_VMCS_GUEST_DS_BASE:
5545 case VMX_VMCS_GUEST_FS_BASE:
5546 case VMX_VMCS_GUEST_GS_BASE:
5547 case VMX_VMCS_GUEST_LDTR_BASE:
5548 case VMX_VMCS_GUEST_TR_BASE:
5549 case VMX_VMCS_GUEST_GDTR_BASE:
5550 case VMX_VMCS_GUEST_IDTR_BASE:
5551 case VMX_VMCS_GUEST_RSP:
5552 case VMX_VMCS_GUEST_RIP:
5553 case VMX_VMCS_GUEST_SYSENTER_ESP:
5554 case VMX_VMCS_GUEST_SYSENTER_EIP:
5555 {
5556 if (!(RT_HI_U32(u64Val)))
5557 {
5558 /* If this field is 64-bit, VT-x will zero out the top bits. */
5559 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5560 }
5561 else
5562 {
5563 /* Assert that only the 32->64 switcher case should ever come here. */
5564 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5565 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5566 }
5567 break;
5568 }
5569
5570 default:
5571 {
5572 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5573 rc = VERR_INVALID_PARAMETER;
5574 break;
5575 }
5576 }
5577 AssertRCReturn(rc, rc);
5578 return rc;
5579}
5580
5581
5582/**
5583 * Queue up a VMWRITE by using the VMCS write cache.
5584 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5585 *
5586 * @param pVCpu The cross context virtual CPU structure.
5587 * @param idxField The VMCS field encoding.
5588 * @param u64Val 16, 32 or 64-bit value.
5589 */
5590VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5591{
5592 AssertPtr(pVCpu);
5593 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5594
5595 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5596 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5597
5598 /* Make sure there are no duplicates. */
5599 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5600 {
5601 if (pCache->Write.aField[i] == idxField)
5602 {
5603 pCache->Write.aFieldVal[i] = u64Val;
5604 return VINF_SUCCESS;
5605 }
5606 }
5607
5608 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5609 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5610 pCache->Write.cValidEntries++;
5611 return VINF_SUCCESS;
5612}
5613#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5614
5615
5616/**
5617 * Sets up the usage of TSC-offsetting and updates the VMCS.
5618 *
5619 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5620 * VMX preemption timer.
5621 *
5622 * @returns VBox status code.
5623 * @param pVCpu The cross context virtual CPU structure.
5624 *
5625 * @remarks No-long-jump zone!!!
5626 */
5627static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5628{
5629 bool fOffsettedTsc;
5630 bool fParavirtTsc;
5631 PVM pVM = pVCpu->CTX_SUFF(pVM);
5632 uint64_t uTscOffset;
5633 if (pVM->hm.s.vmx.fUsePreemptTimer)
5634 {
5635 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
5636
5637 /* Make sure the returned values have sane upper and lower boundaries. */
5638 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5639 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5640 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5641 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5642
5643 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5644 int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
5645 AssertRC(rc);
5646 }
5647 else
5648 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
5649
5650 if (fParavirtTsc)
5651 {
5652 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5653 information before every VM-entry, hence disable it for performance sake. */
5654#if 0
5655 int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5656 AssertRC(rc);
5657#endif
5658 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5659 }
5660
5661 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
5662 if ( fOffsettedTsc
5663 && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5664 {
5665 if (pVCpu->hm.s.vmx.u64TscOffset != uTscOffset)
5666 {
5667 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
5668 AssertRC(rc);
5669 pVCpu->hm.s.vmx.u64TscOffset = uTscOffset;
5670 }
5671
5672 if (uProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT)
5673 {
5674 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5675 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5676 AssertRC(rc);
5677 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5678 }
5679 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5680 }
5681 else
5682 {
5683 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5684 if (!(uProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
5685 {
5686 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5687 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5688 AssertRC(rc);
5689 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5690 }
5691 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5692 }
5693}
5694
5695
5696/**
5697 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5698 * VM-exit interruption info type.
5699 *
5700 * @returns The IEM exception flags.
5701 * @param uVector The event vector.
5702 * @param uVmxVectorType The VMX event type.
5703 *
5704 * @remarks This function currently only constructs flags required for
5705 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5706 * and CR2 aspects of an exception are not included).
5707 */
5708static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5709{
5710 uint32_t fIemXcptFlags;
5711 switch (uVmxVectorType)
5712 {
5713 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5714 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5715 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5716 break;
5717
5718 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5719 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5720 break;
5721
5722 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5723 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5724 break;
5725
5726 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5727 {
5728 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5729 if (uVector == X86_XCPT_BP)
5730 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5731 else if (uVector == X86_XCPT_OF)
5732 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5733 else
5734 {
5735 fIemXcptFlags = 0;
5736 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5737 }
5738 break;
5739 }
5740
5741 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5742 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5743 break;
5744
5745 default:
5746 fIemXcptFlags = 0;
5747 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5748 break;
5749 }
5750 return fIemXcptFlags;
5751}
5752
5753
5754/**
5755 * Sets an event as a pending event to be injected into the guest.
5756 *
5757 * @param pVCpu The cross context virtual CPU structure.
5758 * @param u32IntInfo The VM-entry interruption-information field.
5759 * @param cbInstr The VM-entry instruction length in bytes (for software
5760 * interrupts, exceptions and privileged software
5761 * exceptions).
5762 * @param u32ErrCode The VM-entry exception error code.
5763 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5764 * page-fault.
5765 *
5766 * @remarks Statistics counter assumes this is a guest event being injected or
5767 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5768 * always incremented.
5769 */
5770DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5771 RTGCUINTPTR GCPtrFaultAddress)
5772{
5773 Assert(!pVCpu->hm.s.Event.fPending);
5774 pVCpu->hm.s.Event.fPending = true;
5775 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5776 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5777 pVCpu->hm.s.Event.cbInstr = cbInstr;
5778 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5779}
5780
5781
5782/**
5783 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5784 *
5785 * @param pVCpu The cross context virtual CPU structure.
5786 */
5787DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
5788{
5789 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INT_INFO_VALID;
5790 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
5791 u32IntInfo |= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
5792 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5793}
5794
5795
5796/**
5797 * Handle a condition that occurred while delivering an event through the guest
5798 * IDT.
5799 *
5800 * @returns Strict VBox status code (i.e. informational status codes too).
5801 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5802 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5803 * to continue execution of the guest which will delivery the \#DF.
5804 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5805 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5806 *
5807 * @param pVCpu The cross context virtual CPU structure.
5808 * @param pVmxTransient Pointer to the VMX transient structure.
5809 *
5810 * @remarks No-long-jump zone!!!
5811 */
5812static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5813{
5814 uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5815
5816 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5817 rc2 |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5818 AssertRCReturn(rc2, rc2);
5819
5820 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5821 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5822 {
5823 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5824 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5825
5826 /*
5827 * If the event was a software interrupt (generated with INT n) or a software exception
5828 * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
5829 * can handle the VM-exit and continue guest execution which will re-execute the
5830 * instruction rather than re-injecting the exception, as that can cause premature
5831 * trips to ring-3 before injection and involve TRPM which currently has no way of
5832 * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
5833 * the problem).
5834 */
5835 IEMXCPTRAISE enmRaise;
5836 IEMXCPTRAISEINFO fRaiseInfo;
5837 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5838 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5839 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5840 {
5841 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5842 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5843 }
5844 else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5845 {
5846 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5847 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5848 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5849 /** @todo Make AssertMsgReturn as just AssertMsg later. */
5850 AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
5851 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5852 uExitVectorType), VERR_VMX_IPE_5);
5853
5854 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5855
5856 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5857 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
5858 {
5859 pVmxTransient->fVectoringPF = true;
5860 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5861 }
5862 }
5863 else
5864 {
5865 /*
5866 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5867 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5868 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
5869 */
5870 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5871 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5872 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5873 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5874 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5875 }
5876
5877 /*
5878 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5879 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5880 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5881 * subsequent VM-entry would fail.
5882 *
5883 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5884 */
5885 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5886 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5887 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
5888 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
5889 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5890 {
5891 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5892 }
5893
5894 switch (enmRaise)
5895 {
5896 case IEMXCPTRAISE_CURRENT_XCPT:
5897 {
5898 Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
5899 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
5900 Assert(rcStrict == VINF_SUCCESS);
5901 break;
5902 }
5903
5904 case IEMXCPTRAISE_PREV_EVENT:
5905 {
5906 uint32_t u32ErrCode;
5907 if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
5908 {
5909 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5910 AssertRCReturn(rc2, rc2);
5911 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5912 }
5913 else
5914 u32ErrCode = 0;
5915
5916 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
5917 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5918 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5919 0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
5920
5921 Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
5922 pVCpu->hm.s.Event.u32ErrCode));
5923 Assert(rcStrict == VINF_SUCCESS);
5924 break;
5925 }
5926
5927 case IEMXCPTRAISE_REEXEC_INSTR:
5928 Assert(rcStrict == VINF_SUCCESS);
5929 break;
5930
5931 case IEMXCPTRAISE_DOUBLE_FAULT:
5932 {
5933 /*
5934 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
5935 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
5936 */
5937 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
5938 {
5939 pVmxTransient->fVectoringDoublePF = true;
5940 Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
5941 pVCpu->cpum.GstCtx.cr2));
5942 rcStrict = VINF_SUCCESS;
5943 }
5944 else
5945 {
5946 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5947 hmR0VmxSetPendingXcptDF(pVCpu);
5948 Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
5949 uIdtVector, uExitVector));
5950 rcStrict = VINF_HM_DOUBLE_FAULT;
5951 }
5952 break;
5953 }
5954
5955 case IEMXCPTRAISE_TRIPLE_FAULT:
5956 {
5957 Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
5958 rcStrict = VINF_EM_RESET;
5959 break;
5960 }
5961
5962 case IEMXCPTRAISE_CPU_HANG:
5963 {
5964 Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
5965 rcStrict = VERR_EM_GUEST_CPU_HANG;
5966 break;
5967 }
5968
5969 default:
5970 {
5971 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
5972 rcStrict = VERR_VMX_IPE_2;
5973 break;
5974 }
5975 }
5976 }
5977 else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5978 && VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5979 && uExitVector != X86_XCPT_DF
5980 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5981 {
5982 /*
5983 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5984 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5985 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5986 */
5987 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5988 {
5989 Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
5990 VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5991 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5992 }
5993 }
5994
5995 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
5996 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
5997 return rcStrict;
5998}
5999
6000
6001/**
6002 * Imports a guest segment register from the current VMCS into
6003 * the guest-CPU context.
6004 *
6005 * @returns VBox status code.
6006 * @param pVCpu The cross context virtual CPU structure.
6007 * @param idxSel Index of the selector in the VMCS.
6008 * @param idxLimit Index of the segment limit in the VMCS.
6009 * @param idxBase Index of the segment base in the VMCS.
6010 * @param idxAccess Index of the access rights of the segment in the VMCS.
6011 * @param pSelReg Pointer to the segment selector.
6012 *
6013 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6014 * do not log!
6015 *
6016 * @remarks Never call this function directly!!! Use the
6017 * HMVMX_IMPORT_SREG() macro as that takes care
6018 * of whether to read from the VMCS cache or not.
6019 */
6020static int hmR0VmxImportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6021 PCPUMSELREG pSelReg)
6022{
6023 NOREF(pVCpu);
6024
6025 uint32_t u32Sel;
6026 uint32_t u32Limit;
6027 uint32_t u32Attr;
6028 uint64_t u64Base;
6029 int rc = VMXReadVmcs32(idxSel, &u32Sel);
6030 rc |= VMXReadVmcs32(idxLimit, &u32Limit);
6031 rc |= VMXReadVmcs32(idxAccess, &u32Attr);
6032 rc |= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
6033 AssertRCReturn(rc, rc);
6034
6035 pSelReg->Sel = (uint16_t)u32Sel;
6036 pSelReg->ValidSel = (uint16_t)u32Sel;
6037 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6038 pSelReg->u32Limit = u32Limit;
6039 pSelReg->u64Base = u64Base;
6040 pSelReg->Attr.u = u32Attr;
6041
6042 /*
6043 * If VT-x marks the segment as unusable, most other bits remain undefined:
6044 * - For CS the L, D and G bits have meaning.
6045 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6046 * - For the remaining data segments no bits are defined.
6047 *
6048 * The present bit and the unusable bit has been observed to be set at the
6049 * same time (the selector was supposed to be invalid as we started executing
6050 * a V8086 interrupt in ring-0).
6051 *
6052 * What should be important for the rest of the VBox code, is that the P bit is
6053 * cleared. Some of the other VBox code recognizes the unusable bit, but
6054 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6055 * safe side here, we'll strip off P and other bits we don't care about. If
6056 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6057 *
6058 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6059 */
6060 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6061 {
6062 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6063
6064 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6065 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6066 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6067#ifdef VBOX_STRICT
6068 VMMRZCallRing3Disable(pVCpu);
6069 Log4Func(("Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Sel, pSelReg->Attr.u));
6070# ifdef DEBUG_bird
6071 AssertMsg((u32Attr & ~X86DESCATTR_P) == pSelReg->Attr.u,
6072 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6073 idxSel, u32Sel, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6074# endif
6075 VMMRZCallRing3Enable(pVCpu);
6076#endif
6077 }
6078 return VINF_SUCCESS;
6079}
6080
6081
6082/**
6083 * Imports the guest RIP from the VMCS back into the guest-CPU context.
6084 *
6085 * @returns VBox status code.
6086 * @param pVCpu The cross context virtual CPU structure.
6087 *
6088 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6089 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6090 * instead!!!
6091 */
6092DECLINLINE(int) hmR0VmxImportGuestRip(PVMCPU pVCpu)
6093{
6094 uint64_t u64Val;
6095 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6096 if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
6097 {
6098 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6099 if (RT_SUCCESS(rc))
6100 {
6101 pCtx->rip = u64Val;
6102 EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
6103 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
6104 }
6105 return rc;
6106 }
6107 return VINF_SUCCESS;
6108}
6109
6110
6111/**
6112 * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
6113 *
6114 * @returns VBox status code.
6115 * @param pVCpu The cross context virtual CPU structure.
6116 *
6117 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6118 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6119 * instead!!!
6120 */
6121DECLINLINE(int) hmR0VmxImportGuestRFlags(PVMCPU pVCpu)
6122{
6123 uint32_t u32Val;
6124 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6125 if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
6126 {
6127 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
6128 if (RT_SUCCESS(rc))
6129 {
6130 pCtx->eflags.u32 = u32Val;
6131
6132 /* Restore eflags for real-on-v86-mode hack. */
6133 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6134 {
6135 pCtx->eflags.Bits.u1VM = 0;
6136 pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6137 }
6138 }
6139 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
6140 return rc;
6141 }
6142 return VINF_SUCCESS;
6143}
6144
6145
6146/**
6147 * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
6148 * context.
6149 *
6150 * @returns VBox status code.
6151 * @param pVCpu The cross context virtual CPU structure.
6152 *
6153 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6154 * do not log!
6155 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6156 * instead!!!
6157 */
6158DECLINLINE(int) hmR0VmxImportGuestIntrState(PVMCPU pVCpu)
6159{
6160 uint32_t u32Val;
6161 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6162 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
6163 if (RT_SUCCESS(rc))
6164 {
6165 /*
6166 * We additionally have a requirement to import RIP, RFLAGS depending on whether we
6167 * might need them in hmR0VmxEvaluatePendingEvent().
6168 */
6169 if (!u32Val)
6170 {
6171 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6172 {
6173 rc = hmR0VmxImportGuestRip(pVCpu);
6174 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6175 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6176 }
6177
6178 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6179 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6180 }
6181 else
6182 {
6183 rc = hmR0VmxImportGuestRip(pVCpu);
6184 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6185
6186 if (u32Val & ( VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
6187 | VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
6188 {
6189 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
6190 }
6191 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6192 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6193
6194 if (u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
6195 {
6196 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6197 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6198 }
6199 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6200 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6201 }
6202 }
6203 return rc;
6204}
6205
6206
6207/**
6208 * Worker for VMXR0ImportStateOnDemand.
6209 *
6210 * @returns VBox status code.
6211 * @param pVCpu The cross context virtual CPU structure.
6212 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6213 */
6214static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
6215{
6216#define VMXLOCAL_BREAK_RC(a_rc) \
6217 if (RT_FAILURE(a_rc)) \
6218 break
6219
6220 int rc = VINF_SUCCESS;
6221 PVM pVM = pVCpu->CTX_SUFF(pVM);
6222 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6223 uint64_t u64Val;
6224 uint32_t u32Val;
6225
6226 Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
6227 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
6228
6229 /*
6230 * We disable interrupts to make the updating of the state and in particular
6231 * the fExtrn modification atomic wrt to preemption hooks.
6232 */
6233 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
6234
6235 fWhat &= pCtx->fExtrn;
6236 if (fWhat)
6237 {
6238 do
6239 {
6240 if (fWhat & CPUMCTX_EXTRN_RIP)
6241 {
6242 rc = hmR0VmxImportGuestRip(pVCpu);
6243 VMXLOCAL_BREAK_RC(rc);
6244 }
6245
6246 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
6247 {
6248 rc = hmR0VmxImportGuestRFlags(pVCpu);
6249 VMXLOCAL_BREAK_RC(rc);
6250 }
6251
6252 if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
6253 {
6254 rc = hmR0VmxImportGuestIntrState(pVCpu);
6255 VMXLOCAL_BREAK_RC(rc);
6256 }
6257
6258 if (fWhat & CPUMCTX_EXTRN_RSP)
6259 {
6260 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6261 VMXLOCAL_BREAK_RC(rc);
6262 pCtx->rsp = u64Val;
6263 }
6264
6265 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
6266 {
6267 if (fWhat & CPUMCTX_EXTRN_CS)
6268 {
6269 rc = HMVMX_IMPORT_SREG(CS, &pCtx->cs);
6270 rc |= hmR0VmxImportGuestRip(pVCpu);
6271 VMXLOCAL_BREAK_RC(rc);
6272 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6273 pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6274 EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true);
6275 }
6276 if (fWhat & CPUMCTX_EXTRN_SS)
6277 {
6278 rc = HMVMX_IMPORT_SREG(SS, &pCtx->ss);
6279 VMXLOCAL_BREAK_RC(rc);
6280 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6281 pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6282 }
6283 if (fWhat & CPUMCTX_EXTRN_DS)
6284 {
6285 rc = HMVMX_IMPORT_SREG(DS, &pCtx->ds);
6286 VMXLOCAL_BREAK_RC(rc);
6287 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6288 pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6289 }
6290 if (fWhat & CPUMCTX_EXTRN_ES)
6291 {
6292 rc = HMVMX_IMPORT_SREG(ES, &pCtx->es);
6293 VMXLOCAL_BREAK_RC(rc);
6294 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6295 pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6296 }
6297 if (fWhat & CPUMCTX_EXTRN_FS)
6298 {
6299 rc = HMVMX_IMPORT_SREG(FS, &pCtx->fs);
6300 VMXLOCAL_BREAK_RC(rc);
6301 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6302 pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6303 }
6304 if (fWhat & CPUMCTX_EXTRN_GS)
6305 {
6306 rc = HMVMX_IMPORT_SREG(GS, &pCtx->gs);
6307 VMXLOCAL_BREAK_RC(rc);
6308 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6309 pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6310 }
6311 }
6312
6313 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
6314 {
6315 if (fWhat & CPUMCTX_EXTRN_LDTR)
6316 {
6317 rc = HMVMX_IMPORT_SREG(LDTR, &pCtx->ldtr);
6318 VMXLOCAL_BREAK_RC(rc);
6319 }
6320
6321 if (fWhat & CPUMCTX_EXTRN_GDTR)
6322 {
6323 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6324 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
6325 VMXLOCAL_BREAK_RC(rc);
6326 pCtx->gdtr.pGdt = u64Val;
6327 pCtx->gdtr.cbGdt = u32Val;
6328 }
6329
6330 /* Guest IDTR. */
6331 if (fWhat & CPUMCTX_EXTRN_IDTR)
6332 {
6333 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6334 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
6335 VMXLOCAL_BREAK_RC(rc);
6336 pCtx->idtr.pIdt = u64Val;
6337 pCtx->idtr.cbIdt = u32Val;
6338 }
6339
6340 /* Guest TR. */
6341 if (fWhat & CPUMCTX_EXTRN_TR)
6342 {
6343 /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
6344 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6345 {
6346 rc = HMVMX_IMPORT_SREG(TR, &pCtx->tr);
6347 VMXLOCAL_BREAK_RC(rc);
6348 }
6349 }
6350 }
6351
6352 if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
6353 {
6354 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
6355 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
6356 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
6357 pCtx->SysEnter.cs = u32Val;
6358 VMXLOCAL_BREAK_RC(rc);
6359 }
6360
6361#if HC_ARCH_BITS == 64
6362 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
6363 {
6364 if ( pVM->hm.s.fAllow64BitGuests
6365 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6366 pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
6367 }
6368
6369 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
6370 {
6371 if ( pVM->hm.s.fAllow64BitGuests
6372 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6373 {
6374 pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
6375 pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
6376 pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
6377 }
6378 }
6379#endif
6380
6381 if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
6382#if HC_ARCH_BITS == 32
6383 || (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS))
6384#endif
6385 )
6386 {
6387 PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6388 uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
6389 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6390 {
6391 switch (pMsr->u32Msr)
6392 {
6393#if HC_ARCH_BITS == 32
6394 case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
6395 case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
6396 case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
6397 case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6398#endif
6399 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6400 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6401 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit */ break;
6402 default:
6403 {
6404 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6405 ASMSetFlags(fEFlags);
6406 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
6407 cMsrs));
6408 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6409 }
6410 }
6411 }
6412 }
6413
6414 if (fWhat & CPUMCTX_EXTRN_DR7)
6415 {
6416 if (!pVCpu->hm.s.fUsingHyperDR7)
6417 {
6418 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6419 rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
6420 VMXLOCAL_BREAK_RC(rc);
6421 pCtx->dr[7] = u32Val;
6422 }
6423 }
6424
6425 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
6426 {
6427 uint32_t u32Shadow;
6428 if (fWhat & CPUMCTX_EXTRN_CR0)
6429 {
6430 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
6431 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
6432 VMXLOCAL_BREAK_RC(rc);
6433 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr0Mask)
6434 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr0Mask);
6435 VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
6436 CPUMSetGuestCR0(pVCpu, u32Val);
6437 VMMRZCallRing3Enable(pVCpu);
6438 }
6439
6440 if (fWhat & CPUMCTX_EXTRN_CR4)
6441 {
6442 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
6443 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
6444 VMXLOCAL_BREAK_RC(rc);
6445 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr4Mask)
6446 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr4Mask);
6447 CPUMSetGuestCR4(pVCpu, u32Val);
6448 }
6449
6450 if (fWhat & CPUMCTX_EXTRN_CR3)
6451 {
6452 /* CR0.PG bit changes are always intercepted, so it's up to date. */
6453 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6454 || ( pVM->hm.s.fNestedPaging
6455 && CPUMIsGuestPagingEnabledEx(pCtx)))
6456 {
6457 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6458 if (pCtx->cr3 != u64Val)
6459 {
6460 CPUMSetGuestCR3(pVCpu, u64Val);
6461 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6462 }
6463
6464 /* If the guest is in PAE mode, sync back the PDPE's into the guest state.
6465 Note: CR4.PAE, CR0.PG, EFER bit changes are always intercepted, so they're up to date. */
6466 if (CPUMIsGuestInPAEModeEx(pCtx))
6467 {
6468 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6469 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6470 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6471 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6472 VMXLOCAL_BREAK_RC(rc);
6473 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6474 }
6475 }
6476 }
6477 }
6478 } while (0);
6479
6480 if (RT_SUCCESS(rc))
6481 {
6482 /* Update fExtrn. */
6483 pCtx->fExtrn &= ~fWhat;
6484
6485 /* If everything has been imported, clear the HM keeper bit. */
6486 if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
6487 {
6488 pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
6489 Assert(!pCtx->fExtrn);
6490 }
6491 }
6492 }
6493 else
6494 AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
6495
6496 ASMSetFlags(fEFlags);
6497
6498 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
6499
6500 /*
6501 * Honor any pending CR3 updates.
6502 *
6503 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6504 * -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6505 * -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
6506 *
6507 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6508 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6509 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6510 * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
6511 *
6512 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6513 */
6514 if (VMMRZCallRing3IsEnabled(pVCpu))
6515 {
6516 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6517 {
6518 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
6519 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6520 }
6521
6522 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6523 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6524
6525 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6526 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6527 }
6528
6529 return VINF_SUCCESS;
6530#undef VMXLOCAL_BREAK_RC
6531}
6532
6533
6534/**
6535 * Saves the guest state from the VMCS into the guest-CPU context.
6536 *
6537 * @returns VBox status code.
6538 * @param pVCpu The cross context virtual CPU structure.
6539 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6540 */
6541VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
6542{
6543 return hmR0VmxImportGuestState(pVCpu, fWhat);
6544}
6545
6546
6547/**
6548 * Check per-VM and per-VCPU force flag actions that require us to go back to
6549 * ring-3 for one reason or another.
6550 *
6551 * @returns Strict VBox status code (i.e. informational status codes too)
6552 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6553 * ring-3.
6554 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6555 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6556 * interrupts)
6557 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6558 * all EMTs to be in ring-3.
6559 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6560 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6561 * to the EM loop.
6562 *
6563 * @param pVCpu The cross context virtual CPU structure.
6564 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6565 */
6566static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
6567{
6568 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6569
6570 /*
6571 * Anything pending? Should be more likely than not if we're doing a good job.
6572 */
6573 PVM pVM = pVCpu->CTX_SUFF(pVM);
6574 if ( !fStepping
6575 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6576 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6577 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6578 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6579 return VINF_SUCCESS;
6580
6581 /* Pending PGM C3 sync. */
6582 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6583 {
6584 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6585 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
6586 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
6587 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6588 if (rcStrict2 != VINF_SUCCESS)
6589 {
6590 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6591 Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6592 return rcStrict2;
6593 }
6594 }
6595
6596 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6597 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6598 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6599 {
6600 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6601 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6602 Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6603 return rc2;
6604 }
6605
6606 /* Pending VM request packets, such as hardware interrupts. */
6607 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6608 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6609 {
6610 Log4Func(("Pending VM request forcing us back to ring-3\n"));
6611 return VINF_EM_PENDING_REQUEST;
6612 }
6613
6614 /* Pending PGM pool flushes. */
6615 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6616 {
6617 Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
6618 return VINF_PGM_POOL_FLUSH_PENDING;
6619 }
6620
6621 /* Pending DMA requests. */
6622 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6623 {
6624 Log4Func(("Pending DMA request forcing us back to ring-3\n"));
6625 return VINF_EM_RAW_TO_R3;
6626 }
6627
6628 return VINF_SUCCESS;
6629}
6630
6631
6632/**
6633 * Converts any TRPM trap into a pending HM event. This is typically used when
6634 * entering from ring-3 (not longjmp returns).
6635 *
6636 * @param pVCpu The cross context virtual CPU structure.
6637 */
6638static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6639{
6640 Assert(TRPMHasTrap(pVCpu));
6641 Assert(!pVCpu->hm.s.Event.fPending);
6642
6643 uint8_t uVector;
6644 TRPMEVENT enmTrpmEvent;
6645 RTGCUINT uErrCode;
6646 RTGCUINTPTR GCPtrFaultAddress;
6647 uint8_t cbInstr;
6648
6649 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6650 AssertRC(rc);
6651
6652 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6653 uint32_t u32IntInfo = uVector | VMX_EXIT_INT_INFO_VALID;
6654 if (enmTrpmEvent == TRPM_TRAP)
6655 {
6656 switch (uVector)
6657 {
6658 case X86_XCPT_NMI:
6659 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6660 break;
6661
6662 case X86_XCPT_BP:
6663 case X86_XCPT_OF:
6664 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6665 break;
6666
6667 case X86_XCPT_PF:
6668 case X86_XCPT_DF:
6669 case X86_XCPT_TS:
6670 case X86_XCPT_NP:
6671 case X86_XCPT_SS:
6672 case X86_XCPT_GP:
6673 case X86_XCPT_AC:
6674 u32IntInfo |= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
6675 RT_FALL_THRU();
6676 default:
6677 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6678 break;
6679 }
6680 }
6681 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6682 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6683 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6684 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6685 else
6686 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6687
6688 rc = TRPMResetTrap(pVCpu);
6689 AssertRC(rc);
6690 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6691 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6692
6693 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6694}
6695
6696
6697/**
6698 * Converts the pending HM event into a TRPM trap.
6699 *
6700 * @param pVCpu The cross context virtual CPU structure.
6701 */
6702static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6703{
6704 Assert(pVCpu->hm.s.Event.fPending);
6705
6706 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6707 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6708 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
6709 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6710
6711 /* If a trap was already pending, we did something wrong! */
6712 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6713
6714 TRPMEVENT enmTrapType;
6715 switch (uVectorType)
6716 {
6717 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6718 enmTrapType = TRPM_HARDWARE_INT;
6719 break;
6720
6721 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6722 enmTrapType = TRPM_SOFTWARE_INT;
6723 break;
6724
6725 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6726 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6727 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6728 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6729 enmTrapType = TRPM_TRAP;
6730 break;
6731
6732 default:
6733 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6734 enmTrapType = TRPM_32BIT_HACK;
6735 break;
6736 }
6737
6738 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6739
6740 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6741 AssertRC(rc);
6742
6743 if (fErrorCodeValid)
6744 TRPMSetErrorCode(pVCpu, uErrorCode);
6745
6746 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6747 && uVector == X86_XCPT_PF)
6748 {
6749 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6750 }
6751 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6752 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6753 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6754 {
6755 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6756 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6757 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6758 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6759 }
6760
6761 /* Clear any pending events from the VMCS. */
6762 VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
6763 VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, 0);
6764
6765 /* We're now done converting the pending event. */
6766 pVCpu->hm.s.Event.fPending = false;
6767}
6768
6769
6770/**
6771 * Does the necessary state syncing before returning to ring-3 for any reason
6772 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6773 *
6774 * @returns VBox status code.
6775 * @param pVCpu The cross context virtual CPU structure.
6776 * @param fImportState Whether to import the guest state from the VMCS back
6777 * to the guest-CPU context.
6778 *
6779 * @remarks No-long-jmp zone!!!
6780 */
6781static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
6782{
6783 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6784 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6785
6786 RTCPUID idCpu = RTMpCpuId();
6787 Log4Func(("HostCpuId=%u\n", idCpu));
6788
6789 /*
6790 * !!! IMPORTANT !!!
6791 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
6792 */
6793
6794 /* Save the guest state if necessary. */
6795 if (fImportState)
6796 {
6797 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
6798 AssertRCReturn(rc, rc);
6799 }
6800
6801 /* Restore host FPU state if necessary. We will resync on next R0 reentry. */
6802 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
6803 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6804
6805 /* Restore host debug registers if necessary. We will resync on next R0 reentry. */
6806#ifdef VBOX_STRICT
6807 if (CPUMIsHyperDebugStateActive(pVCpu))
6808 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6809#endif
6810 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6811 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6812 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6813
6814#if HC_ARCH_BITS == 64
6815 /* Restore host-state bits that VT-x only restores partially. */
6816 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6817 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6818 {
6819 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6820 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6821 }
6822 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6823#endif
6824
6825 /* Restore the lazy host MSRs as we're leaving VT-x context. */
6826 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
6827 {
6828 /* We shouldn't restore the host MSRs without saving the guest MSRs first. */
6829 if (!fImportState)
6830 {
6831 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS);
6832 AssertRCReturn(rc, rc);
6833 }
6834 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6835 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
6836 }
6837 else
6838 pVCpu->hm.s.vmx.fLazyMsrs = 0;
6839
6840 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6841 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6842
6843 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6844 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
6845 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
6846 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
6847 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
6848 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6849 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6850 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6851 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6852
6853 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6854
6855 /** @todo This partially defeats the purpose of having preemption hooks.
6856 * The problem is, deregistering the hooks should be moved to a place that
6857 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6858 * context.
6859 */
6860 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6861 {
6862 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6863 AssertRCReturn(rc, rc);
6864
6865 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6866 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6867 }
6868 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6869 NOREF(idCpu);
6870
6871 return VINF_SUCCESS;
6872}
6873
6874
6875/**
6876 * Leaves the VT-x session.
6877 *
6878 * @returns VBox status code.
6879 * @param pVCpu The cross context virtual CPU structure.
6880 *
6881 * @remarks No-long-jmp zone!!!
6882 */
6883static int hmR0VmxLeaveSession(PVMCPU pVCpu)
6884{
6885 HM_DISABLE_PREEMPT(pVCpu);
6886 HMVMX_ASSERT_CPU_SAFE(pVCpu);
6887 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6888 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6889
6890 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6891 and done this from the VMXR0ThreadCtxCallback(). */
6892 if (!pVCpu->hm.s.fLeaveDone)
6893 {
6894 int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
6895 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
6896 pVCpu->hm.s.fLeaveDone = true;
6897 }
6898 Assert(!pVCpu->cpum.GstCtx.fExtrn);
6899
6900 /*
6901 * !!! IMPORTANT !!!
6902 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6903 */
6904
6905 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6906 /** @todo Deregistering here means we need to VMCLEAR always
6907 * (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
6908 * for calling VMMR0ThreadCtxHookDisable here! */
6909 VMMR0ThreadCtxHookDisable(pVCpu);
6910
6911 /* Leave HM context. This takes care of local init (term). */
6912 int rc = HMR0LeaveCpu(pVCpu);
6913
6914 HM_RESTORE_PREEMPT();
6915 return rc;
6916}
6917
6918
6919/**
6920 * Does the necessary state syncing before doing a longjmp to ring-3.
6921 *
6922 * @returns VBox status code.
6923 * @param pVCpu The cross context virtual CPU structure.
6924 *
6925 * @remarks No-long-jmp zone!!!
6926 */
6927DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
6928{
6929 return hmR0VmxLeaveSession(pVCpu);
6930}
6931
6932
6933/**
6934 * Take necessary actions before going back to ring-3.
6935 *
6936 * An action requires us to go back to ring-3. This function does the necessary
6937 * steps before we can safely return to ring-3. This is not the same as longjmps
6938 * to ring-3, this is voluntary and prepares the guest so it may continue
6939 * executing outside HM (recompiler/IEM).
6940 *
6941 * @returns VBox status code.
6942 * @param pVCpu The cross context virtual CPU structure.
6943 * @param rcExit The reason for exiting to ring-3. Can be
6944 * VINF_VMM_UNKNOWN_RING3_CALL.
6945 */
6946static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
6947{
6948 Assert(pVCpu);
6949 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6950
6951 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6952 {
6953 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6954 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6955 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6956 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6957 }
6958
6959 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6960 VMMRZCallRing3Disable(pVCpu);
6961 Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
6962
6963 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6964 if (pVCpu->hm.s.Event.fPending)
6965 {
6966 hmR0VmxPendingEventToTrpmTrap(pVCpu);
6967 Assert(!pVCpu->hm.s.Event.fPending);
6968 }
6969
6970 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
6971 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
6972
6973 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
6974 and if we're injecting an event we should have a TRPM trap pending. */
6975 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6976#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
6977 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6978#endif
6979
6980 /* Save guest state and restore host state bits. */
6981 int rc = hmR0VmxLeaveSession(pVCpu);
6982 AssertRCReturn(rc, rc);
6983 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6984 /* Thread-context hooks are unregistered at this point!!! */
6985
6986 /* Sync recompiler state. */
6987 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6988 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6989 | CPUM_CHANGED_LDTR
6990 | CPUM_CHANGED_GDTR
6991 | CPUM_CHANGED_IDTR
6992 | CPUM_CHANGED_TR
6993 | CPUM_CHANGED_HIDDEN_SEL_REGS);
6994 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
6995 && CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
6996 {
6997 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6998 }
6999
7000 Assert(!pVCpu->hm.s.fClearTrapFlag);
7001
7002 /* Update the exit-to-ring 3 reason. */
7003 pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
7004
7005 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7006 if (rcExit != VINF_EM_RAW_INTERRUPT)
7007 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7008
7009 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7010
7011 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7012 VMMRZCallRing3RemoveNotification(pVCpu);
7013 VMMRZCallRing3Enable(pVCpu);
7014
7015 return rc;
7016}
7017
7018
7019/**
7020 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7021 * longjump to ring-3 and possibly get preempted.
7022 *
7023 * @returns VBox status code.
7024 * @param pVCpu The cross context virtual CPU structure.
7025 * @param enmOperation The operation causing the ring-3 longjump.
7026 * @param pvUser User argument, currently unused, NULL.
7027 */
7028static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7029{
7030 RT_NOREF(pvUser);
7031 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7032 {
7033 /*
7034 * !!! IMPORTANT !!!
7035 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7036 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7037 */
7038 VMMRZCallRing3RemoveNotification(pVCpu);
7039 VMMRZCallRing3Disable(pVCpu);
7040 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7041 RTThreadPreemptDisable(&PreemptState);
7042
7043 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
7044 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7045 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7046
7047#if HC_ARCH_BITS == 64
7048 /* Restore host-state bits that VT-x only restores partially. */
7049 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7050 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7051 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7052 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7053#endif
7054
7055 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7056 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7057 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7058
7059 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7060 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7061 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7062 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7063 {
7064 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7065 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7066 }
7067
7068 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7069 VMMR0ThreadCtxHookDisable(pVCpu);
7070 HMR0LeaveCpu(pVCpu);
7071 RTThreadPreemptRestore(&PreemptState);
7072 return VINF_SUCCESS;
7073 }
7074
7075 Assert(pVCpu);
7076 Assert(pvUser);
7077 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7078 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7079
7080 VMMRZCallRing3Disable(pVCpu);
7081 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7082
7083 Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
7084
7085 int rc = hmR0VmxLongJmpToRing3(pVCpu);
7086 AssertRCReturn(rc, rc);
7087
7088 VMMRZCallRing3Enable(pVCpu);
7089 return VINF_SUCCESS;
7090}
7091
7092
7093/**
7094 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7095 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7096 *
7097 * @param pVCpu The cross context virtual CPU structure.
7098 */
7099DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7100{
7101 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7102 {
7103 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7104 {
7105 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7106 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7107 AssertRC(rc);
7108 Log4Func(("Setup interrupt-window exiting\n"));
7109 }
7110 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7111}
7112
7113
7114/**
7115 * Clears the interrupt-window exiting control in the VMCS.
7116 *
7117 * @param pVCpu The cross context virtual CPU structure.
7118 */
7119DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7120{
7121 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7122 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7123 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7124 AssertRC(rc);
7125 Log4Func(("Cleared interrupt-window exiting\n"));
7126}
7127
7128
7129/**
7130 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7131 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7132 *
7133 * @param pVCpu The cross context virtual CPU structure.
7134 */
7135DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7136{
7137 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7138 {
7139 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7140 {
7141 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7142 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7143 AssertRC(rc);
7144 Log4Func(("Setup NMI-window exiting\n"));
7145 }
7146 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7147}
7148
7149
7150/**
7151 * Clears the NMI-window exiting control in the VMCS.
7152 *
7153 * @param pVCpu The cross context virtual CPU structure.
7154 */
7155DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7156{
7157 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7158 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7159 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7160 AssertRC(rc);
7161 Log4Func(("Cleared NMI-window exiting\n"));
7162}
7163
7164
7165/**
7166 * Evaluates the event to be delivered to the guest and sets it as the pending
7167 * event.
7168 *
7169 * @returns The VT-x guest-interruptibility state.
7170 * @param pVCpu The cross context virtual CPU structure.
7171 */
7172static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu)
7173{
7174 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7175 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7176 uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu);
7177 bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7178 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7179 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7180
7181 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7182 Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7183 Assert(!fBlockSti || pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7184 Assert(!TRPMHasTrap(pVCpu));
7185
7186 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7187 APICUpdatePendingInterrupts(pVCpu);
7188
7189 /*
7190 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7191 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7192 */
7193 /** @todo SMI. SMIs take priority over NMIs. */
7194 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7195 {
7196 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7197 if ( !pVCpu->hm.s.Event.fPending
7198 && !fBlockNmi
7199 && !fBlockSti
7200 && !fBlockMovSS)
7201 {
7202 Log4Func(("Pending NMI\n"));
7203 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INT_INFO_VALID;
7204 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7205
7206 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7207 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7208 }
7209 else
7210 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7211 }
7212 /*
7213 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7214 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7215 */
7216 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7217 && !pVCpu->hm.s.fSingleInstruction)
7218 {
7219 Assert(!DBGFIsStepping(pVCpu));
7220 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7221 AssertRCReturn(rc, 0);
7222 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7223 if ( !pVCpu->hm.s.Event.fPending
7224 && !fBlockInt
7225 && !fBlockSti
7226 && !fBlockMovSS)
7227 {
7228 uint8_t u8Interrupt;
7229 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7230 if (RT_SUCCESS(rc))
7231 {
7232 Log4Func(("Pending external interrupt u8Interrupt=%#x\n", u8Interrupt));
7233 uint32_t u32IntInfo = u8Interrupt
7234 | VMX_EXIT_INT_INFO_VALID
7235 | (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7236
7237 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7238 }
7239 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7240 {
7241 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7242 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7243 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7244
7245 /*
7246 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7247 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7248 * need to re-set this force-flag here.
7249 */
7250 }
7251 else
7252 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7253 }
7254 else
7255 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7256 }
7257
7258 return fIntrState;
7259}
7260
7261
7262/**
7263 * Sets a pending-debug exception to be delivered to the guest if the guest is
7264 * single-stepping in the VMCS.
7265 *
7266 * @param pVCpu The cross context virtual CPU structure.
7267 */
7268DECLINLINE(int) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7269{
7270 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7271 RT_NOREF(pVCpu);
7272 return VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
7273}
7274
7275
7276/**
7277 * Injects any pending events into the guest if the guest is in a state to
7278 * receive them.
7279 *
7280 * @returns Strict VBox status code (i.e. informational status codes too).
7281 * @param pVCpu The cross context virtual CPU structure.
7282 * @param fIntrState The VT-x guest-interruptibility state.
7283 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7284 * return VINF_EM_DBG_STEPPED if the event was
7285 * dispatched directly.
7286 */
7287static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, uint32_t fIntrState, bool fStepping)
7288{
7289 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7290 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7291
7292 bool fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7293 bool fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7294
7295 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7296 Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7297 Assert(!fBlockSti || pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7298 Assert(!TRPMHasTrap(pVCpu));
7299
7300 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7301 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7302 if (pVCpu->hm.s.Event.fPending)
7303 {
7304 /*
7305 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7306 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7307 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7308 *
7309 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7310 */
7311 uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7312#ifdef VBOX_STRICT
7313 if (uIntType == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
7314 {
7315 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7316 Assert(!fBlockInt);
7317 Assert(!fBlockSti);
7318 Assert(!fBlockMovSS);
7319 }
7320 else if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
7321 {
7322 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7323 Assert(!fBlockSti);
7324 Assert(!fBlockMovSS);
7325 Assert(!fBlockNmi);
7326 }
7327#endif
7328 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7329 uIntType));
7330 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7331 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping,
7332 &fIntrState);
7333 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7334
7335 /* Update the interruptibility-state as it could have been changed by
7336 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7337 fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7338 fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7339
7340 if (uIntType == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
7341 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7342 else
7343 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7344 }
7345
7346 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7347 if ( fBlockSti
7348 || fBlockMovSS)
7349 {
7350 if (!pVCpu->hm.s.fSingleInstruction)
7351 {
7352 /*
7353 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7354 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7355 * See Intel spec. 27.3.4 "Saving Non-Register State".
7356 */
7357 Assert(!DBGFIsStepping(pVCpu));
7358 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7359 AssertRCReturn(rc, rc);
7360 if (pCtx->eflags.Bits.u1TF)
7361 {
7362 int rc2 = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7363 AssertRCReturn(rc2, rc2);
7364 }
7365 }
7366 else if (pCtx->eflags.Bits.u1TF)
7367 {
7368 /*
7369 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7370 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7371 */
7372 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7373 fIntrState = 0;
7374 }
7375 }
7376
7377 /*
7378 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7379 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7380 */
7381 int rc3 = hmR0VmxExportGuestIntrState(pVCpu, fIntrState);
7382 AssertRCReturn(rc3, rc3);
7383
7384 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7385 NOREF(fBlockMovSS); NOREF(fBlockSti);
7386 return rcStrict;
7387}
7388
7389
7390/**
7391 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7392 *
7393 * @param pVCpu The cross context virtual CPU structure.
7394 */
7395DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
7396{
7397 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INT_INFO_VALID;
7398 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7399}
7400
7401
7402/**
7403 * Injects a double-fault (\#DF) exception into the VM.
7404 *
7405 * @returns Strict VBox status code (i.e. informational status codes too).
7406 * @param pVCpu The cross context virtual CPU structure.
7407 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7408 * and should return VINF_EM_DBG_STEPPED if the event
7409 * is injected directly (register modified by us, not
7410 * by hardware on VM-entry).
7411 * @param pfIntrState Pointer to the current guest interruptibility-state.
7412 * This interruptibility-state will be updated if
7413 * necessary. This cannot not be NULL.
7414 */
7415DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, bool fStepping, uint32_t *pfIntrState)
7416{
7417 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INT_INFO_VALID;
7418 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7419 u32IntInfo |= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7420 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */, fStepping,
7421 pfIntrState);
7422}
7423
7424
7425/**
7426 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7427 *
7428 * @param pVCpu The cross context virtual CPU structure.
7429 */
7430DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
7431{
7432 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INT_INFO_VALID;
7433 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7434 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7435}
7436
7437
7438/**
7439 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7440 *
7441 * @param pVCpu The cross context virtual CPU structure.
7442 * @param cbInstr The value of RIP that is to be pushed on the guest
7443 * stack.
7444 */
7445DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, uint32_t cbInstr)
7446{
7447 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INT_INFO_VALID;
7448 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7449 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7450}
7451
7452
7453/**
7454 * Injects a general-protection (\#GP) fault into the VM.
7455 *
7456 * @returns Strict VBox status code (i.e. informational status codes too).
7457 * @param pVCpu The cross context virtual CPU structure.
7458 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7459 * mode, i.e. in real-mode it's not valid).
7460 * @param u32ErrorCode The error code associated with the \#GP.
7461 * @param fStepping Whether we're running in
7462 * hmR0VmxRunGuestCodeStep() and should return
7463 * VINF_EM_DBG_STEPPED if the event is injected
7464 * directly (register modified by us, not by
7465 * hardware on VM-entry).
7466 * @param pfIntrState Pointer to the current guest interruptibility-state.
7467 * This interruptibility-state will be updated if
7468 * necessary. This cannot not be NULL.
7469 */
7470DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, bool fErrorCodeValid, uint32_t u32ErrorCode, bool fStepping,
7471 uint32_t *pfIntrState)
7472{
7473 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INT_INFO_VALID;
7474 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7475 if (fErrorCodeValid)
7476 u32IntInfo |= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7477 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */, fStepping,
7478 pfIntrState);
7479}
7480
7481
7482/**
7483 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7484 *
7485 * @param pVCpu The cross context virtual CPU structure.
7486 * @param uVector The software interrupt vector number.
7487 * @param cbInstr The value of RIP that is to be pushed on the guest
7488 * stack.
7489 */
7490DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, uint16_t uVector, uint32_t cbInstr)
7491{
7492 uint32_t u32IntInfo = uVector | VMX_EXIT_INT_INFO_VALID;
7493 if ( uVector == X86_XCPT_BP
7494 || uVector == X86_XCPT_OF)
7495 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7496 else
7497 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7498 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7499}
7500
7501
7502/**
7503 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7504 * stack.
7505 *
7506 * @returns Strict VBox status code (i.e. informational status codes too).
7507 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7508 * @param pVCpu The cross context virtual CPU structure.
7509 * @param uValue The value to push to the guest stack.
7510 */
7511static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
7512{
7513 /*
7514 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7515 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7516 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7517 */
7518 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7519 if (pCtx->sp == 1)
7520 return VINF_EM_RESET;
7521 pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7522 int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
7523 AssertRC(rc);
7524 return rc;
7525}
7526
7527
7528/**
7529 * Injects an event into the guest upon VM-entry by updating the relevant fields
7530 * in the VM-entry area in the VMCS.
7531 *
7532 * @returns Strict VBox status code (i.e. informational status codes too).
7533 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7534 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7535 *
7536 * @param pVCpu The cross context virtual CPU structure.
7537 * @param u64IntInfo The VM-entry interruption-information field.
7538 * @param cbInstr The VM-entry instruction length in bytes (for
7539 * software interrupts, exceptions and privileged
7540 * software exceptions).
7541 * @param u32ErrCode The VM-entry exception error code.
7542 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7543 * @param pfIntrState Pointer to the current guest interruptibility-state.
7544 * This interruptibility-state will be updated if
7545 * necessary. This cannot not be NULL.
7546 * @param fStepping Whether we're running in
7547 * hmR0VmxRunGuestCodeStep() and should return
7548 * VINF_EM_DBG_STEPPED if the event is injected
7549 * directly (register modified by us, not by
7550 * hardware on VM-entry).
7551 */
7552static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7553 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState)
7554{
7555 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7556 AssertMsg(!RT_HI_U32(u64IntInfo), ("%#RX64\n", u64IntInfo));
7557 Assert(pfIntrState);
7558
7559 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7560 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7561 uint32_t const uVector = VMX_EXIT_INT_INFO_VECTOR(u32IntInfo);
7562 uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(u32IntInfo);
7563
7564#ifdef VBOX_STRICT
7565 /*
7566 * Validate the error-code-valid bit for hardware exceptions.
7567 * No error codes for exceptions in real-mode.
7568 *
7569 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7570 */
7571 if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
7572 && !CPUMIsGuestInRealModeEx(pCtx))
7573 {
7574 switch (uVector)
7575 {
7576 case X86_XCPT_PF:
7577 case X86_XCPT_DF:
7578 case X86_XCPT_TS:
7579 case X86_XCPT_NP:
7580 case X86_XCPT_SS:
7581 case X86_XCPT_GP:
7582 case X86_XCPT_AC:
7583 AssertMsg(VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
7584 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7585 RT_FALL_THRU();
7586 default:
7587 break;
7588 }
7589 }
7590#endif
7591
7592 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7593 Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
7594 || !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
7595
7596 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7597
7598 /*
7599 * Hardware interrupts & exceptions cannot be delivered through the software interrupt
7600 * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
7601 * interrupt handler in the (real-mode) guest.
7602 *
7603 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
7604 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7605 */
7606 if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
7607 {
7608 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
7609 {
7610 /*
7611 * For unrestricted execution enabled CPUs running real-mode guests, we must not
7612 * set the deliver-error-code bit.
7613 *
7614 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7615 */
7616 u32IntInfo &= ~VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7617 }
7618 else
7619 {
7620 PVM pVM = pVCpu->CTX_SUFF(pVM);
7621 Assert(PDMVmmDevHeapIsEnabled(pVM));
7622 Assert(pVM->hm.s.vmx.pRealModeTSS);
7623
7624 /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
7625 int rc2 = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_TABLE_MASK | CPUMCTX_EXTRN_RIP
7626 | CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_RFLAGS);
7627 AssertRCReturn(rc2, rc2);
7628
7629 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7630 size_t const cbIdtEntry = sizeof(X86IDTR16);
7631 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
7632 {
7633 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7634 if (uVector == X86_XCPT_DF)
7635 return VINF_EM_RESET;
7636
7637 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7638 if (uVector == X86_XCPT_GP)
7639 return hmR0VmxInjectXcptDF(pVCpu, fStepping, pfIntrState);
7640
7641 /*
7642 * If we're injecting an event with no valid IDT entry, inject a #GP.
7643 * No error codes for exceptions in real-mode.
7644 *
7645 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7646 */
7647 return hmR0VmxInjectXcptGP(pVCpu, false /* fErrCodeValid */, 0 /* u32ErrCode */, fStepping, pfIntrState);
7648 }
7649
7650 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7651 uint16_t uGuestIp = pCtx->ip;
7652 if (uIntType == VMX_EXIT_INT_INFO_TYPE_SW_XCPT)
7653 {
7654 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7655 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7656 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7657 }
7658 else if (uIntType == VMX_EXIT_INT_INFO_TYPE_SW_INT)
7659 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7660
7661 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7662 X86IDTR16 IdtEntry;
7663 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
7664 rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7665 AssertRCReturn(rc2, rc2);
7666
7667 /* Construct the stack frame for the interrupt/exception handler. */
7668 VBOXSTRICTRC rcStrict;
7669 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
7670 if (rcStrict == VINF_SUCCESS)
7671 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
7672 if (rcStrict == VINF_SUCCESS)
7673 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
7674
7675 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7676 if (rcStrict == VINF_SUCCESS)
7677 {
7678 pCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7679 pCtx->rip = IdtEntry.offSel;
7680 pCtx->cs.Sel = IdtEntry.uSel;
7681 pCtx->cs.ValidSel = IdtEntry.uSel;
7682 pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7683 if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
7684 && uVector == X86_XCPT_PF)
7685 pCtx->cr2 = GCPtrFaultAddress;
7686
7687 /* If any other guest-state bits are changed here, make sure to update
7688 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7689 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS | HM_CHANGED_GUEST_CR2
7690 | HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
7691 | HM_CHANGED_GUEST_RSP);
7692
7693 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7694 if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
7695 {
7696 Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
7697 && uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
7698 Log4Func(("Clearing inhibition due to STI\n"));
7699 *pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
7700 }
7701 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7702 u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
7703
7704 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7705 it, if we are returning to ring-3 before executing guest code. */
7706 pVCpu->hm.s.Event.fPending = false;
7707
7708 /* Make hmR0VmxPreRunGuest() return if we're stepping since we've changed cs:rip. */
7709 if (fStepping)
7710 rcStrict = VINF_EM_DBG_STEPPED;
7711 }
7712 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7713 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7714 return rcStrict;
7715 }
7716 }
7717
7718 /* Validate. */
7719 Assert(VMX_EXIT_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7720 Assert(!VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7721 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7722
7723 /* Inject. */
7724 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7725 if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
7726 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7727 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7728 AssertRCReturn(rc, rc);
7729
7730 /* Update CR2. */
7731 if ( VMX_EXIT_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
7732 && uVector == X86_XCPT_PF)
7733 pCtx->cr2 = GCPtrFaultAddress;
7734
7735 Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
7736
7737 return VINF_SUCCESS;
7738}
7739
7740
7741/**
7742 * Clears the interrupt-window exiting control in the VMCS and if necessary
7743 * clears the current event in the VMCS as well.
7744 *
7745 * @returns VBox status code.
7746 * @param pVCpu The cross context virtual CPU structure.
7747 *
7748 * @remarks Use this function only to clear events that have not yet been
7749 * delivered to the guest but are injected in the VMCS!
7750 * @remarks No-long-jump zone!!!
7751 */
7752static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
7753{
7754 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7755 {
7756 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7757 Log4Func(("Cleared interrupt widow\n"));
7758 }
7759
7760 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
7761 {
7762 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
7763 Log4Func(("Cleared interrupt widow\n"));
7764 }
7765}
7766
7767
7768/**
7769 * Enters the VT-x session.
7770 *
7771 * @returns VBox status code.
7772 * @param pVCpu The cross context virtual CPU structure.
7773 * @param pHostCpu Pointer to the global CPU info struct.
7774 */
7775VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu, PHMGLOBALCPUINFO pHostCpu)
7776{
7777 AssertPtr(pVCpu);
7778 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
7779 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7780 RT_NOREF(pHostCpu);
7781
7782 LogFlowFunc(("pVCpu=%p\n", pVCpu));
7783 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7784 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7785
7786#ifdef VBOX_STRICT
7787 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
7788 RTCCUINTREG uHostCR4 = ASMGetCR4();
7789 if (!(uHostCR4 & X86_CR4_VMXE))
7790 {
7791 LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
7792 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7793 }
7794#endif
7795
7796 /*
7797 * Load the VCPU's VMCS as the current (and active) one.
7798 */
7799 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7800 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7801 if (RT_FAILURE(rc))
7802 return rc;
7803
7804 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7805 pVCpu->hm.s.fLeaveDone = false;
7806 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7807
7808 return VINF_SUCCESS;
7809}
7810
7811
7812/**
7813 * The thread-context callback (only on platforms which support it).
7814 *
7815 * @param enmEvent The thread-context event.
7816 * @param pVCpu The cross context virtual CPU structure.
7817 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7818 * @thread EMT(pVCpu)
7819 */
7820VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7821{
7822 NOREF(fGlobalInit);
7823
7824 switch (enmEvent)
7825 {
7826 case RTTHREADCTXEVENT_OUT:
7827 {
7828 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7829 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7830 VMCPU_ASSERT_EMT(pVCpu);
7831
7832 /* No longjmps (logger flushes, locks) in this fragile context. */
7833 VMMRZCallRing3Disable(pVCpu);
7834 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7835
7836 /*
7837 * Restore host-state (FPU, debug etc.)
7838 */
7839 if (!pVCpu->hm.s.fLeaveDone)
7840 {
7841 /*
7842 * Do -not- import the guest-state here as we might already be in the middle of importing
7843 * it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
7844 */
7845 hmR0VmxLeave(pVCpu, false /* fImportState */);
7846 pVCpu->hm.s.fLeaveDone = true;
7847 }
7848
7849 /* Leave HM context, takes care of local init (term). */
7850 int rc = HMR0LeaveCpu(pVCpu);
7851 AssertRC(rc); NOREF(rc);
7852
7853 /* Restore longjmp state. */
7854 VMMRZCallRing3Enable(pVCpu);
7855 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
7856 break;
7857 }
7858
7859 case RTTHREADCTXEVENT_IN:
7860 {
7861 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7862 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7863 VMCPU_ASSERT_EMT(pVCpu);
7864
7865 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7866 VMMRZCallRing3Disable(pVCpu);
7867 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7868
7869 /* Initialize the bare minimum state required for HM. This takes care of
7870 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7871 int rc = hmR0EnterCpu(pVCpu);
7872 AssertRC(rc);
7873 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7874 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7875
7876 /* Load the active VMCS as the current one. */
7877 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7878 {
7879 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7880 AssertRC(rc); NOREF(rc);
7881 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7882 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7883 }
7884 pVCpu->hm.s.fLeaveDone = false;
7885
7886 /* Restore longjmp state. */
7887 VMMRZCallRing3Enable(pVCpu);
7888 break;
7889 }
7890
7891 default:
7892 break;
7893 }
7894}
7895
7896
7897/**
7898 * Exports the host state into the VMCS host-state area.
7899 * Sets up the VM-exit MSR-load area.
7900 *
7901 * The CPU state will be loaded from these fields on every successful VM-exit.
7902 *
7903 * @returns VBox status code.
7904 * @param pVCpu The cross context virtual CPU structure.
7905 *
7906 * @remarks No-long-jump zone!!!
7907 */
7908static int hmR0VmxExportHostState(PVMCPU pVCpu)
7909{
7910 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7911
7912 int rc = VINF_SUCCESS;
7913 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
7914 {
7915 rc = hmR0VmxExportHostControlRegs();
7916 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7917
7918 rc = hmR0VmxExportHostSegmentRegs(pVCpu);
7919 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7920
7921 rc = hmR0VmxExportHostMsrs(pVCpu);
7922 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7923
7924 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
7925 }
7926 return rc;
7927}
7928
7929
7930/**
7931 * Saves the host state in the VMCS host-state.
7932 *
7933 * @returns VBox status code.
7934 * @param pVCpu The cross context virtual CPU structure.
7935 *
7936 * @remarks No-long-jump zone!!!
7937 */
7938VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
7939{
7940 AssertPtr(pVCpu);
7941 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7942
7943 /*
7944 * Export the host state here while entering HM context.
7945 * When thread-context hooks are used, we might get preempted and have to re-save the host
7946 * state but most of the time we won't be, so do it here before we disable interrupts.
7947 */
7948 return hmR0VmxExportHostState(pVCpu);
7949}
7950
7951
7952/**
7953 * Exports the guest state into the VMCS guest-state area.
7954 *
7955 * The will typically be done before VM-entry when the guest-CPU state and the
7956 * VMCS state may potentially be out of sync.
7957 *
7958 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
7959 * VM-entry controls.
7960 * Sets up the appropriate VMX non-root function to execute guest code based on
7961 * the guest CPU mode.
7962 *
7963 * @returns VBox strict status code.
7964 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
7965 * without unrestricted guest access and the VMMDev is not presently
7966 * mapped (e.g. EFI32).
7967 *
7968 * @param pVCpu The cross context virtual CPU structure.
7969 *
7970 * @remarks No-long-jump zone!!!
7971 */
7972static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu)
7973{
7974 AssertPtr(pVCpu);
7975 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7976
7977 LogFlowFunc(("pVCpu=%p\n", pVCpu));
7978
7979 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
7980
7981 /* Determine real-on-v86 mode. */
7982 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7983 if ( !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
7984 && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
7985 {
7986 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7987 }
7988
7989 /*
7990 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7991 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
7992 */
7993 int rc = hmR0VmxSelectVMRunHandler(pVCpu);
7994 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7995
7996 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7997 rc = hmR0VmxExportGuestEntryCtls(pVCpu);
7998 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7999
8000 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8001 rc = hmR0VmxExportGuestExitCtls(pVCpu);
8002 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8003
8004 rc = hmR0VmxExportGuestCR0(pVCpu);
8005 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8006
8007 VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu);
8008 if (rcStrict == VINF_SUCCESS)
8009 { /* likely */ }
8010 else
8011 {
8012 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8013 return rcStrict;
8014 }
8015
8016 rc = hmR0VmxExportGuestSegmentRegs(pVCpu);
8017 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8018
8019 /* This needs to be done after hmR0VmxExportGuestEntryCtls() and hmR0VmxExportGuestExitCtls() as it
8020 may alter controls if we determine we don't have to swap EFER after all. */
8021 rc = hmR0VmxExportGuestMsrs(pVCpu);
8022 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8023
8024 rc = hmR0VmxExportGuestApicTpr(pVCpu);
8025 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8026
8027 rc = hmR0VmxExportGuestXcptIntercepts(pVCpu);
8028 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8029
8030 /* Exporting RFLAGS here is fine, even though RFLAGS.TF might depend on guest debug state which is
8031 not exported here. It is re-evaluated and updated if necessary in hmR0VmxExportSharedState(). */
8032 rc = hmR0VmxExportGuestRip(pVCpu);
8033 rc |= hmR0VmxExportGuestRsp(pVCpu);
8034 rc |= hmR0VmxExportGuestRflags(pVCpu);
8035 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8036
8037 /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
8038 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
8039 | HM_CHANGED_GUEST_CR2
8040 | (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
8041 | HM_CHANGED_GUEST_X87
8042 | HM_CHANGED_GUEST_SSE_AVX
8043 | HM_CHANGED_GUEST_OTHER_XSAVE
8044 | HM_CHANGED_GUEST_XCRx
8045 | HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
8046 | HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
8047 | HM_CHANGED_GUEST_TSC_AUX
8048 | HM_CHANGED_GUEST_OTHER_MSRS
8049 | HM_CHANGED_GUEST_HWVIRT
8050 | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
8051
8052 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
8053 return rc;
8054}
8055
8056
8057/**
8058 * Exports the state shared between the host and guest into the VMCS.
8059 *
8060 * @param pVCpu The cross context virtual CPU structure.
8061 *
8062 * @remarks No-long-jump zone!!!
8063 */
8064static void hmR0VmxExportSharedState(PVMCPU pVCpu)
8065{
8066 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8067 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8068
8069 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
8070 {
8071 int rc = hmR0VmxExportSharedDebugState(pVCpu);
8072 AssertRC(rc);
8073 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
8074
8075 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8076 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
8077 {
8078 rc = hmR0VmxExportGuestRflags(pVCpu);
8079 AssertRC(rc);
8080 }
8081 }
8082
8083 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
8084 {
8085 hmR0VmxLazyLoadGuestMsrs(pVCpu);
8086 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
8087 }
8088
8089 AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
8090 ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8091}
8092
8093
8094/**
8095 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8096 *
8097 * @returns Strict VBox status code (i.e. informational status codes too).
8098 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8099 * without unrestricted guest access and the VMMDev is not presently
8100 * mapped (e.g. EFI32).
8101 *
8102 * @param pVCpu The cross context virtual CPU structure.
8103 *
8104 * @remarks No-long-jump zone!!!
8105 */
8106static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu)
8107{
8108 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8109 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8110 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8111
8112#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8113 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8114#endif
8115
8116 /*
8117 * For many exits it's only RIP that changes and hence try to export it first
8118 * without going through a lot of change flag checks.
8119 */
8120 VBOXSTRICTRC rcStrict;
8121 uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8122 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8123 if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
8124 {
8125 rcStrict = hmR0VmxExportGuestRip(pVCpu);
8126 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8127 { /* likely */}
8128 else
8129 AssertMsgFailedReturn(("hmR0VmxExportGuestRip failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8130 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
8131 }
8132 else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8133 {
8134 rcStrict = hmR0VmxExportGuestState(pVCpu);
8135 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8136 { /* likely */}
8137 else
8138 {
8139 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("hmR0VmxExportGuestState failed! rc=%Rrc\n",
8140 VBOXSTRICTRC_VAL(rcStrict)));
8141 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8142 return rcStrict;
8143 }
8144 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
8145 }
8146 else
8147 rcStrict = VINF_SUCCESS;
8148
8149#ifdef VBOX_STRICT
8150 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8151 fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8152 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8153 AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
8154 ("fCtxChanged=%#RX64\n", fCtxChanged));
8155#endif
8156 return rcStrict;
8157}
8158
8159
8160/**
8161 * Does the preparations before executing guest code in VT-x.
8162 *
8163 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8164 * recompiler/IEM. We must be cautious what we do here regarding committing
8165 * guest-state information into the VMCS assuming we assuredly execute the
8166 * guest in VT-x mode.
8167 *
8168 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8169 * the common-state (TRPM/forceflags), we must undo those changes so that the
8170 * recompiler/IEM can (and should) use them when it resumes guest execution.
8171 * Otherwise such operations must be done when we can no longer exit to ring-3.
8172 *
8173 * @returns Strict VBox status code (i.e. informational status codes too).
8174 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8175 * have been disabled.
8176 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8177 * double-fault into the guest.
8178 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8179 * dispatched directly.
8180 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8181 *
8182 * @param pVCpu The cross context virtual CPU structure.
8183 * @param pVmxTransient Pointer to the VMX transient structure.
8184 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8185 * us ignore some of the reasons for returning to
8186 * ring-3, and return VINF_EM_DBG_STEPPED if event
8187 * dispatching took place.
8188 */
8189static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
8190{
8191 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8192
8193#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8194 PGMRZDynMapFlushAutoSet(pVCpu);
8195#endif
8196
8197 /* Check force flag actions that might require us to go back to ring-3. */
8198 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
8199 if (rcStrict == VINF_SUCCESS)
8200 { /* FFs doesn't get set all the time. */ }
8201 else
8202 return rcStrict;
8203
8204 /*
8205 * Setup the virtualized-APIC accesses.
8206 *
8207 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8208 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8209 *
8210 * This is the reason we do it here and not in hmR0VmxExportGuestState().
8211 */
8212 PVM pVM = pVCpu->CTX_SUFF(pVM);
8213 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8214 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8215 && PDMHasApic(pVM))
8216 {
8217 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8218 Assert(u64MsrApicBase);
8219 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8220
8221 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8222
8223 /* Unalias any existing mapping. */
8224 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8225 AssertRCReturn(rc, rc);
8226
8227 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8228 Log4Func(("Mapped HC APIC-access page at %#RGp\n", GCPhysApicBase));
8229 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8230 AssertRCReturn(rc, rc);
8231
8232 /* Update the per-VCPU cache of the APIC base MSR. */
8233 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8234 }
8235
8236 if (TRPMHasTrap(pVCpu))
8237 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8238 uint32_t fIntrState = hmR0VmxEvaluatePendingEvent(pVCpu);
8239
8240 /*
8241 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
8242 * needs to be done with longjmps or interrupts + preemption enabled. Event injection might
8243 * also result in triple-faulting the VM.
8244 */
8245 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, fIntrState, fStepping);
8246 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8247 { /* likely */ }
8248 else
8249 {
8250 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8251 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8252 return rcStrict;
8253 }
8254
8255 /*
8256 * A longjump might result in importing CR3 even for VM-exits that don't necessarily
8257 * import CR3 themselves. We will need to update them here, as even as late as the above
8258 * hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
8259 * the below force flags to be set.
8260 */
8261 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8262 {
8263 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
8264 int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8265 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
8266 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
8267 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8268 }
8269 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8270 {
8271 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8272 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8273 }
8274
8275 /*
8276 * No longjmps to ring-3 from this point on!!!
8277 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8278 * This also disables flushing of the R0-logger instance (if any).
8279 */
8280 VMMRZCallRing3Disable(pVCpu);
8281
8282 /*
8283 * Export the guest state bits.
8284 *
8285 * We cannot perform longjmps while loading the guest state because we do not preserve the
8286 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8287 * CPU migration.
8288 *
8289 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8290 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8291 * Hence, loading of the guest state needs to be done -after- injection of events.
8292 */
8293 rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu);
8294 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8295 { /* likely */ }
8296 else
8297 {
8298 VMMRZCallRing3Enable(pVCpu);
8299 return rcStrict;
8300 }
8301
8302 /*
8303 * We disable interrupts so that we don't miss any interrupts that would flag preemption
8304 * (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
8305 * preemption disabled for a while. Since this is purly to aid the
8306 * RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
8307 * disable interrupt on NT.
8308 *
8309 * We need to check for force-flags that could've possible been altered since we last
8310 * checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
8311 * see @bugref{6398}).
8312 *
8313 * We also check a couple of other force-flags as a last opportunity to get the EMT back
8314 * to ring-3 before executing guest code.
8315 */
8316 pVmxTransient->fEFlags = ASMIntDisableFlags();
8317
8318 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8319 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8320 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8321 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8322 {
8323 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8324 {
8325 pVCpu->hm.s.Event.fPending = false;
8326
8327 /*
8328 * We've injected any pending events. This is really the point of no return (to ring-3).
8329 *
8330 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
8331 * returns from this function, so don't enable them here.
8332 */
8333 return VINF_SUCCESS;
8334 }
8335
8336 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
8337 rcStrict = VINF_EM_RAW_INTERRUPT;
8338 }
8339 else
8340 {
8341 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8342 rcStrict = VINF_EM_RAW_TO_R3;
8343 }
8344
8345 ASMSetFlags(pVmxTransient->fEFlags);
8346 VMMRZCallRing3Enable(pVCpu);
8347
8348 return rcStrict;
8349}
8350
8351
8352/**
8353 * Prepares to run guest code in VT-x and we've committed to doing so. This
8354 * means there is no backing out to ring-3 or anywhere else at this
8355 * point.
8356 *
8357 * @param pVCpu The cross context virtual CPU structure.
8358 * @param pVmxTransient Pointer to the VMX transient structure.
8359 *
8360 * @remarks Called with preemption disabled.
8361 * @remarks No-long-jump zone!!!
8362 */
8363static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
8364{
8365 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8366 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8367 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8368
8369 /*
8370 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8371 */
8372 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8373 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8374
8375 PVM pVM = pVCpu->CTX_SUFF(pVM);
8376 if (!CPUMIsGuestFPUStateActive(pVCpu))
8377 {
8378 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8379 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8380 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT;
8381 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8382 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
8383 }
8384
8385 /*
8386 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8387 */
8388 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8389 && pVCpu->hm.s.vmx.cMsrs > 0)
8390 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8391
8392 /*
8393 * Re-save the host state bits as we may've been preempted (only happens when
8394 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8395 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8396 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8397 * See @bugref{8432}.
8398 */
8399 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8400 {
8401 int rc = hmR0VmxExportHostState(pVCpu);
8402 AssertRC(rc);
8403 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
8404 }
8405 Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
8406
8407 /*
8408 * Export the state shared between host and guest (FPU, debug, lazy MSRs).
8409 */
8410 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
8411 hmR0VmxExportSharedState(pVCpu);
8412 AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8413
8414 /* Store status of the shared guest-host state at the time of VM-entry. */
8415#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8416 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
8417 {
8418 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8419 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8420 }
8421 else
8422#endif
8423 {
8424 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8425 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8426 }
8427
8428 /*
8429 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8430 */
8431 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8432 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8433
8434 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8435 RTCPUID idCurrentCpu = pCpu->idCpu;
8436 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8437 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8438 {
8439 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8440 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8441 }
8442
8443 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8444 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8445 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8446 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8447
8448 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8449
8450 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8451 to start executing. */
8452
8453 /*
8454 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8455 */
8456 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8457 {
8458 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8459 {
8460 bool fMsrUpdated;
8461 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
8462 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8463 &fMsrUpdated);
8464 AssertRC(rc2);
8465 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8466 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8467 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8468 }
8469 else
8470 {
8471 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8472 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8473 }
8474 }
8475
8476 if (pVM->cpum.ro.GuestFeatures.fIbrs)
8477 {
8478 bool fMsrUpdated;
8479 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_OTHER_MSRS);
8480 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
8481 &fMsrUpdated);
8482 AssertRC(rc2);
8483 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8484 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8485 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8486 }
8487
8488#ifdef VBOX_STRICT
8489 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8490 hmR0VmxCheckHostEferMsr(pVCpu);
8491 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8492#endif
8493#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8494 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
8495 {
8496 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
8497 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8498 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8499 }
8500#endif
8501}
8502
8503
8504/**
8505 * Performs some essential restoration of state after running guest code in
8506 * VT-x.
8507 *
8508 * @param pVCpu The cross context virtual CPU structure.
8509 * @param pVmxTransient Pointer to the VMX transient structure.
8510 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8511 *
8512 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8513 *
8514 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8515 * unconditionally when it is safe to do so.
8516 */
8517static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8518{
8519 uint64_t const uHostTsc = ASMReadTSC();
8520 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8521
8522 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8523 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8524 pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8525 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8526 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8527 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8528
8529 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8530 TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TscOffset);
8531
8532 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
8533 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8534 Assert(!ASMIntAreEnabled());
8535 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8536
8537#if HC_ARCH_BITS == 64
8538 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8539#endif
8540#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8541 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8542 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8543 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8544#else
8545 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8546#endif
8547#ifdef VBOX_STRICT
8548 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8549#endif
8550 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8551
8552 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8553 uint32_t uExitReason;
8554 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8555 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8556 AssertRC(rc);
8557 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8558 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INT_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8559
8560 if (rcVMRun == VINF_SUCCESS)
8561 {
8562 /*
8563 * Update the VM-exit history array here even if the VM-entry failed due to:
8564 * - Invalid guest state.
8565 * - MSR loading.
8566 * - Machine-check event.
8567 *
8568 * In any of the above cases we will still have a "valid" VM-exit reason
8569 * despite @a fVMEntryFailed being false.
8570 *
8571 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8572 *
8573 * Note! We don't have CS or RIP at this point. Will probably address that later
8574 * by amending the history entry added here.
8575 */
8576 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
8577 UINT64_MAX, uHostTsc);
8578
8579 if (!pVmxTransient->fVMEntryFailed)
8580 {
8581 VMMRZCallRing3Enable(pVCpu);
8582
8583 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8584 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8585
8586#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8587 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
8588 AssertRC(rc);
8589#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8590 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_RFLAGS);
8591 AssertRC(rc);
8592#else
8593 /*
8594 * Import the guest-interruptibility state always as we need it while evaluating
8595 * injecting events on re-entry.
8596 *
8597 * We don't import CR0 (when Unrestricted guest execution is unavailable) despite
8598 * checking for real-mode while exporting the state because all bits that cause
8599 * mode changes wrt CR0 are intercepted.
8600 */
8601 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
8602 AssertRC(rc);
8603#endif
8604
8605 /*
8606 * Sync the TPR shadow with our APIC state.
8607 */
8608 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8609 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8610 {
8611 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8612 AssertRC(rc);
8613 ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8614 }
8615
8616 return;
8617 }
8618 }
8619 else
8620 Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
8621
8622 VMMRZCallRing3Enable(pVCpu);
8623}
8624
8625
8626/**
8627 * Runs the guest code using VT-x the normal way.
8628 *
8629 * @returns VBox status code.
8630 * @param pVCpu The cross context virtual CPU structure.
8631 *
8632 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8633 */
8634static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu)
8635{
8636 VMXTRANSIENT VmxTransient;
8637 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8638 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8639 uint32_t cLoops = 0;
8640
8641 for (;; cLoops++)
8642 {
8643 Assert(!HMR0SuspendPending());
8644 HMVMX_ASSERT_CPU_SAFE(pVCpu);
8645
8646 /* Preparatory work for running guest code, this may force us to return
8647 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8648 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8649 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
8650 if (rcStrict != VINF_SUCCESS)
8651 break;
8652
8653 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
8654 int rcRun = hmR0VmxRunGuest(pVCpu);
8655
8656 /* Restore any residual host-state and save any bits shared between host
8657 and guest into the guest-CPU state. Re-enables interrupts! */
8658 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
8659
8660 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8661 if (RT_SUCCESS(rcRun))
8662 { /* very likely */ }
8663 else
8664 {
8665 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
8666 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
8667 return rcRun;
8668 }
8669
8670 /* Profile the VM-exit. */
8671 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8672 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8673 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8674 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
8675 HMVMX_START_EXIT_DISPATCH_PROF();
8676
8677 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
8678
8679 /* Handle the VM-exit. */
8680#ifdef HMVMX_USE_FUNCTION_TABLE
8681 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
8682#else
8683 rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient, VmxTransient.uExitReason);
8684#endif
8685 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
8686 if (rcStrict == VINF_SUCCESS)
8687 {
8688 if (cLoops <= pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
8689 continue; /* likely */
8690 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8691 rcStrict = VINF_EM_RAW_INTERRUPT;
8692 }
8693 break;
8694 }
8695
8696 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8697 return rcStrict;
8698}
8699
8700
8701
8702/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8703 * probes.
8704 *
8705 * The following few functions and associated structure contains the bloat
8706 * necessary for providing detailed debug events and dtrace probes as well as
8707 * reliable host side single stepping. This works on the principle of
8708 * "subclassing" the normal execution loop and workers. We replace the loop
8709 * method completely and override selected helpers to add necessary adjustments
8710 * to their core operation.
8711 *
8712 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8713 * any performance for debug and analysis features.
8714 *
8715 * @{
8716 */
8717
8718/**
8719 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
8720 * the debug run loop.
8721 */
8722typedef struct VMXRUNDBGSTATE
8723{
8724 /** The RIP we started executing at. This is for detecting that we stepped. */
8725 uint64_t uRipStart;
8726 /** The CS we started executing with. */
8727 uint16_t uCsStart;
8728
8729 /** Whether we've actually modified the 1st execution control field. */
8730 bool fModifiedProcCtls : 1;
8731 /** Whether we've actually modified the 2nd execution control field. */
8732 bool fModifiedProcCtls2 : 1;
8733 /** Whether we've actually modified the exception bitmap. */
8734 bool fModifiedXcptBitmap : 1;
8735
8736 /** We desire the modified the CR0 mask to be cleared. */
8737 bool fClearCr0Mask : 1;
8738 /** We desire the modified the CR4 mask to be cleared. */
8739 bool fClearCr4Mask : 1;
8740 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
8741 uint32_t fCpe1Extra;
8742 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
8743 uint32_t fCpe1Unwanted;
8744 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
8745 uint32_t fCpe2Extra;
8746 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
8747 uint32_t bmXcptExtra;
8748 /** The sequence number of the Dtrace provider settings the state was
8749 * configured against. */
8750 uint32_t uDtraceSettingsSeqNo;
8751 /** VM-exits to check (one bit per VM-exit). */
8752 uint32_t bmExitsToCheck[3];
8753
8754 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
8755 uint32_t fProcCtlsInitial;
8756 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
8757 uint32_t fProcCtls2Initial;
8758 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
8759 uint32_t bmXcptInitial;
8760} VMXRUNDBGSTATE;
8761AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
8762typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
8763
8764
8765/**
8766 * Initializes the VMXRUNDBGSTATE structure.
8767 *
8768 * @param pVCpu The cross context virtual CPU structure of the
8769 * calling EMT.
8770 * @param pDbgState The structure to initialize.
8771 */
8772static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
8773{
8774 pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
8775 pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
8776
8777 pDbgState->fModifiedProcCtls = false;
8778 pDbgState->fModifiedProcCtls2 = false;
8779 pDbgState->fModifiedXcptBitmap = false;
8780 pDbgState->fClearCr0Mask = false;
8781 pDbgState->fClearCr4Mask = false;
8782 pDbgState->fCpe1Extra = 0;
8783 pDbgState->fCpe1Unwanted = 0;
8784 pDbgState->fCpe2Extra = 0;
8785 pDbgState->bmXcptExtra = 0;
8786 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
8787 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
8788 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
8789}
8790
8791
8792/**
8793 * Updates the VMSC fields with changes requested by @a pDbgState.
8794 *
8795 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
8796 * immediately before executing guest code, i.e. when interrupts are disabled.
8797 * We don't check status codes here as we cannot easily assert or return in the
8798 * latter case.
8799 *
8800 * @param pVCpu The cross context virtual CPU structure.
8801 * @param pDbgState The debug state.
8802 */
8803static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
8804{
8805 /*
8806 * Ensure desired flags in VMCS control fields are set.
8807 * (Ignoring write failure here, as we're committed and it's just debug extras.)
8808 *
8809 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
8810 * there should be no stale data in pCtx at this point.
8811 */
8812 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
8813 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
8814 {
8815 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
8816 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
8817 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8818 Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
8819 pDbgState->fModifiedProcCtls = true;
8820 }
8821
8822 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
8823 {
8824 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
8825 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
8826 Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
8827 pDbgState->fModifiedProcCtls2 = true;
8828 }
8829
8830 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
8831 {
8832 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
8833 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8834 Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
8835 pDbgState->fModifiedXcptBitmap = true;
8836 }
8837
8838 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32Cr0Mask != 0)
8839 {
8840 pVCpu->hm.s.vmx.u32Cr0Mask = 0;
8841 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
8842 Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
8843 }
8844
8845 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32Cr4Mask != 0)
8846 {
8847 pVCpu->hm.s.vmx.u32Cr4Mask = 0;
8848 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
8849 Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
8850 }
8851}
8852
8853
8854/**
8855 * Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
8856 * re-entry next time around.
8857 *
8858 * @returns Strict VBox status code (i.e. informational status codes too).
8859 * @param pVCpu The cross context virtual CPU structure.
8860 * @param pDbgState The debug state.
8861 * @param rcStrict The return code from executing the guest using single
8862 * stepping.
8863 */
8864static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
8865{
8866 /*
8867 * Restore VM-exit control settings as we may not reenter this function the
8868 * next time around.
8869 */
8870 /* We reload the initial value, trigger what we can of recalculations the
8871 next time around. From the looks of things, that's all that's required atm. */
8872 if (pDbgState->fModifiedProcCtls)
8873 {
8874 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
8875 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
8876 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
8877 AssertRCReturn(rc2, rc2);
8878 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
8879 }
8880
8881 /* We're currently the only ones messing with this one, so just restore the
8882 cached value and reload the field. */
8883 if ( pDbgState->fModifiedProcCtls2
8884 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
8885 {
8886 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
8887 AssertRCReturn(rc2, rc2);
8888 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
8889 }
8890
8891 /* If we've modified the exception bitmap, we restore it and trigger
8892 reloading and partial recalculation the next time around. */
8893 if (pDbgState->fModifiedXcptBitmap)
8894 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
8895
8896 return rcStrict;
8897}
8898
8899
8900/**
8901 * Configures VM-exit controls for current DBGF and DTrace settings.
8902 *
8903 * This updates @a pDbgState and the VMCS execution control fields to reflect
8904 * the necessary VM-exits demanded by DBGF and DTrace.
8905 *
8906 * @param pVCpu The cross context virtual CPU structure.
8907 * @param pDbgState The debug state.
8908 * @param pVmxTransient Pointer to the VMX transient structure. May update
8909 * fUpdateTscOffsettingAndPreemptTimer.
8910 */
8911static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
8912{
8913 /*
8914 * Take down the dtrace serial number so we can spot changes.
8915 */
8916 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
8917 ASMCompilerBarrier();
8918
8919 /*
8920 * We'll rebuild most of the middle block of data members (holding the
8921 * current settings) as we go along here, so start by clearing it all.
8922 */
8923 pDbgState->bmXcptExtra = 0;
8924 pDbgState->fCpe1Extra = 0;
8925 pDbgState->fCpe1Unwanted = 0;
8926 pDbgState->fCpe2Extra = 0;
8927 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
8928 pDbgState->bmExitsToCheck[i] = 0;
8929
8930 /*
8931 * Software interrupts (INT XXh) - no idea how to trigger these...
8932 */
8933 PVM pVM = pVCpu->CTX_SUFF(pVM);
8934 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
8935 || VBOXVMM_INT_SOFTWARE_ENABLED())
8936 {
8937 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
8938 }
8939
8940 /*
8941 * INT3 breakpoints - triggered by #BP exceptions.
8942 */
8943 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
8944 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
8945
8946 /*
8947 * Exception bitmap and XCPT events+probes.
8948 */
8949 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
8950 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
8951 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
8952
8953 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
8954 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
8955 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
8956 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
8957 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
8958 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
8959 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
8960 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
8961 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
8962 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
8963 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
8964 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
8965 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
8966 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
8967 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
8968 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
8969 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
8970 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
8971
8972 if (pDbgState->bmXcptExtra)
8973 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
8974
8975 /*
8976 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
8977 *
8978 * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
8979 * So, when adding/changing/removing please don't forget to update it.
8980 *
8981 * Some of the macros are picking up local variables to save horizontal space,
8982 * (being able to see it in a table is the lesser evil here).
8983 */
8984#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
8985 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
8986 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
8987#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
8988 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
8989 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
8990 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
8991 } else do { } while (0)
8992#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
8993 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
8994 { \
8995 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
8996 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
8997 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
8998 } else do { } while (0)
8999#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9000 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9001 { \
9002 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9003 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9004 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9005 } else do { } while (0)
9006#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9007 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9008 { \
9009 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9010 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9011 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9012 } else do { } while (0)
9013
9014 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9015 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9016 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9017 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9018 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9019
9020 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9021 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9022 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9023 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9024 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9025 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9026 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9027 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9028 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9029 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9030 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9031 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9032 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9033 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9034 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9035 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9036 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9037 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9038 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9039 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9040 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9041 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9042 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9043 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9044 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9045 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9046 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9047 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9048 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9049 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9050 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9051 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9052 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9053 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9054 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9055 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9056
9057 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9058 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9059 {
9060 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_APIC_TPR);
9061 AssertRC(rc);
9062
9063#if 0 /** @todo fix me */
9064 pDbgState->fClearCr0Mask = true;
9065 pDbgState->fClearCr4Mask = true;
9066#endif
9067 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9068 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9069 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9070 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9071 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9072 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9073 require clearing here and in the loop if we start using it. */
9074 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9075 }
9076 else
9077 {
9078 if (pDbgState->fClearCr0Mask)
9079 {
9080 pDbgState->fClearCr0Mask = false;
9081 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
9082 }
9083 if (pDbgState->fClearCr4Mask)
9084 {
9085 pDbgState->fClearCr4Mask = false;
9086 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
9087 }
9088 }
9089 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9090 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9091
9092 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9093 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9094 {
9095 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9096 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9097 }
9098 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9099 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9100
9101 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9102 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9103 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9104 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9105 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9106 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9107 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9108 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9109#if 0 /** @todo too slow, fix handler. */
9110 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9111#endif
9112 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9113
9114 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9115 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9116 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9117 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9118 {
9119 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9120 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9121 }
9122 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9123 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9124 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9125 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9126
9127 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9128 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9129 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9130 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9131 {
9132 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9133 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9134 }
9135 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9136 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9137 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9138 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9139
9140 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9141 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9142 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9143 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9144 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9145 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9146 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9147 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9148 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9149 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9150 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9151 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9152 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9153 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9154 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9155 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9156 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9157 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9158 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9159 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9160 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9161 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9162
9163#undef IS_EITHER_ENABLED
9164#undef SET_ONLY_XBM_IF_EITHER_EN
9165#undef SET_CPE1_XBM_IF_EITHER_EN
9166#undef SET_CPEU_XBM_IF_EITHER_EN
9167#undef SET_CPE2_XBM_IF_EITHER_EN
9168
9169 /*
9170 * Sanitize the control stuff.
9171 */
9172 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
9173 if (pDbgState->fCpe2Extra)
9174 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9175 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
9176 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0;
9177 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9178 {
9179 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9180 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9181 }
9182
9183 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9184 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9185 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9186 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9187}
9188
9189
9190/**
9191 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9192 * appropriate.
9193 *
9194 * The caller has checked the VM-exit against the
9195 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9196 * already, so we don't have to do that either.
9197 *
9198 * @returns Strict VBox status code (i.e. informational status codes too).
9199 * @param pVCpu The cross context virtual CPU structure.
9200 * @param pVmxTransient Pointer to the VMX-transient structure.
9201 * @param uExitReason The VM-exit reason.
9202 *
9203 * @remarks The name of this function is displayed by dtrace, so keep it short
9204 * and to the point. No longer than 33 chars long, please.
9205 */
9206static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9207{
9208 /*
9209 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9210 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9211 *
9212 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9213 * does. Must add/change/remove both places. Same ordering, please.
9214 *
9215 * Added/removed events must also be reflected in the next section
9216 * where we dispatch dtrace events.
9217 */
9218 bool fDtrace1 = false;
9219 bool fDtrace2 = false;
9220 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9221 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9222 uint32_t uEventArg = 0;
9223#define SET_EXIT(a_EventSubName) \
9224 do { \
9225 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9226 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9227 } while (0)
9228#define SET_BOTH(a_EventSubName) \
9229 do { \
9230 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9231 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9232 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9233 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9234 } while (0)
9235 switch (uExitReason)
9236 {
9237 case VMX_EXIT_MTF:
9238 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9239
9240 case VMX_EXIT_XCPT_OR_NMI:
9241 {
9242 uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9243 switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
9244 {
9245 case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
9246 case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
9247 case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
9248 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9249 {
9250 if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
9251 {
9252 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9253 uEventArg = pVmxTransient->uExitIntErrorCode;
9254 }
9255 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9256 switch (enmEvent1)
9257 {
9258 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9259 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9260 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9261 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9262 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9263 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9264 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9265 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9266 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9267 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9268 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9269 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9270 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9271 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9272 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9273 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9274 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9275 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9276 default: break;
9277 }
9278 }
9279 else
9280 AssertFailed();
9281 break;
9282
9283 case VMX_EXIT_INT_INFO_TYPE_SW_INT:
9284 uEventArg = idxVector;
9285 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9286 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9287 break;
9288 }
9289 break;
9290 }
9291
9292 case VMX_EXIT_TRIPLE_FAULT:
9293 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9294 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9295 break;
9296 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9297 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9298 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9299 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9300 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9301
9302 /* Instruction specific VM-exits: */
9303 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9304 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9305 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9306 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9307 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9308 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9309 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9310 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9311 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9312 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9313 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9314 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9315 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9316 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9317 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9318 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9319 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9320 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9321 case VMX_EXIT_MOV_CRX:
9322 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9323 if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
9324 SET_BOTH(CRX_READ);
9325 else
9326 SET_BOTH(CRX_WRITE);
9327 uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQualification);
9328 break;
9329 case VMX_EXIT_MOV_DRX:
9330 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9331 if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification)
9332 == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
9333 SET_BOTH(DRX_READ);
9334 else
9335 SET_BOTH(DRX_WRITE);
9336 uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification);
9337 break;
9338 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9339 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9340 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9341 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9342 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9343 case VMX_EXIT_XDTR_ACCESS:
9344 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9345 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9346 {
9347 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9348 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9349 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9350 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9351 }
9352 break;
9353
9354 case VMX_EXIT_TR_ACCESS:
9355 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9356 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9357 {
9358 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9359 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9360 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9361 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9362 }
9363 break;
9364
9365 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9366 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9367 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9368 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9369 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9370 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9371 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9372 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9373 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9374 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9375 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9376
9377 /* Events that aren't relevant at this point. */
9378 case VMX_EXIT_EXT_INT:
9379 case VMX_EXIT_INT_WINDOW:
9380 case VMX_EXIT_NMI_WINDOW:
9381 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9382 case VMX_EXIT_PREEMPT_TIMER:
9383 case VMX_EXIT_IO_INSTR:
9384 break;
9385
9386 /* Errors and unexpected events. */
9387 case VMX_EXIT_INIT_SIGNAL:
9388 case VMX_EXIT_SIPI:
9389 case VMX_EXIT_IO_SMI:
9390 case VMX_EXIT_SMI:
9391 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9392 case VMX_EXIT_ERR_MSR_LOAD:
9393 case VMX_EXIT_ERR_MACHINE_CHECK:
9394 break;
9395
9396 default:
9397 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9398 break;
9399 }
9400#undef SET_BOTH
9401#undef SET_EXIT
9402
9403 /*
9404 * Dtrace tracepoints go first. We do them here at once so we don't
9405 * have to copy the guest state saving and stuff a few dozen times.
9406 * Down side is that we've got to repeat the switch, though this time
9407 * we use enmEvent since the probes are a subset of what DBGF does.
9408 */
9409 if (fDtrace1 || fDtrace2)
9410 {
9411 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9412 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9413 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9414 switch (enmEvent1)
9415 {
9416 /** @todo consider which extra parameters would be helpful for each probe. */
9417 case DBGFEVENT_END: break;
9418 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
9419 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
9420 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
9421 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
9422 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
9423 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
9424 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
9425 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
9426 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
9427 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
9428 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
9429 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
9430 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
9431 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
9432 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
9433 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
9434 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
9435 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
9436 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9437 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9438 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
9439 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
9440 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
9441 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
9442 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
9443 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
9444 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
9445 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9446 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9447 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9448 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9449 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9450 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
9451 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9452 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
9453 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
9454 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
9455 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
9456 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
9457 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
9458 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
9459 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
9460 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
9461 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
9462 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
9463 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
9464 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
9465 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
9466 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
9467 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
9468 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
9469 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
9470 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
9471 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
9472 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
9473 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
9474 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
9475 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
9476 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
9477 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
9478 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
9479 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
9480 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
9481 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
9482 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
9483 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
9484 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9485 }
9486 switch (enmEvent2)
9487 {
9488 /** @todo consider which extra parameters would be helpful for each probe. */
9489 case DBGFEVENT_END: break;
9490 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
9491 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9492 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
9493 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
9494 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
9495 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
9496 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
9497 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
9498 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
9499 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9500 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9501 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9502 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9503 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9504 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
9505 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9506 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
9507 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
9508 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
9509 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
9510 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
9511 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
9512 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
9513 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
9514 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
9515 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
9516 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
9517 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
9518 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
9519 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
9520 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
9521 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
9522 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
9523 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
9524 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
9525 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
9526 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
9527 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
9528 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
9529 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
9530 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
9531 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
9532 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
9533 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
9534 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
9535 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
9536 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
9537 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
9538 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
9539 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
9540 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
9541 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
9542 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9543 }
9544 }
9545
9546 /*
9547 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9548 * the DBGF call will do a full check).
9549 *
9550 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9551 * Note! If we have to events, we prioritize the first, i.e. the instruction
9552 * one, in order to avoid event nesting.
9553 */
9554 PVM pVM = pVCpu->CTX_SUFF(pVM);
9555 if ( enmEvent1 != DBGFEVENT_END
9556 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9557 {
9558 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9559 if (rcStrict != VINF_SUCCESS)
9560 return rcStrict;
9561 }
9562 else if ( enmEvent2 != DBGFEVENT_END
9563 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9564 {
9565 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9566 if (rcStrict != VINF_SUCCESS)
9567 return rcStrict;
9568 }
9569
9570 return VINF_SUCCESS;
9571}
9572
9573
9574/**
9575 * Single-stepping VM-exit filtering.
9576 *
9577 * This is preprocessing the VM-exits and deciding whether we've gotten far
9578 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9579 * handling is performed.
9580 *
9581 * @returns Strict VBox status code (i.e. informational status codes too).
9582 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9583 * @param pVmxTransient Pointer to the VMX-transient structure.
9584 * @param pDbgState The debug state.
9585 */
9586DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
9587{
9588 /*
9589 * Expensive (saves context) generic dtrace VM-exit probe.
9590 */
9591 uint32_t const uExitReason = pVmxTransient->uExitReason;
9592 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9593 { /* more likely */ }
9594 else
9595 {
9596 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9597 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9598 AssertRC(rc);
9599 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9600 }
9601
9602 /*
9603 * Check for host NMI, just to get that out of the way.
9604 */
9605 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9606 { /* normally likely */ }
9607 else
9608 {
9609 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9610 AssertRCReturn(rc2, rc2);
9611 uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
9612 if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
9613 return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
9614 }
9615
9616 /*
9617 * Check for single stepping event if we're stepping.
9618 */
9619 if (pVCpu->hm.s.fSingleInstruction)
9620 {
9621 switch (uExitReason)
9622 {
9623 case VMX_EXIT_MTF:
9624 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9625
9626 /* Various events: */
9627 case VMX_EXIT_XCPT_OR_NMI:
9628 case VMX_EXIT_EXT_INT:
9629 case VMX_EXIT_TRIPLE_FAULT:
9630 case VMX_EXIT_INT_WINDOW:
9631 case VMX_EXIT_NMI_WINDOW:
9632 case VMX_EXIT_TASK_SWITCH:
9633 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9634 case VMX_EXIT_APIC_ACCESS:
9635 case VMX_EXIT_EPT_VIOLATION:
9636 case VMX_EXIT_EPT_MISCONFIG:
9637 case VMX_EXIT_PREEMPT_TIMER:
9638
9639 /* Instruction specific VM-exits: */
9640 case VMX_EXIT_CPUID:
9641 case VMX_EXIT_GETSEC:
9642 case VMX_EXIT_HLT:
9643 case VMX_EXIT_INVD:
9644 case VMX_EXIT_INVLPG:
9645 case VMX_EXIT_RDPMC:
9646 case VMX_EXIT_RDTSC:
9647 case VMX_EXIT_RSM:
9648 case VMX_EXIT_VMCALL:
9649 case VMX_EXIT_VMCLEAR:
9650 case VMX_EXIT_VMLAUNCH:
9651 case VMX_EXIT_VMPTRLD:
9652 case VMX_EXIT_VMPTRST:
9653 case VMX_EXIT_VMREAD:
9654 case VMX_EXIT_VMRESUME:
9655 case VMX_EXIT_VMWRITE:
9656 case VMX_EXIT_VMXOFF:
9657 case VMX_EXIT_VMXON:
9658 case VMX_EXIT_MOV_CRX:
9659 case VMX_EXIT_MOV_DRX:
9660 case VMX_EXIT_IO_INSTR:
9661 case VMX_EXIT_RDMSR:
9662 case VMX_EXIT_WRMSR:
9663 case VMX_EXIT_MWAIT:
9664 case VMX_EXIT_MONITOR:
9665 case VMX_EXIT_PAUSE:
9666 case VMX_EXIT_XDTR_ACCESS:
9667 case VMX_EXIT_TR_ACCESS:
9668 case VMX_EXIT_INVEPT:
9669 case VMX_EXIT_RDTSCP:
9670 case VMX_EXIT_INVVPID:
9671 case VMX_EXIT_WBINVD:
9672 case VMX_EXIT_XSETBV:
9673 case VMX_EXIT_RDRAND:
9674 case VMX_EXIT_INVPCID:
9675 case VMX_EXIT_VMFUNC:
9676 case VMX_EXIT_RDSEED:
9677 case VMX_EXIT_XSAVES:
9678 case VMX_EXIT_XRSTORS:
9679 {
9680 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
9681 AssertRCReturn(rc, rc);
9682 if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
9683 || pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
9684 return VINF_EM_DBG_STEPPED;
9685 break;
9686 }
9687
9688 /* Errors and unexpected events: */
9689 case VMX_EXIT_INIT_SIGNAL:
9690 case VMX_EXIT_SIPI:
9691 case VMX_EXIT_IO_SMI:
9692 case VMX_EXIT_SMI:
9693 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9694 case VMX_EXIT_ERR_MSR_LOAD:
9695 case VMX_EXIT_ERR_MACHINE_CHECK:
9696 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9697 break;
9698
9699 default:
9700 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9701 break;
9702 }
9703 }
9704
9705 /*
9706 * Check for debugger event breakpoints and dtrace probes.
9707 */
9708 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9709 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9710 {
9711 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
9712 if (rcStrict != VINF_SUCCESS)
9713 return rcStrict;
9714 }
9715
9716 /*
9717 * Normal processing.
9718 */
9719#ifdef HMVMX_USE_FUNCTION_TABLE
9720 return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
9721#else
9722 return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
9723#endif
9724}
9725
9726
9727/**
9728 * Single steps guest code using VT-x.
9729 *
9730 * @returns Strict VBox status code (i.e. informational status codes too).
9731 * @param pVCpu The cross context virtual CPU structure.
9732 *
9733 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
9734 */
9735static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu)
9736{
9737 VMXTRANSIENT VmxTransient;
9738 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9739
9740 /* Set HMCPU indicators. */
9741 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
9742 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
9743 pVCpu->hm.s.fDebugWantRdTscExit = false;
9744 pVCpu->hm.s.fUsingDebugLoop = true;
9745
9746 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
9747 VMXRUNDBGSTATE DbgState;
9748 hmR0VmxRunDebugStateInit(pVCpu, &DbgState);
9749 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9750
9751 /*
9752 * The loop.
9753 */
9754 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9755 for (uint32_t cLoops = 0; ; cLoops++)
9756 {
9757 Assert(!HMR0SuspendPending());
9758 HMVMX_ASSERT_CPU_SAFE(pVCpu);
9759 bool fStepping = pVCpu->hm.s.fSingleInstruction;
9760
9761 /*
9762 * Preparatory work for running guest code, this may force us to return
9763 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
9764 */
9765 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9766 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
9767 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
9768 if (rcStrict != VINF_SUCCESS)
9769 break;
9770
9771 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
9772 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
9773
9774 /*
9775 * Now we can run the guest code.
9776 */
9777 int rcRun = hmR0VmxRunGuest(pVCpu);
9778
9779 /*
9780 * Restore any residual host-state and save any bits shared between host
9781 * and guest into the guest-CPU state. Re-enables interrupts!
9782 */
9783 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
9784
9785 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9786 if (RT_SUCCESS(rcRun))
9787 { /* very likely */ }
9788 else
9789 {
9790 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
9791 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
9792 return rcRun;
9793 }
9794
9795 /* Profile the VM-exit. */
9796 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9797 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9798 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9799 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
9800 HMVMX_START_EXIT_DISPATCH_PROF();
9801
9802 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
9803
9804 /*
9805 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
9806 */
9807 rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
9808 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
9809 if (rcStrict != VINF_SUCCESS)
9810 break;
9811 if (cLoops > pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
9812 {
9813 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9814 rcStrict = VINF_EM_RAW_INTERRUPT;
9815 break;
9816 }
9817
9818 /*
9819 * Stepping: Did the RIP change, if so, consider it a single step.
9820 * Otherwise, make sure one of the TFs gets set.
9821 */
9822 if (fStepping)
9823 {
9824 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
9825 AssertRC(rc);
9826 if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
9827 || pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
9828 {
9829 rcStrict = VINF_EM_DBG_STEPPED;
9830 break;
9831 }
9832 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
9833 }
9834
9835 /*
9836 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
9837 */
9838 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
9839 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9840 }
9841
9842 /*
9843 * Clear the X86_EFL_TF if necessary.
9844 */
9845 if (pVCpu->hm.s.fClearTrapFlag)
9846 {
9847 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
9848 AssertRC(rc);
9849 pVCpu->hm.s.fClearTrapFlag = false;
9850 pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
9851 }
9852 /** @todo there seems to be issues with the resume flag when the monitor trap
9853 * flag is pending without being used. Seen early in bios init when
9854 * accessing APIC page in protected mode. */
9855
9856 /*
9857 * Restore VM-exit control settings as we may not reenter this function the
9858 * next time around.
9859 */
9860 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
9861
9862 /* Restore HMCPU indicators. */
9863 pVCpu->hm.s.fUsingDebugLoop = false;
9864 pVCpu->hm.s.fDebugWantRdTscExit = false;
9865 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
9866
9867 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9868 return rcStrict;
9869}
9870
9871
9872/** @} */
9873
9874
9875/**
9876 * Checks if any expensive dtrace probes are enabled and we should go to the
9877 * debug loop.
9878 *
9879 * @returns true if we should use debug loop, false if not.
9880 */
9881static bool hmR0VmxAnyExpensiveProbesEnabled(void)
9882{
9883 /* It's probably faster to OR the raw 32-bit counter variables together.
9884 Since the variables are in an array and the probes are next to one
9885 another (more or less), we have good locality. So, better read
9886 eight-nine cache lines ever time and only have one conditional, than
9887 128+ conditionals, right? */
9888 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
9889 | VBOXVMM_XCPT_DE_ENABLED_RAW()
9890 | VBOXVMM_XCPT_DB_ENABLED_RAW()
9891 | VBOXVMM_XCPT_BP_ENABLED_RAW()
9892 | VBOXVMM_XCPT_OF_ENABLED_RAW()
9893 | VBOXVMM_XCPT_BR_ENABLED_RAW()
9894 | VBOXVMM_XCPT_UD_ENABLED_RAW()
9895 | VBOXVMM_XCPT_NM_ENABLED_RAW()
9896 | VBOXVMM_XCPT_DF_ENABLED_RAW()
9897 | VBOXVMM_XCPT_TS_ENABLED_RAW()
9898 | VBOXVMM_XCPT_NP_ENABLED_RAW()
9899 | VBOXVMM_XCPT_SS_ENABLED_RAW()
9900 | VBOXVMM_XCPT_GP_ENABLED_RAW()
9901 | VBOXVMM_XCPT_PF_ENABLED_RAW()
9902 | VBOXVMM_XCPT_MF_ENABLED_RAW()
9903 | VBOXVMM_XCPT_AC_ENABLED_RAW()
9904 | VBOXVMM_XCPT_XF_ENABLED_RAW()
9905 | VBOXVMM_XCPT_VE_ENABLED_RAW()
9906 | VBOXVMM_XCPT_SX_ENABLED_RAW()
9907 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
9908 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
9909 ) != 0
9910 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
9911 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
9912 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
9913 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
9914 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
9915 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
9916 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
9917 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
9918 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
9919 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
9920 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
9921 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
9922 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
9923 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
9924 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
9925 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
9926 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
9927 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
9928 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
9929 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
9930 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
9931 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
9932 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
9933 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
9934 | VBOXVMM_INSTR_STR_ENABLED_RAW()
9935 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
9936 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
9937 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
9938 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
9939 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
9940 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
9941 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
9942 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
9943 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
9944 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
9945 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
9946 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
9947 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
9948 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
9949 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
9950 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
9951 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
9952 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
9953 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
9954 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
9955 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
9956 ) != 0
9957 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
9958 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
9959 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
9960 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
9961 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
9962 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
9963 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
9964 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
9965 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
9966 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
9967 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
9968 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
9969 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
9970 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
9971 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
9972 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
9973 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
9974 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
9975 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
9976 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
9977 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
9978 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
9979 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
9980 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
9981 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
9982 | VBOXVMM_EXIT_STR_ENABLED_RAW()
9983 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
9984 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
9985 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
9986 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
9987 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
9988 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
9989 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
9990 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
9991 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
9992 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
9993 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
9994 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
9995 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
9996 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
9997 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
9998 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
9999 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10000 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10001 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10002 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10003 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10004 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10005 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10006 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10007 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10008 ) != 0;
10009}
10010
10011
10012/**
10013 * Runs the guest code using VT-x.
10014 *
10015 * @returns Strict VBox status code (i.e. informational status codes too).
10016 * @param pVCpu The cross context virtual CPU structure.
10017 */
10018VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
10019{
10020 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10021 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10022 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10023 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
10024
10025 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10026
10027 VBOXSTRICTRC rcStrict;
10028 if ( !pVCpu->hm.s.fUseDebugLoop
10029 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10030 && !DBGFIsStepping(pVCpu)
10031 && !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
10032 rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu);
10033 else
10034 rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu);
10035
10036 if (rcStrict == VERR_EM_INTERPRETER)
10037 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10038 else if (rcStrict == VINF_EM_RESET)
10039 rcStrict = VINF_EM_TRIPLE_FAULT;
10040
10041 int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
10042 if (RT_FAILURE(rc2))
10043 {
10044 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10045 rcStrict = rc2;
10046 }
10047 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10048 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10049 return rcStrict;
10050}
10051
10052
10053#ifndef HMVMX_USE_FUNCTION_TABLE
10054DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10055{
10056#ifdef DEBUG_ramshankar
10057#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
10058 do { \
10059 if (a_fSave != 0) \
10060 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
10061 VBOXSTRICTRC rcStrict = a_CallExpr; \
10062 if (a_fSave != 0) \
10063 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
10064 return rcStrict; \
10065 } while (0)
10066#else
10067# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
10068#endif
10069 switch (rcReason)
10070 {
10071 case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
10072 case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
10073 case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
10074 case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
10075 case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
10076 case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
10077 case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
10078 case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
10079 case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
10080 case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
10081 case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
10082 case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
10083 case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
10084 case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
10085 case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
10086 case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
10087 case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
10088 case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
10089 case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
10090 case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
10091 case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
10092 case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
10093 case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
10094 case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pVmxTransient));
10095 case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
10096 case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
10097 case VMX_EXIT_XDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10098 case VMX_EXIT_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10099 case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
10100 case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
10101 case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
10102 case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
10103 case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
10104 case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
10105
10106 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
10107 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
10108 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pVmxTransient);
10109 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pVmxTransient);
10110 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pVmxTransient);
10111 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pVmxTransient);
10112 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pVmxTransient);
10113 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
10114 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pVmxTransient);
10115
10116 case VMX_EXIT_VMCLEAR:
10117 case VMX_EXIT_VMLAUNCH:
10118 case VMX_EXIT_VMPTRLD:
10119 case VMX_EXIT_VMPTRST:
10120 case VMX_EXIT_VMREAD:
10121 case VMX_EXIT_VMRESUME:
10122 case VMX_EXIT_VMWRITE:
10123 case VMX_EXIT_VMXOFF:
10124 case VMX_EXIT_VMXON:
10125 case VMX_EXIT_INVEPT:
10126 case VMX_EXIT_INVVPID:
10127 case VMX_EXIT_VMFUNC:
10128 case VMX_EXIT_XSAVES:
10129 case VMX_EXIT_XRSTORS:
10130 return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
10131
10132 case VMX_EXIT_ENCLS:
10133 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10134 case VMX_EXIT_PML_FULL:
10135 default:
10136 return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
10137 }
10138#undef VMEXIT_CALL_RET
10139}
10140#endif /* !HMVMX_USE_FUNCTION_TABLE */
10141
10142
10143#ifdef VBOX_STRICT
10144/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10145# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10146 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10147
10148# define HMVMX_ASSERT_PREEMPT_CPUID() \
10149 do { \
10150 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10151 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10152 } while (0)
10153
10154# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10155 do { \
10156 AssertPtr((a_pVCpu)); \
10157 AssertPtr((a_pVmxTransient)); \
10158 Assert((a_pVmxTransient)->fVMEntryFailed == false); \
10159 Assert(ASMIntAreEnabled()); \
10160 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10161 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10162 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
10163 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10164 if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
10165 HMVMX_ASSERT_PREEMPT_CPUID(); \
10166 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10167 } while (0)
10168
10169# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10170 do { \
10171 Log4Func(("\n")); \
10172 } while (0)
10173#else
10174# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10175 do { \
10176 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10177 NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
10178 } while (0)
10179# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
10180#endif
10181
10182
10183/**
10184 * Advances the guest RIP by the specified number of bytes.
10185 *
10186 * @param pVCpu The cross context virtual CPU structure.
10187 * @param cbInstr Number of bytes to advance the RIP by.
10188 *
10189 * @remarks No-long-jump zone!!!
10190 */
10191DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
10192{
10193 /* Advance the RIP. */
10194 pVCpu->cpum.GstCtx.rip += cbInstr;
10195 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
10196
10197 /* Update interrupt inhibition. */
10198 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10199 && pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
10200 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10201}
10202
10203
10204/**
10205 * Advances the guest RIP after reading it from the VMCS.
10206 *
10207 * @returns VBox status code, no informational status codes.
10208 * @param pVCpu The cross context virtual CPU structure.
10209 * @param pVmxTransient Pointer to the VMX transient structure.
10210 *
10211 * @remarks No-long-jump zone!!!
10212 */
10213static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10214{
10215 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10216 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS);
10217 AssertRCReturn(rc, rc);
10218
10219 hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
10220
10221 /*
10222 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10223 * pending debug exception field as it takes care of priority of events.
10224 *
10225 * See Intel spec. 32.2.1 "Debug Exceptions".
10226 */
10227 if ( !pVCpu->hm.s.fSingleInstruction
10228 && pVCpu->cpum.GstCtx.eflags.Bits.u1TF)
10229 {
10230 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10231 AssertRCReturn(rc, rc);
10232 }
10233
10234 return VINF_SUCCESS;
10235}
10236
10237
10238/**
10239 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10240 * and update error record fields accordingly.
10241 *
10242 * @return VMX_IGS_* return codes.
10243 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10244 * wrong with the guest state.
10245 *
10246 * @param pVCpu The cross context virtual CPU structure.
10247 *
10248 * @remarks This function assumes our cache of the VMCS controls
10249 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10250 */
10251static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu)
10252{
10253#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10254#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10255 uError = (err); \
10256 break; \
10257 } else do { } while (0)
10258
10259 int rc;
10260 PVM pVM = pVCpu->CTX_SUFF(pVM);
10261 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10262 uint32_t uError = VMX_IGS_ERROR;
10263 uint32_t u32Val;
10264 bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10265
10266 do
10267 {
10268 /*
10269 * CR0.
10270 */
10271 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10272 uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10273 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10274 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10275 if (fUnrestrictedGuest)
10276 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
10277
10278 uint32_t u32GuestCr0;
10279 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10280 AssertRCBreak(rc);
10281 HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10282 HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10283 if ( !fUnrestrictedGuest
10284 && (u32GuestCr0 & X86_CR0_PG)
10285 && !(u32GuestCr0 & X86_CR0_PE))
10286 {
10287 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10288 }
10289
10290 /*
10291 * CR4.
10292 */
10293 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10294 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10295
10296 uint32_t u32GuestCr4;
10297 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10298 AssertRCBreak(rc);
10299 HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10300 HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10301
10302 /*
10303 * IA32_DEBUGCTL MSR.
10304 */
10305 uint64_t u64Val;
10306 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10307 AssertRCBreak(rc);
10308 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10309 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10310 {
10311 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10312 }
10313 uint64_t u64DebugCtlMsr = u64Val;
10314
10315#ifdef VBOX_STRICT
10316 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10317 AssertRCBreak(rc);
10318 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10319#endif
10320 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10321
10322 /*
10323 * RIP and RFLAGS.
10324 */
10325 uint32_t u32Eflags;
10326#if HC_ARCH_BITS == 64
10327 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10328 AssertRCBreak(rc);
10329 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10330 if ( !fLongModeGuest
10331 || !pCtx->cs.Attr.n.u1Long)
10332 {
10333 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10334 }
10335 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10336 * must be identical if the "IA-32e mode guest" VM-entry
10337 * control is 1 and CS.L is 1. No check applies if the
10338 * CPU supports 64 linear-address bits. */
10339
10340 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10341 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10342 AssertRCBreak(rc);
10343 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10344 VMX_IGS_RFLAGS_RESERVED);
10345 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10346 u32Eflags = u64Val;
10347#else
10348 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10349 AssertRCBreak(rc);
10350 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10351 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10352#endif
10353
10354 if ( fLongModeGuest
10355 || ( fUnrestrictedGuest
10356 && !(u32GuestCr0 & X86_CR0_PE)))
10357 {
10358 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10359 }
10360
10361 uint32_t u32EntryInfo;
10362 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10363 AssertRCBreak(rc);
10364 if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
10365 && VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
10366 {
10367 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10368 }
10369
10370 /*
10371 * 64-bit checks.
10372 */
10373#if HC_ARCH_BITS == 64
10374 if (fLongModeGuest)
10375 {
10376 HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10377 HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10378 }
10379
10380 if ( !fLongModeGuest
10381 && (u32GuestCr4 & X86_CR4_PCIDE))
10382 {
10383 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10384 }
10385
10386 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10387 * 51:32 beyond the processor's physical-address width are 0. */
10388
10389 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10390 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10391 {
10392 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10393 }
10394
10395 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10396 AssertRCBreak(rc);
10397 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10398
10399 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10400 AssertRCBreak(rc);
10401 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10402#endif
10403
10404 /*
10405 * PERF_GLOBAL MSR.
10406 */
10407 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10408 {
10409 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10410 AssertRCBreak(rc);
10411 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10412 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10413 }
10414
10415 /*
10416 * PAT MSR.
10417 */
10418 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10419 {
10420 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10421 AssertRCBreak(rc);
10422 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10423 for (unsigned i = 0; i < 8; i++)
10424 {
10425 uint8_t u8Val = (u64Val & 0xff);
10426 if ( u8Val != 0 /* UC */
10427 && u8Val != 1 /* WC */
10428 && u8Val != 4 /* WT */
10429 && u8Val != 5 /* WP */
10430 && u8Val != 6 /* WB */
10431 && u8Val != 7 /* UC- */)
10432 {
10433 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10434 }
10435 u64Val >>= 8;
10436 }
10437 }
10438
10439 /*
10440 * EFER MSR.
10441 */
10442 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10443 {
10444 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10445 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10446 AssertRCBreak(rc);
10447 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10448 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10449 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10450 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10451 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10452 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10453 || !(u32GuestCr0 & X86_CR0_PG)
10454 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10455 VMX_IGS_EFER_LMA_LME_MISMATCH);
10456 }
10457
10458 /*
10459 * Segment registers.
10460 */
10461 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10462 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10463 if (!(u32Eflags & X86_EFL_VM))
10464 {
10465 /* CS */
10466 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10467 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10468 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10469 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10470 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10471 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10472 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10473 /* CS cannot be loaded with NULL in protected mode. */
10474 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10475 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10476 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10477 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10478 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10479 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10480 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10481 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10482 else
10483 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10484
10485 /* SS */
10486 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10487 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10488 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10489 if ( !(pCtx->cr0 & X86_CR0_PE)
10490 || pCtx->cs.Attr.n.u4Type == 3)
10491 {
10492 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10493 }
10494 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10495 {
10496 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10497 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10498 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10499 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10500 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10501 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10502 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10503 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10504 }
10505
10506 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmenReg(). */
10507 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10508 {
10509 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10510 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10511 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10512 || pCtx->ds.Attr.n.u4Type > 11
10513 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10514 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10515 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10516 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10517 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10518 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10519 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10520 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10521 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10522 }
10523 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10524 {
10525 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10526 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10527 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10528 || pCtx->es.Attr.n.u4Type > 11
10529 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10530 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10531 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10532 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10533 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10534 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10535 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10536 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10537 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10538 }
10539 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10540 {
10541 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10542 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10543 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10544 || pCtx->fs.Attr.n.u4Type > 11
10545 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10546 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10547 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10548 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10549 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10550 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10551 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10552 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10553 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10554 }
10555 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10556 {
10557 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10558 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10559 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10560 || pCtx->gs.Attr.n.u4Type > 11
10561 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10562 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10563 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10564 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10565 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10566 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10567 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10568 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10569 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10570 }
10571 /* 64-bit capable CPUs. */
10572#if HC_ARCH_BITS == 64
10573 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10574 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10575 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10576 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10577 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10578 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10579 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10580 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10581 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10582 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10583 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10584#endif
10585 }
10586 else
10587 {
10588 /* V86 mode checks. */
10589 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10590 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10591 {
10592 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10593 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10594 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10595 }
10596 else
10597 {
10598 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10599 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10600 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10601 }
10602
10603 /* CS */
10604 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10605 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10606 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10607 /* SS */
10608 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10609 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10610 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10611 /* DS */
10612 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10613 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10614 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10615 /* ES */
10616 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10617 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10618 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10619 /* FS */
10620 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10621 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10622 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10623 /* GS */
10624 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10625 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10626 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10627 /* 64-bit capable CPUs. */
10628#if HC_ARCH_BITS == 64
10629 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10630 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10631 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10632 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10633 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10634 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10635 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10636 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10637 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10638 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10639 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10640#endif
10641 }
10642
10643 /*
10644 * TR.
10645 */
10646 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10647 /* 64-bit capable CPUs. */
10648#if HC_ARCH_BITS == 64
10649 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10650#endif
10651 if (fLongModeGuest)
10652 {
10653 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10654 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10655 }
10656 else
10657 {
10658 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10659 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10660 VMX_IGS_TR_ATTR_TYPE_INVALID);
10661 }
10662 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10663 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10664 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10665 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10666 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10667 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10668 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10669 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10670
10671 /*
10672 * GDTR and IDTR.
10673 */
10674#if HC_ARCH_BITS == 64
10675 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10676 AssertRCBreak(rc);
10677 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10678
10679 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10680 AssertRCBreak(rc);
10681 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10682#endif
10683
10684 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10685 AssertRCBreak(rc);
10686 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10687
10688 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10689 AssertRCBreak(rc);
10690 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10691
10692 /*
10693 * Guest Non-Register State.
10694 */
10695 /* Activity State. */
10696 uint32_t u32ActivityState;
10697 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10698 AssertRCBreak(rc);
10699 HMVMX_CHECK_BREAK( !u32ActivityState
10700 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
10701 VMX_IGS_ACTIVITY_STATE_INVALID);
10702 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10703 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10704 uint32_t u32IntrState;
10705 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
10706 AssertRCBreak(rc);
10707 if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
10708 || u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10709 {
10710 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10711 }
10712
10713 /** @todo Activity state and injecting interrupts. Left as a todo since we
10714 * currently don't use activity states but ACTIVE. */
10715
10716 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10717 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10718
10719 /* Guest interruptibility-state. */
10720 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10721 HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
10722 != (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10723 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10724 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10725 || !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
10726 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10727 if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
10728 {
10729 if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
10730 {
10731 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10732 && !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10733 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10734 }
10735 else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
10736 {
10737 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10738 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10739 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
10740 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10741 }
10742 }
10743 /** @todo Assumes the processor is not in SMM. */
10744 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
10745 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10746 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10747 || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
10748 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10749 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
10750 && VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
10751 && VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
10752 {
10753 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
10754 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10755 }
10756
10757 /* Pending debug exceptions. */
10758#if HC_ARCH_BITS == 64
10759 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
10760 AssertRCBreak(rc);
10761 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10762 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10763 u32Val = u64Val; /* For pending debug exceptions checks below. */
10764#else
10765 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
10766 AssertRCBreak(rc);
10767 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10768 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10769#endif
10770
10771 if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10772 || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
10773 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10774 {
10775 if ( (u32Eflags & X86_EFL_TF)
10776 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10777 {
10778 /* Bit 14 is PendingDebug.BS. */
10779 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10780 }
10781 if ( !(u32Eflags & X86_EFL_TF)
10782 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10783 {
10784 /* Bit 14 is PendingDebug.BS. */
10785 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10786 }
10787 }
10788
10789 /* VMCS link pointer. */
10790 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10791 AssertRCBreak(rc);
10792 if (u64Val != UINT64_C(0xffffffffffffffff))
10793 {
10794 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10795 /** @todo Bits beyond the processor's physical-address width MBZ. */
10796 /** @todo 32-bit located in memory referenced by value of this field (as a
10797 * physical address) must contain the processor's VMCS revision ID. */
10798 /** @todo SMM checks. */
10799 }
10800
10801 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10802 * not using Nested Paging? */
10803 if ( pVM->hm.s.fNestedPaging
10804 && !fLongModeGuest
10805 && CPUMIsGuestInPAEModeEx(pCtx))
10806 {
10807 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10808 AssertRCBreak(rc);
10809 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10810
10811 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10812 AssertRCBreak(rc);
10813 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10814
10815 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10816 AssertRCBreak(rc);
10817 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10818
10819 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10820 AssertRCBreak(rc);
10821 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10822 }
10823
10824 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10825 if (uError == VMX_IGS_ERROR)
10826 uError = VMX_IGS_REASON_NOT_FOUND;
10827 } while (0);
10828
10829 pVCpu->hm.s.u32HMError = uError;
10830 return uError;
10831
10832#undef HMVMX_ERROR_BREAK
10833#undef HMVMX_CHECK_BREAK
10834}
10835
10836/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10837/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
10838/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10839
10840/** @name VM-exit handlers.
10841 * @{
10842 */
10843
10844/**
10845 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
10846 */
10847HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10848{
10849 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10850 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
10851 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
10852 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
10853 return VINF_SUCCESS;
10854 return VINF_EM_RAW_INTERRUPT;
10855}
10856
10857
10858/**
10859 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
10860 */
10861HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10862{
10863 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10864 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
10865
10866 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10867 AssertRCReturn(rc, rc);
10868
10869 uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
10870 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
10871 && uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
10872 Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
10873
10874 if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
10875 {
10876 /*
10877 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
10878 * injected it ourselves and anything we inject is not going to cause a VM-exit directly
10879 * for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
10880 *
10881 * [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
10882 * [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
10883 */
10884 VMXDispatchHostNmi();
10885 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
10886 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10887 return VINF_SUCCESS;
10888 }
10889
10890 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10891 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
10892 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
10893 { /* likely */ }
10894 else
10895 {
10896 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
10897 rcStrictRc1 = VINF_SUCCESS;
10898 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10899 return rcStrictRc1;
10900 }
10901
10902 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
10903 uint32_t uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
10904 switch (uIntType)
10905 {
10906 case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
10907 Assert(uVector == X86_XCPT_DB);
10908 RT_FALL_THRU();
10909 case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
10910 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
10911 RT_FALL_THRU();
10912 case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
10913 {
10914 /*
10915 * If there's any exception caused as a result of event injection, the resulting
10916 * secondary/final execption will be pending, we shall continue guest execution
10917 * after injecting the event. The page-fault case is complicated and we manually
10918 * handle any currently pending event in hmR0VmxExitXcptPF.
10919 */
10920 if (!pVCpu->hm.s.Event.fPending)
10921 { /* likely */ }
10922 else if (uVector != X86_XCPT_PF)
10923 {
10924 rc = VINF_SUCCESS;
10925 break;
10926 }
10927
10928 switch (uVector)
10929 {
10930 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
10931 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
10932 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
10933 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
10934 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
10935 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
10936
10937 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10938 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10939 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
10940 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10941 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
10942 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10943 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
10944 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10945 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
10946 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10947 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
10948 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10949 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
10950 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10951 default:
10952 {
10953 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
10954 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10955 {
10956 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
10957 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
10958 Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
10959
10960 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
10961 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10962 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10963 AssertRCReturn(rc, rc);
10964 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
10965 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
10966 0 /* GCPtrFaultAddress */);
10967 }
10968 else
10969 {
10970 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
10971 pVCpu->hm.s.u32HMError = uVector;
10972 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10973 }
10974 break;
10975 }
10976 }
10977 break;
10978 }
10979
10980 default:
10981 {
10982 pVCpu->hm.s.u32HMError = uExitIntInfo;
10983 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
10984 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
10985 break;
10986 }
10987 }
10988 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10989 return rc;
10990}
10991
10992
10993/**
10994 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
10995 */
10996HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10997{
10998 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10999
11000 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11001 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11002
11003 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11004 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11005 return VINF_SUCCESS;
11006}
11007
11008
11009/**
11010 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11011 */
11012HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11013{
11014 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11015 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11016 {
11017 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11018 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11019 }
11020
11021 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11022
11023 /*
11024 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11025 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11026 */
11027 uint32_t fIntrState = 0;
11028 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
11029 AssertRCReturn(rc, rc);
11030
11031 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
11032 if ( fBlockSti
11033 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11034 {
11035 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11036 }
11037
11038 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11039 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11040
11041 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11042 return VINF_SUCCESS;
11043}
11044
11045
11046/**
11047 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11048 */
11049HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11050{
11051 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11052 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11053}
11054
11055
11056/**
11057 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11058 */
11059HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11060{
11061 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11062 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11063}
11064
11065
11066/**
11067 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11068 */
11069HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11070{
11071 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11072
11073 /*
11074 * Get the state we need and update the exit history entry.
11075 */
11076 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11077 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX);
11078 AssertRCReturn(rc, rc);
11079
11080 VBOXSTRICTRC rcStrict;
11081 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11082 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
11083 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11084 if (!pExitRec)
11085 {
11086 /*
11087 * Regular CPUID instruction execution.
11088 */
11089 rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
11090 if (rcStrict == VINF_SUCCESS)
11091 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_RAX
11092 | HM_CHANGED_GUEST_RCX | HM_CHANGED_GUEST_RDX | HM_CHANGED_GUEST_RBX);
11093 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11094 {
11095 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11096 rcStrict = VINF_SUCCESS;
11097 }
11098 }
11099 else
11100 {
11101 /*
11102 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
11103 */
11104 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11105 AssertRCReturn(rc2, rc2);
11106
11107 Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11108 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11109
11110 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11111 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
11112
11113 Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11114 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11115 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11116 }
11117 return rcStrict;
11118}
11119
11120
11121/**
11122 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11123 */
11124HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11125{
11126 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11127 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4);
11128 AssertRCReturn(rc, rc);
11129
11130 if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
11131 return VINF_EM_RAW_EMULATE_INSTR;
11132
11133 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11134 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11135}
11136
11137
11138/**
11139 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11140 */
11141HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11142{
11143 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11144 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11145 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11146 AssertRCReturn(rc, rc);
11147
11148 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11149 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11150 {
11151 /* If we get a spurious VM-exit when offsetting is enabled,
11152 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11153 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11154 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11155 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11156 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX);
11157 }
11158 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11159 {
11160 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11161 rcStrict = VINF_SUCCESS;
11162 }
11163 return rcStrict;
11164}
11165
11166
11167/**
11168 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11169 */
11170HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11171{
11172 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11173 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_TSC_AUX);
11174 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11175 AssertRCReturn(rc, rc);
11176
11177 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11178 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11179 {
11180 /* If we get a spurious VM-exit when offsetting is enabled,
11181 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11182 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11183 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11184 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11185 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX | HM_CHANGED_GUEST_RCX);
11186 }
11187 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11188 {
11189 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11190 rcStrict = VINF_SUCCESS;
11191 }
11192 return rcStrict;
11193}
11194
11195
11196/**
11197 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11198 */
11199HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11200{
11201 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11202 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11203 AssertRCReturn(rc, rc);
11204
11205 PVM pVM = pVCpu->CTX_SUFF(pVM);
11206 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11207 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11208 if (RT_LIKELY(rc == VINF_SUCCESS))
11209 {
11210 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11211 Assert(pVmxTransient->cbInstr == 2);
11212 }
11213 else
11214 {
11215 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11216 rc = VERR_EM_INTERPRETER;
11217 }
11218 return rc;
11219}
11220
11221
11222/**
11223 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11224 */
11225HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11226{
11227 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11228
11229 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11230 if (EMAreHypercallInstructionsEnabled(pVCpu))
11231 {
11232 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_SS
11233 | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_EFER);
11234 AssertRCReturn(rc, rc);
11235
11236 /* Perform the hypercall. */
11237 rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
11238 if (rcStrict == VINF_SUCCESS)
11239 {
11240 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11241 AssertRCReturn(rc, rc);
11242 }
11243 else
11244 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11245 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11246 || RT_FAILURE(rcStrict));
11247
11248 /* If the hypercall changes anything other than guest's general-purpose registers,
11249 we would need to reload the guest changed bits here before VM-entry. */
11250 }
11251 else
11252 Log4Func(("Hypercalls not enabled\n"));
11253
11254 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11255 if (RT_FAILURE(rcStrict))
11256 {
11257 hmR0VmxSetPendingXcptUD(pVCpu);
11258 rcStrict = VINF_SUCCESS;
11259 }
11260
11261 return rcStrict;
11262}
11263
11264
11265/**
11266 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11267 */
11268HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11269{
11270 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11271 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11272
11273 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11274 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11275 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_DS);
11276 AssertRCReturn(rc, rc);
11277
11278 VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQualification);
11279
11280 if (rcStrict == VINF_SUCCESS || rcStrict == VINF_PGM_SYNC_CR3)
11281 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11282 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11283 {
11284 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11285 rcStrict = VINF_SUCCESS;
11286 }
11287 else
11288 AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) sttus: %Rrc\n",
11289 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11290 return rcStrict;
11291}
11292
11293
11294/**
11295 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11296 */
11297HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11298{
11299 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11300 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11301 AssertRCReturn(rc, rc);
11302
11303 PVM pVM = pVCpu->CTX_SUFF(pVM);
11304 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11305 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11306 if (RT_LIKELY(rc == VINF_SUCCESS))
11307 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11308 else
11309 {
11310 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11311 rc = VERR_EM_INTERPRETER;
11312 }
11313 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11314 return rc;
11315}
11316
11317
11318/**
11319 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11320 */
11321HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11322{
11323 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11324 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11325 AssertRCReturn(rc, rc);
11326
11327 PVM pVM = pVCpu->CTX_SUFF(pVM);
11328 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11329 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11330 rc = VBOXSTRICTRC_VAL(rc2);
11331 if (RT_LIKELY( rc == VINF_SUCCESS
11332 || rc == VINF_EM_HALT))
11333 {
11334 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11335 AssertRCReturn(rc3, rc3);
11336
11337 if ( rc == VINF_EM_HALT
11338 && EMMonitorWaitShouldContinue(pVCpu, pCtx))
11339 rc = VINF_SUCCESS;
11340 }
11341 else
11342 {
11343 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11344 rc = VERR_EM_INTERPRETER;
11345 }
11346 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11347 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11348 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11349 return rc;
11350}
11351
11352
11353/**
11354 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11355 */
11356HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11357{
11358 /*
11359 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
11360 * mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
11361 * treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
11362 * VMX root operation. If we get here, something funny is going on.
11363 *
11364 * See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
11365 */
11366 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11367 AssertMsgFailed(("Unexpected RSM VM-exit\n"));
11368 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11369}
11370
11371
11372/**
11373 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11374 */
11375HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11376{
11377 /*
11378 * This can only happen if we support dual-monitor treatment of SMI, which can be activated
11379 * by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
11380 * this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
11381 * an SMI. If we get here, something funny is going on.
11382 *
11383 * See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11384 * See Intel spec. 25.3 "Other Causes of VM-Exits"
11385 */
11386 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11387 AssertMsgFailed(("Unexpected SMI VM-exit\n"));
11388 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11389}
11390
11391
11392/**
11393 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11394 */
11395HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11396{
11397 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11398 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11399 AssertMsgFailed(("Unexpected IO SMI VM-exit\n"));
11400 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11401}
11402
11403
11404/**
11405 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11406 */
11407HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11408{
11409 /*
11410 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
11411 * We don't make use of it as our guests don't have direct access to the host LAPIC.
11412 * See Intel spec. 25.3 "Other Causes of VM-exits".
11413 */
11414 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11415 AssertMsgFailed(("Unexpected SIPI VM-exit\n"));
11416 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11417}
11418
11419
11420/**
11421 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11422 * VM-exit.
11423 */
11424HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11425{
11426 /*
11427 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11428 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11429 *
11430 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11431 * See Intel spec. "23.8 Restrictions on VMX operation".
11432 */
11433 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11434 return VINF_SUCCESS;
11435}
11436
11437
11438/**
11439 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11440 * VM-exit.
11441 */
11442HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11443{
11444 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11445 return VINF_EM_RESET;
11446}
11447
11448
11449/**
11450 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11451 */
11452HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11453{
11454 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11455 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11456
11457 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11458 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
11459 AssertRCReturn(rc, rc);
11460
11461 if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
11462 rc = VINF_SUCCESS;
11463 else
11464 rc = VINF_EM_HALT;
11465
11466 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11467 if (rc != VINF_SUCCESS)
11468 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11469 return rc;
11470}
11471
11472
11473/**
11474 * VM-exit handler for instructions that result in a \#UD exception delivered to
11475 * the guest.
11476 */
11477HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11478{
11479 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11480 hmR0VmxSetPendingXcptUD(pVCpu);
11481 return VINF_SUCCESS;
11482}
11483
11484
11485/**
11486 * VM-exit handler for expiry of the VMX preemption timer.
11487 */
11488HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11489{
11490 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11491
11492 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11493 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11494
11495 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11496 PVM pVM = pVCpu->CTX_SUFF(pVM);
11497 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11498 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11499 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11500}
11501
11502
11503/**
11504 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11505 */
11506HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11507{
11508 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11509
11510 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11511 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_CR4);
11512 AssertRCReturn(rc, rc);
11513
11514 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11515 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11516 : HM_CHANGED_RAISED_XCPT_MASK);
11517
11518 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11519 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
11520
11521 return rcStrict;
11522}
11523
11524
11525/**
11526 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11527 */
11528HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11529{
11530 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11531 /** @todo Use VM-exit instruction information. */
11532 return VERR_EM_INTERPRETER;
11533}
11534
11535
11536/**
11537 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11538 * Error VM-exit.
11539 */
11540HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11541{
11542 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11543 AssertRCReturn(rc, rc);
11544 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11545 if (RT_FAILURE(rc))
11546 return rc;
11547
11548 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
11549 NOREF(uInvalidReason);
11550
11551#ifdef VBOX_STRICT
11552 uint32_t fIntrState;
11553 RTHCUINTREG uHCReg;
11554 uint64_t u64Val;
11555 uint32_t u32Val;
11556
11557 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11558 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11559 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11560 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
11561 AssertRCReturn(rc, rc);
11562
11563 Log4(("uInvalidReason %u\n", uInvalidReason));
11564 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11565 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11566 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11567 Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
11568
11569 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11570 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11571 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11572 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11573 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11574 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11575 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11576 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11577 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11578 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11579 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11580 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11581
11582 hmR0DumpRegs(pVCpu);
11583#else
11584 NOREF(pVmxTransient);
11585#endif
11586
11587 return VERR_VMX_INVALID_GUEST_STATE;
11588}
11589
11590
11591/**
11592 * VM-exit handler for VM-entry failure due to an MSR-load
11593 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11594 */
11595HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11596{
11597 AssertMsgFailed(("Unexpected MSR-load exit\n"));
11598 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11599}
11600
11601
11602/**
11603 * VM-exit handler for VM-entry failure due to a machine-check event
11604 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11605 */
11606HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11607{
11608 AssertMsgFailed(("Unexpected machine-check event exit\n"));
11609 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11610}
11611
11612
11613/**
11614 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11615 * theory.
11616 */
11617HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11618{
11619 RT_NOREF2(pVCpu, pVmxTransient);
11620 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d\n", pVmxTransient->uExitReason));
11621 return VERR_VMX_UNDEFINED_EXIT_CODE;
11622}
11623
11624
11625/**
11626 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11627 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11628 * Conditional VM-exit.
11629 */
11630HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11631{
11632 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11633
11634 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
11635 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11636 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
11637 return VERR_EM_INTERPRETER;
11638 AssertMsgFailed(("Unexpected XDTR access\n"));
11639 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11640}
11641
11642
11643/**
11644 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11645 */
11646HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11647{
11648 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11649
11650 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
11651 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
11652 return VERR_EM_INTERPRETER;
11653 AssertMsgFailed(("Unexpected RDRAND exit\n"));
11654 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11655}
11656
11657
11658/**
11659 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11660 */
11661HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11662{
11663 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11664
11665 /** @todo Optimize this: We currently drag in in the whole MSR state
11666 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11667 * MSRs required. That would require changes to IEM and possibly CPUM too.
11668 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11669 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; NOREF(idMsr); /* Save it. */
11670 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11671 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
11672 AssertRCReturn(rc, rc);
11673
11674 Log4Func(("ecx=%#RX32\n", idMsr));
11675
11676#ifdef VBOX_STRICT
11677 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
11678 {
11679 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr)
11680 && idMsr != MSR_K6_EFER)
11681 {
11682 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
11683 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11684 }
11685 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11686 {
11687 VMXMSREXITREAD enmRead;
11688 VMXMSREXITWRITE enmWrite;
11689 int rc2 = hmR0VmxGetMsrPermission(pVCpu, idMsr, &enmRead, &enmWrite);
11690 AssertRCReturn(rc2, rc2);
11691 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11692 {
11693 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
11694 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11695 }
11696 }
11697 }
11698#endif
11699
11700 VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
11701 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11702 if (rcStrict == VINF_SUCCESS)
11703 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11704 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX);
11705 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11706 {
11707 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11708 rcStrict = VINF_SUCCESS;
11709 }
11710 else
11711 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11712
11713 return rcStrict;
11714}
11715
11716
11717/**
11718 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11719 */
11720HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11721{
11722 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11723
11724 /** @todo Optimize this: We currently drag in in the whole MSR state
11725 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11726 * MSRs required. That would require changes to IEM and possibly CPUM too.
11727 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11728 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; /* Save it. */
11729 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11730 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
11731 AssertRCReturn(rc, rc);
11732
11733 Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
11734
11735 VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
11736 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
11737
11738 if (rcStrict == VINF_SUCCESS)
11739 {
11740 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11741
11742 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
11743 if ( idMsr == MSR_IA32_APICBASE
11744 || ( idMsr >= MSR_IA32_X2APIC_START
11745 && idMsr <= MSR_IA32_X2APIC_END))
11746 {
11747 /*
11748 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
11749 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before IEM changes it.
11750 */
11751 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11752 }
11753 else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
11754 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11755 else if (idMsr == MSR_K6_EFER)
11756 {
11757 /*
11758 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
11759 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
11760 * the other bits as well, SCE and NXE. See @bugref{7368}.
11761 */
11762 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS
11763 | HM_CHANGED_VMX_EXIT_CTLS);
11764 }
11765
11766 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
11767 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11768 {
11769 switch (idMsr)
11770 {
11771 case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
11772 case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
11773 case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
11774 case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
11775 case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
11776 case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
11777 default:
11778 {
11779 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
11780 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
11781 else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11782 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
11783 break;
11784 }
11785 }
11786 }
11787#ifdef VBOX_STRICT
11788 else
11789 {
11790 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
11791 switch (idMsr)
11792 {
11793 case MSR_IA32_SYSENTER_CS:
11794 case MSR_IA32_SYSENTER_EIP:
11795 case MSR_IA32_SYSENTER_ESP:
11796 case MSR_K8_FS_BASE:
11797 case MSR_K8_GS_BASE:
11798 {
11799 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
11800 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11801 }
11802
11803 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
11804 default:
11805 {
11806 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
11807 {
11808 /* EFER writes are always intercepted, see hmR0VmxExportGuestMsrs(). */
11809 if (idMsr != MSR_K6_EFER)
11810 {
11811 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11812 idMsr));
11813 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11814 }
11815 }
11816
11817 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11818 {
11819 VMXMSREXITREAD enmRead;
11820 VMXMSREXITWRITE enmWrite;
11821 int rc2 = hmR0VmxGetMsrPermission(pVCpu, idMsr, &enmRead, &enmWrite);
11822 AssertRCReturn(rc2, rc2);
11823 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
11824 {
11825 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
11826 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11827 }
11828 }
11829 break;
11830 }
11831 }
11832 }
11833#endif /* VBOX_STRICT */
11834 }
11835 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11836 {
11837 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11838 rcStrict = VINF_SUCCESS;
11839 }
11840 else
11841 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11842
11843 return rcStrict;
11844}
11845
11846
11847/**
11848 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
11849 */
11850HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11851{
11852 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11853 /** @todo The guest has likely hit a contended spinlock. We might want to
11854 * poke a schedule different guest VCPU. */
11855 return VINF_EM_RAW_INTERRUPT;
11856}
11857
11858
11859/**
11860 * VM-exit handler for when the TPR value is lowered below the specified
11861 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
11862 */
11863HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11864{
11865 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11866 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
11867
11868 /*
11869 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
11870 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
11871 */
11872 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
11873 return VINF_SUCCESS;
11874}
11875
11876
11877/**
11878 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
11879 * VM-exit.
11880 *
11881 * @retval VINF_SUCCESS when guest execution can continue.
11882 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
11883 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
11884 * interpreter.
11885 */
11886HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11887{
11888 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11889 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
11890
11891 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11892 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11893 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11894 AssertRCReturn(rc, rc);
11895
11896 VBOXSTRICTRC rcStrict;
11897 PVM pVM = pVCpu->CTX_SUFF(pVM);
11898 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
11899 uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQualification);
11900 switch (uAccessType)
11901 {
11902 case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
11903 {
11904 uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
11905 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
11906 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
11907 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification));
11908 AssertMsg( rcStrict == VINF_SUCCESS
11909 || rcStrict == VINF_IEM_RAISED_XCPT
11910 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11911
11912 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
11913 {
11914 case 0:
11915 {
11916 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
11917 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
11918 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
11919 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
11920
11921 /*
11922 * This is a kludge for handling switches back to real mode when we try to use
11923 * V86 mode to run real mode code directly. Problem is that V86 mode cannot
11924 * deal with special selector values, so we have to return to ring-3 and run
11925 * there till the selector values are V86 mode compatible.
11926 *
11927 * Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
11928 * latter is an alias for VINF_IEM_RAISED_XCPT which is converted to VINF_SUCCESs
11929 * at the end of this function.
11930 */
11931 if ( rc == VINF_SUCCESS
11932 && !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
11933 && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
11934 && (uOldCr0 & X86_CR0_PE)
11935 && !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
11936 {
11937 /** @todo check selectors rather than returning all the time. */
11938 Log4(("CRx CR0 write: back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
11939 rcStrict = VINF_EM_RESCHEDULE_REM;
11940 }
11941 break;
11942 }
11943
11944 case 2:
11945 {
11946 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
11947 /* Nothing to do here, CR2 it's not part of the VMCS. */
11948 break;
11949 }
11950
11951 case 3:
11952 {
11953 Assert( !pVM->hm.s.fNestedPaging
11954 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
11955 || pVCpu->hm.s.fUsingDebugLoop);
11956 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
11957 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
11958 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR3);
11959 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
11960 break;
11961 }
11962
11963 case 4:
11964 {
11965 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
11966 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
11967 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR4);
11968 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
11969 pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
11970 break;
11971 }
11972
11973 case 8:
11974 {
11975 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
11976 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
11977 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
11978 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_APIC_TPR);
11979 break;
11980 }
11981 default:
11982 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification)));
11983 break;
11984 }
11985 break;
11986 }
11987
11988 case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
11989 {
11990 Assert( !pVM->hm.s.fNestedPaging
11991 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
11992 || pVCpu->hm.s.fUsingDebugLoop
11993 || VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 3);
11994 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
11995 Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 8
11996 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
11997
11998 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
11999 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification),
12000 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification));
12001 AssertMsg( rcStrict == VINF_SUCCESS
12002 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12003#ifdef VBOX_WITH_STATISTICS
12004 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
12005 {
12006 case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
12007 case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
12008 case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
12009 case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
12010 case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
12011 }
12012#endif
12013 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
12014 VBOXSTRICTRC_VAL(rcStrict)));
12015 if (VMX_EXIT_QUAL_CRX_GENREG(uExitQualification) == X86_GREG_xSP)
12016 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_RSP);
12017 else
12018 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12019 break;
12020 }
12021
12022 case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12023 {
12024 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12025 AssertMsg( rcStrict == VINF_SUCCESS
12026 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12027
12028 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12029 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12030 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12031 break;
12032 }
12033
12034 case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12035 {
12036 /* Note! LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here. */
12037 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12038 VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQualification));
12039 AssertMsg( rcStrict == VINF_SUCCESS
12040 || rcStrict == VINF_IEM_RAISED_XCPT
12041 , ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12042
12043 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12044 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12045 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12046 break;
12047 }
12048
12049 default:
12050 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12051 VERR_VMX_UNEXPECTED_EXCEPTION);
12052 }
12053
12054 Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS))
12055 == (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS));
12056 if (rcStrict == VINF_IEM_RAISED_XCPT)
12057 {
12058 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
12059 rcStrict = VINF_SUCCESS;
12060 }
12061
12062 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12063 NOREF(pVM);
12064 return rcStrict;
12065}
12066
12067
12068/**
12069 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12070 * VM-exit.
12071 */
12072HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12073{
12074 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12075 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12076
12077 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12078 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12079 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12080 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_EFER);
12081 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12082 AssertRCReturn(rc, rc);
12083
12084 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12085 uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQualification);
12086 uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQualification);
12087 bool fIOWrite = ( VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQualification)
12088 == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12089 bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQualification);
12090 bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
12091 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12092 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12093
12094 /*
12095 * Update exit history to see if this exit can be optimized.
12096 */
12097 VBOXSTRICTRC rcStrict;
12098 PCEMEXITREC pExitRec = NULL;
12099 if ( !fGstStepping
12100 && !fDbgStepping)
12101 pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12102 !fIOString
12103 ? !fIOWrite
12104 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12105 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12106 : !fIOWrite
12107 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12108 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12109 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12110 if (!pExitRec)
12111 {
12112 /* I/O operation lookup arrays. */
12113 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12114 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12115 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12116 uint32_t const cbInstr = pVmxTransient->cbInstr;
12117 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12118 PVM pVM = pVCpu->CTX_SUFF(pVM);
12119 if (fIOString)
12120 {
12121 /*
12122 * INS/OUTS - I/O String instruction.
12123 *
12124 * Use instruction-information if available, otherwise fall back on
12125 * interpreting the instruction.
12126 */
12127 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12128 AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
12129 if (MSR_IA32_VMX_BASIC_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64Basic))
12130 {
12131 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12132 AssertRCReturn(rc2, rc2);
12133 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12134 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12135 IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12136 bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification);
12137 if (fIOWrite)
12138 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12139 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12140 else
12141 {
12142 /*
12143 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12144 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12145 * See Intel Instruction spec. for "INS".
12146 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12147 */
12148 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12149 }
12150 }
12151 else
12152 rcStrict = IEMExecOne(pVCpu);
12153
12154 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12155 fUpdateRipAlready = true;
12156 }
12157 else
12158 {
12159 /*
12160 * IN/OUT - I/O instruction.
12161 */
12162 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12163 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12164 Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification));
12165 if (fIOWrite)
12166 {
12167 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
12168 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12169 if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
12170 && !pCtx->eflags.Bits.u1TF)
12171 rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
12172 }
12173 else
12174 {
12175 uint32_t u32Result = 0;
12176 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12177 if (IOM_SUCCESS(rcStrict))
12178 {
12179 /* Save result of I/O IN instr. in AL/AX/EAX. */
12180 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12181 }
12182 if ( rcStrict == VINF_IOM_R3_IOPORT_READ
12183 && !pCtx->eflags.Bits.u1TF)
12184 rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
12185 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12186 }
12187 }
12188
12189 if (IOM_SUCCESS(rcStrict))
12190 {
12191 if (!fUpdateRipAlready)
12192 {
12193 hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
12194 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12195 }
12196
12197 /*
12198 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12199 * while booting Fedora 17 64-bit guest.
12200 *
12201 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12202 */
12203 if (fIOString)
12204 {
12205 /** @todo Single-step for INS/OUTS with REP prefix? */
12206 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12207 }
12208 else if ( !fDbgStepping
12209 && fGstStepping)
12210 {
12211 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12212 AssertRCReturn(rc, rc);
12213 }
12214
12215 /*
12216 * If any I/O breakpoints are armed, we need to check if one triggered
12217 * and take appropriate action.
12218 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12219 */
12220 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12221 AssertRCReturn(rc, rc);
12222
12223 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12224 * execution engines about whether hyper BPs and such are pending. */
12225 uint32_t const uDr7 = pCtx->dr[7];
12226 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12227 && X86_DR7_ANY_RW_IO(uDr7)
12228 && (pCtx->cr4 & X86_CR4_DE))
12229 || DBGFBpIsHwIoArmed(pVM)))
12230 {
12231 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12232
12233 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12234 VMMRZCallRing3Disable(pVCpu);
12235 HM_DISABLE_PREEMPT(pVCpu);
12236
12237 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12238
12239 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
12240 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12241 {
12242 /* Raise #DB. */
12243 if (fIsGuestDbgActive)
12244 ASMSetDR6(pCtx->dr[6]);
12245 if (pCtx->dr[7] != uDr7)
12246 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_DR7;
12247
12248 hmR0VmxSetPendingXcptDB(pVCpu);
12249 }
12250 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12251 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12252 else if ( rcStrict2 != VINF_SUCCESS
12253 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12254 rcStrict = rcStrict2;
12255 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12256
12257 HM_RESTORE_PREEMPT();
12258 VMMRZCallRing3Enable(pVCpu);
12259 }
12260 }
12261
12262#ifdef VBOX_STRICT
12263 if ( rcStrict == VINF_IOM_R3_IOPORT_READ
12264 || rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
12265 Assert(!fIOWrite);
12266 else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
12267 || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
12268 || rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
12269 Assert(fIOWrite);
12270 else
12271 {
12272# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12273 * statuses, that the VMM device and some others may return. See
12274 * IOM_SUCCESS() for guidance. */
12275 AssertMsg( RT_FAILURE(rcStrict)
12276 || rcStrict == VINF_SUCCESS
12277 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12278 || rcStrict == VINF_EM_DBG_BREAKPOINT
12279 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12280 || rcStrict == VINF_EM_RAW_TO_R3
12281 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12282# endif
12283 }
12284#endif
12285 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12286 }
12287 else
12288 {
12289 /*
12290 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12291 */
12292 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12293 AssertRCReturn(rc2, rc2);
12294 STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
12295 : fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
12296 Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12297 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12298 VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification) ? "REP " : "",
12299 fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
12300
12301 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12302 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12303
12304 Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12305 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12306 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12307 }
12308 return rcStrict;
12309}
12310
12311
12312/**
12313 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12314 * VM-exit.
12315 */
12316HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12317{
12318 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12319
12320 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12321 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12322 AssertRCReturn(rc, rc);
12323 if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12324 {
12325 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12326 AssertRCReturn(rc, rc);
12327 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12328 {
12329 uint32_t uErrCode;
12330 RTGCUINTPTR GCPtrFaultAddress;
12331 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12332 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12333 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
12334 if (fErrorCodeValid)
12335 {
12336 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12337 AssertRCReturn(rc, rc);
12338 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12339 }
12340 else
12341 uErrCode = 0;
12342
12343 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12344 && uVector == X86_XCPT_PF)
12345 GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
12346 else
12347 GCPtrFaultAddress = 0;
12348
12349 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12350 0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
12351
12352 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12353 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12354 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12355 }
12356 }
12357
12358 /* Fall back to the interpreter to emulate the task-switch. */
12359 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12360 return VERR_EM_INTERPRETER;
12361}
12362
12363
12364/**
12365 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12366 */
12367HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12368{
12369 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12370 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12371 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12372 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12373 AssertRCReturn(rc, rc);
12374 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12375 return VINF_EM_DBG_STEPPED;
12376}
12377
12378
12379/**
12380 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12381 */
12382HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12383{
12384 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12385
12386 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12387
12388 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12389 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12390 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12391 {
12392 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12393 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12394 {
12395 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12396 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12397 }
12398 }
12399 else
12400 {
12401 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12402 rcStrict1 = VINF_SUCCESS;
12403 return rcStrict1;
12404 }
12405
12406 /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12407 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12408 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12409 AssertRCReturn(rc, rc);
12410
12411 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12412 uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12413 VBOXSTRICTRC rcStrict2;
12414 switch (uAccessType)
12415 {
12416 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12417 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12418 {
12419 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12420 || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
12421 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12422
12423 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12424 GCPhys &= PAGE_BASE_GC_MASK;
12425 GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12426 PVM pVM = pVCpu->CTX_SUFF(pVM);
12427 Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12428 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12429
12430 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12431 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12432 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12433 CPUMCTX2CORE(pCtx), GCPhys);
12434 Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12435 if ( rcStrict2 == VINF_SUCCESS
12436 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12437 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12438 {
12439 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12440 | HM_CHANGED_GUEST_APIC_TPR);
12441 rcStrict2 = VINF_SUCCESS;
12442 }
12443 break;
12444 }
12445
12446 default:
12447 Log4Func(("uAccessType=%#x\n", uAccessType));
12448 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12449 break;
12450 }
12451
12452 if (rcStrict2 != VINF_SUCCESS)
12453 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12454 return rcStrict2;
12455}
12456
12457
12458/**
12459 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12460 * VM-exit.
12461 */
12462HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12463{
12464 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12465
12466 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12467 if (pVmxTransient->fWasGuestDebugStateActive)
12468 {
12469 AssertMsgFailed(("Unexpected MOV DRx exit\n"));
12470 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12471 }
12472
12473 if ( !pVCpu->hm.s.fSingleInstruction
12474 && !pVmxTransient->fWasHyperDebugStateActive)
12475 {
12476 Assert(!DBGFIsStepping(pVCpu));
12477 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12478
12479 /* Don't intercept MOV DRx any more. */
12480 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12481 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12482 AssertRCReturn(rc, rc);
12483
12484 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12485 VMMRZCallRing3Disable(pVCpu);
12486 HM_DISABLE_PREEMPT(pVCpu);
12487
12488 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12489 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12490 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12491
12492 HM_RESTORE_PREEMPT();
12493 VMMRZCallRing3Enable(pVCpu);
12494
12495#ifdef VBOX_WITH_STATISTICS
12496 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12497 AssertRCReturn(rc, rc);
12498 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12499 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12500 else
12501 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12502#endif
12503 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12504 return VINF_SUCCESS;
12505 }
12506
12507 /*
12508 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12509 * Update the segment registers and DR7 from the CPU.
12510 */
12511 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12512 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12513 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_DR7);
12514 AssertRCReturn(rc, rc);
12515 Log4Func(("CS:RIP=%04x:%08RX64\n", pCtx->cs.Sel, pCtx->rip));
12516
12517 PVM pVM = pVCpu->CTX_SUFF(pVM);
12518 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12519 {
12520 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12521 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification),
12522 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification));
12523 if (RT_SUCCESS(rc))
12524 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12525 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12526 }
12527 else
12528 {
12529 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12530 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification),
12531 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification));
12532 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12533 }
12534
12535 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12536 if (RT_SUCCESS(rc))
12537 {
12538 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12539 AssertRCReturn(rc2, rc2);
12540 return VINF_SUCCESS;
12541 }
12542 return rc;
12543}
12544
12545
12546/**
12547 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12548 * Conditional VM-exit.
12549 */
12550HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12551{
12552 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12553 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12554
12555 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12556 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12557 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12558 {
12559 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12560 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12561 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12562 {
12563 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12564 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12565 }
12566 }
12567 else
12568 {
12569 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12570 rcStrict1 = VINF_SUCCESS;
12571 return rcStrict1;
12572 }
12573
12574 /*
12575 * Get sufficent state and update the exit history entry.
12576 */
12577 RTGCPHYS GCPhys;
12578 int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
12579 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12580 AssertRCReturn(rc, rc);
12581
12582 VBOXSTRICTRC rcStrict;
12583 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12584 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
12585 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12586 if (!pExitRec)
12587 {
12588 /*
12589 * If we succeed, resume guest execution.
12590 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12591 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12592 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12593 * weird case. See @bugref{6043}.
12594 */
12595 PVM pVM = pVCpu->CTX_SUFF(pVM);
12596 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12597 rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
12598 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12599 if ( rcStrict == VINF_SUCCESS
12600 || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12601 || rcStrict == VERR_PAGE_NOT_PRESENT)
12602 {
12603 /* Successfully handled MMIO operation. */
12604 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12605 | HM_CHANGED_GUEST_APIC_TPR);
12606 rcStrict = VINF_SUCCESS;
12607 }
12608 }
12609 else
12610 {
12611 /*
12612 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12613 */
12614 int rc2 = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12615 AssertRCReturn(rc2, rc2);
12616
12617 Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12618 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12619
12620 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12621 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12622
12623 Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12624 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12625 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12626 }
12627 return VBOXSTRICTRC_TODO(rcStrict);
12628}
12629
12630
12631/**
12632 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12633 * VM-exit.
12634 */
12635HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12636{
12637 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12638 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12639
12640 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12641 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12642 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12643 {
12644 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12645 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12646 Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12647 }
12648 else
12649 {
12650 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12651 rcStrict1 = VINF_SUCCESS;
12652 return rcStrict1;
12653 }
12654
12655 RTGCPHYS GCPhys;
12656 int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
12657 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12658 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12659 AssertRCReturn(rc, rc);
12660
12661 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12662 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12663
12664 RTGCUINT uErrorCode = 0;
12665 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
12666 uErrorCode |= X86_TRAP_PF_ID;
12667 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_DATA_WRITE)
12668 uErrorCode |= X86_TRAP_PF_RW;
12669 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
12670 uErrorCode |= X86_TRAP_PF_P;
12671
12672 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12673
12674
12675 /* Handle the pagefault trap for the nested shadow table. */
12676 PVM pVM = pVCpu->CTX_SUFF(pVM);
12677 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12678
12679 Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12680 uErrorCode, pCtx->cs.Sel, pCtx->rip));
12681
12682 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
12683 TRPMResetTrap(pVCpu);
12684
12685 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12686 if ( rcStrict2 == VINF_SUCCESS
12687 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12688 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12689 {
12690 /* Successfully synced our nested page tables. */
12691 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12692 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
12693 return VINF_SUCCESS;
12694 }
12695
12696 Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12697 return rcStrict2;
12698}
12699
12700/** @} */
12701
12702/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12703/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12704/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12705
12706/** @name VM-exit exception handlers.
12707 * @{
12708 */
12709
12710/**
12711 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12712 */
12713static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12714{
12715 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12716 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12717
12718 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
12719 AssertRCReturn(rc, rc);
12720
12721 if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
12722 {
12723 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12724 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12725
12726 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12727 * provides VM-exit instruction length. If this causes problem later,
12728 * disassemble the instruction like it's done on AMD-V. */
12729 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12730 AssertRCReturn(rc2, rc2);
12731 return rc;
12732 }
12733
12734 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12735 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12736 return rc;
12737}
12738
12739
12740/**
12741 * VM-exit exception handler for \#BP (Breakpoint exception).
12742 */
12743static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12744{
12745 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12746 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12747
12748 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12749 AssertRCReturn(rc, rc);
12750
12751 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12752 rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
12753 if (rc == VINF_EM_RAW_GUEST_TRAP)
12754 {
12755 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12756 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12757 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12758 AssertRCReturn(rc, rc);
12759
12760 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12761 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12762 }
12763
12764 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
12765 return rc;
12766}
12767
12768
12769/**
12770 * VM-exit exception handler for \#AC (alignment check exception).
12771 */
12772static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12773{
12774 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12775
12776 /*
12777 * Re-inject it. We'll detect any nesting before getting here.
12778 */
12779 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12780 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12781 AssertRCReturn(rc, rc);
12782 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
12783
12784 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12785 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12786 return VINF_SUCCESS;
12787}
12788
12789
12790/**
12791 * VM-exit exception handler for \#DB (Debug exception).
12792 */
12793static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12794{
12795 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12796 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
12797
12798 /*
12799 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
12800 * for processing.
12801 */
12802 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12803
12804 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
12805 uint64_t uDR6 = X86_DR6_INIT_VAL;
12806 uDR6 |= ( pVmxTransient->uExitQualification
12807 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
12808
12809 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12810 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
12811 Log6Func(("rc=%Rrc\n", rc));
12812 if (rc == VINF_EM_RAW_GUEST_TRAP)
12813 {
12814 /*
12815 * The exception was for the guest. Update DR6, DR7.GD and
12816 * IA32_DEBUGCTL.LBR before forwarding it.
12817 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
12818 */
12819 VMMRZCallRing3Disable(pVCpu);
12820 HM_DISABLE_PREEMPT(pVCpu);
12821
12822 pCtx->dr[6] &= ~X86_DR6_B_MASK;
12823 pCtx->dr[6] |= uDR6;
12824 if (CPUMIsGuestDebugStateActive(pVCpu))
12825 ASMSetDR6(pCtx->dr[6]);
12826
12827 HM_RESTORE_PREEMPT();
12828 VMMRZCallRing3Enable(pVCpu);
12829
12830 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12831 AssertRCReturn(rc, rc);
12832
12833 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
12834 pCtx->dr[7] &= ~X86_DR7_GD;
12835
12836 /* Paranoia. */
12837 pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
12838 pCtx->dr[7] |= X86_DR7_RA1_MASK;
12839
12840 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
12841 AssertRCReturn(rc, rc);
12842
12843 /*
12844 * Raise #DB in the guest.
12845 *
12846 * It is important to reflect exactly what the VM-exit gave us (preserving the
12847 * interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
12848 * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
12849 * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
12850 *
12851 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
12852 * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
12853 */
12854 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12855 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12856 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12857 AssertRCReturn(rc, rc);
12858 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12859 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12860 return VINF_SUCCESS;
12861 }
12862
12863 /*
12864 * Not a guest trap, must be a hypervisor related debug event then.
12865 * Update DR6 in case someone is interested in it.
12866 */
12867 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
12868 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
12869 CPUMSetHyperDR6(pVCpu, uDR6);
12870
12871 return rc;
12872}
12873
12874/**
12875 * VM-exit exception handler for \#GP (General-protection exception).
12876 *
12877 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
12878 */
12879static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12880{
12881 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12882 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
12883
12884 int rc;
12885 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12886 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
12887 { /* likely */ }
12888 else
12889 {
12890#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12891 Assert(pVCpu->hm.s.fUsingDebugLoop);
12892#endif
12893 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
12894 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12895 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12896 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12897 rc |= hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12898 AssertRCReturn(rc, rc);
12899 Log4Func(("Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pCtx->cs.Sel, pCtx->rip,
12900 pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
12901 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12902 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12903 return rc;
12904 }
12905
12906 Assert(CPUMIsGuestInRealModeEx(pCtx));
12907 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
12908
12909 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
12910 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12911 AssertRCReturn(rc, rc);
12912
12913 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12914 uint32_t cbOp = 0;
12915 PVM pVM = pVCpu->CTX_SUFF(pVM);
12916 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12917 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
12918 if (RT_SUCCESS(rc))
12919 {
12920 rc = VINF_SUCCESS;
12921 Assert(cbOp == pDis->cbInstr);
12922 Log4Func(("Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pCtx->cs.Sel, pCtx->rip));
12923 switch (pDis->pCurInstr->uOpcode)
12924 {
12925 case OP_CLI:
12926 {
12927 pCtx->eflags.Bits.u1IF = 0;
12928 pCtx->eflags.Bits.u1RF = 0;
12929 pCtx->rip += pDis->cbInstr;
12930 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12931 if ( !fDbgStepping
12932 && pCtx->eflags.Bits.u1TF)
12933 {
12934 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12935 AssertRCReturn(rc, rc);
12936 }
12937 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
12938 break;
12939 }
12940
12941 case OP_STI:
12942 {
12943 bool fOldIF = pCtx->eflags.Bits.u1IF;
12944 pCtx->eflags.Bits.u1IF = 1;
12945 pCtx->eflags.Bits.u1RF = 0;
12946 pCtx->rip += pDis->cbInstr;
12947 if (!fOldIF)
12948 {
12949 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
12950 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
12951 }
12952 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12953 if ( !fDbgStepping
12954 && pCtx->eflags.Bits.u1TF)
12955 {
12956 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12957 AssertRCReturn(rc, rc);
12958 }
12959 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
12960 break;
12961 }
12962
12963 case OP_HLT:
12964 {
12965 rc = VINF_EM_HALT;
12966 pCtx->rip += pDis->cbInstr;
12967 pCtx->eflags.Bits.u1RF = 0;
12968 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12969 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
12970 break;
12971 }
12972
12973 case OP_POPF:
12974 {
12975 Log4Func(("POPF CS:EIP %04x:%04RX64\n", pCtx->cs.Sel, pCtx->rip));
12976 uint32_t cbParm;
12977 uint32_t uMask;
12978 bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
12979 if (pDis->fPrefix & DISPREFIX_OPSIZE)
12980 {
12981 cbParm = 4;
12982 uMask = 0xffffffff;
12983 }
12984 else
12985 {
12986 cbParm = 2;
12987 uMask = 0xffff;
12988 }
12989
12990 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
12991 RTGCPTR GCPtrStack = 0;
12992 X86EFLAGS Eflags;
12993 Eflags.u32 = 0;
12994 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
12995 &GCPtrStack);
12996 if (RT_SUCCESS(rc))
12997 {
12998 Assert(sizeof(Eflags.u32) >= cbParm);
12999 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13000 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13001 }
13002 if (RT_FAILURE(rc))
13003 {
13004 rc = VERR_EM_INTERPRETER;
13005 break;
13006 }
13007 Log4Func(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pCtx->rsp, uMask, pCtx->rip));
13008 pCtx->eflags.u32 = (pCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13009 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13010 pCtx->esp += cbParm;
13011 pCtx->esp &= uMask;
13012 pCtx->rip += pDis->cbInstr;
13013 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
13014 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13015 POPF restores EFLAGS.TF. */
13016 if ( !fDbgStepping
13017 && fGstStepping)
13018 {
13019 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13020 AssertRCReturn(rc, rc);
13021 }
13022 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13023 break;
13024 }
13025
13026 case OP_PUSHF:
13027 {
13028 uint32_t cbParm;
13029 uint32_t uMask;
13030 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13031 {
13032 cbParm = 4;
13033 uMask = 0xffffffff;
13034 }
13035 else
13036 {
13037 cbParm = 2;
13038 uMask = 0xffff;
13039 }
13040
13041 /* Get the stack pointer & push the contents of eflags onto the stack. */
13042 RTGCPTR GCPtrStack = 0;
13043 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask,
13044 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13045 if (RT_FAILURE(rc))
13046 {
13047 rc = VERR_EM_INTERPRETER;
13048 break;
13049 }
13050 X86EFLAGS Eflags = pCtx->eflags;
13051 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13052 Eflags.Bits.u1RF = 0;
13053 Eflags.Bits.u1VM = 0;
13054
13055 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13056 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13057 {
13058 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13059 rc = VERR_EM_INTERPRETER;
13060 break;
13061 }
13062 Log4Func(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13063 pCtx->esp -= cbParm;
13064 pCtx->esp &= uMask;
13065 pCtx->rip += pDis->cbInstr;
13066 pCtx->eflags.Bits.u1RF = 0;
13067 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
13068 if ( !fDbgStepping
13069 && pCtx->eflags.Bits.u1TF)
13070 {
13071 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13072 AssertRCReturn(rc, rc);
13073 }
13074 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13075 break;
13076 }
13077
13078 case OP_IRET:
13079 {
13080 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13081 * instruction reference. */
13082 RTGCPTR GCPtrStack = 0;
13083 uint32_t uMask = 0xffff;
13084 bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
13085 uint16_t aIretFrame[3];
13086 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13087 {
13088 rc = VERR_EM_INTERPRETER;
13089 break;
13090 }
13091 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13092 &GCPtrStack);
13093 if (RT_SUCCESS(rc))
13094 {
13095 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13096 PGMACCESSORIGIN_HM));
13097 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13098 }
13099 if (RT_FAILURE(rc))
13100 {
13101 rc = VERR_EM_INTERPRETER;
13102 break;
13103 }
13104 pCtx->eip = 0;
13105 pCtx->ip = aIretFrame[0];
13106 pCtx->cs.Sel = aIretFrame[1];
13107 pCtx->cs.ValidSel = aIretFrame[1];
13108 pCtx->cs.u64Base = (uint64_t)pCtx->cs.Sel << 4;
13109 pCtx->eflags.u32 = (pCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13110 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13111 pCtx->sp += sizeof(aIretFrame);
13112 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
13113 | HM_CHANGED_GUEST_CS);
13114 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13115 if ( !fDbgStepping
13116 && fGstStepping)
13117 {
13118 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13119 AssertRCReturn(rc, rc);
13120 }
13121 Log4Func(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pCtx->cs.Sel, pCtx->ip));
13122 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13123 break;
13124 }
13125
13126 case OP_INT:
13127 {
13128 uint16_t uVector = pDis->Param1.uValue & 0xff;
13129 hmR0VmxSetPendingIntN(pVCpu, uVector, pDis->cbInstr);
13130 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13131 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13132 break;
13133 }
13134
13135 case OP_INTO:
13136 {
13137 if (pCtx->eflags.Bits.u1OF)
13138 {
13139 hmR0VmxSetPendingXcptOF(pVCpu, pDis->cbInstr);
13140 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13141 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13142 }
13143 else
13144 {
13145 pCtx->eflags.Bits.u1RF = 0;
13146 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
13147 }
13148 break;
13149 }
13150
13151 default:
13152 {
13153 pCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13154 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pCtx), 0 /* pvFault */,
13155 EMCODETYPE_SUPERVISOR);
13156 rc = VBOXSTRICTRC_VAL(rc2);
13157 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13158 /** @todo We have to set pending-debug exceptions here when the guest is
13159 * single-stepping depending on the instruction that was interpreted. */
13160
13161 /*
13162 * HACK ALERT! Detect mode change and go to ring-3 to properly exit this
13163 * real mode emulation stuff.
13164 */
13165 if ( rc == VINF_SUCCESS
13166 && (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
13167 {
13168 Log4(("hmR0VmxExitXcptGP: mode changed -> VINF_EM_RESCHEDULE\n"));
13169 /** @todo Exit fRealOnV86Active here w/o dropping back to ring-3. */
13170 rc = VINF_EM_RESCHEDULE;
13171 }
13172
13173 Log4Func(("#GP rc=%Rrc\n", rc));
13174 break;
13175 }
13176 }
13177 }
13178 else
13179 rc = VERR_EM_INTERPRETER;
13180
13181 AssertMsg( rc == VINF_SUCCESS
13182 || rc == VERR_EM_INTERPRETER
13183 || rc == VINF_EM_HALT
13184 || rc == VINF_EM_RESCHEDULE
13185 , ("#GP Unexpected rc=%Rrc\n", rc));
13186 return rc;
13187}
13188
13189
13190/**
13191 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13192 * the exception reported in the VMX transient structure back into the VM.
13193 *
13194 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13195 * up-to-date.
13196 */
13197static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13198{
13199 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13200#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13201 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13202 ("uVector=%#x u32XcptBitmap=%#X32\n",
13203 VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13204#endif
13205
13206 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13207 hmR0VmxCheckExitDueToEventDelivery(). */
13208 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13209 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13210 AssertRCReturn(rc, rc);
13211 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13212
13213#ifdef DEBUG_ramshankar
13214 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
13215 uint8_t uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13216 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13217#endif
13218
13219 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13220 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13221 return VINF_SUCCESS;
13222}
13223
13224
13225/**
13226 * VM-exit exception handler for \#PF (Page-fault exception).
13227 */
13228static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13229{
13230 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13231 PVM pVM = pVCpu->CTX_SUFF(pVM);
13232 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13233 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13234 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13235 AssertRCReturn(rc, rc);
13236
13237 if (!pVM->hm.s.fNestedPaging)
13238 { /* likely */ }
13239 else
13240 {
13241#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13242 Assert(pVCpu->hm.s.fUsingDebugLoop);
13243#endif
13244 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13245 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13246 {
13247 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13248 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13249 }
13250 else
13251 {
13252 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13253 hmR0VmxSetPendingXcptDF(pVCpu);
13254 Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13255 }
13256 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13257 return rc;
13258 }
13259
13260 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13261 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13262 if (pVmxTransient->fVectoringPF)
13263 {
13264 Assert(pVCpu->hm.s.Event.fPending);
13265 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13266 }
13267
13268 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13269 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13270 AssertRCReturn(rc, rc);
13271
13272 Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13273 pCtx->cs.Sel, pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
13274
13275 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13276 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx),
13277 (RTGCPTR)pVmxTransient->uExitQualification);
13278
13279 Log4Func(("#PF: rc=%Rrc\n", rc));
13280 if (rc == VINF_SUCCESS)
13281 {
13282 /*
13283 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13284 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13285 */
13286 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13287 TRPMResetTrap(pVCpu);
13288 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13289 return rc;
13290 }
13291
13292 if (rc == VINF_EM_RAW_GUEST_TRAP)
13293 {
13294 if (!pVmxTransient->fVectoringDoublePF)
13295 {
13296 /* It's a guest page fault and needs to be reflected to the guest. */
13297 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13298 TRPMResetTrap(pVCpu);
13299 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13300 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13301 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13302 }
13303 else
13304 {
13305 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13306 TRPMResetTrap(pVCpu);
13307 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13308 hmR0VmxSetPendingXcptDF(pVCpu);
13309 Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13310 }
13311
13312 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13313 return VINF_SUCCESS;
13314 }
13315
13316 TRPMResetTrap(pVCpu);
13317 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13318 return rc;
13319}
13320
13321/** @} */
13322
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette