VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 73754

Last change on this file since 73754 was 73754, checked in by vboxsync, 6 years ago

VMM/HVMXMR0: const nit.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 567.8 KB
Line 
1/* $Id: HMVMXR0.cpp 73754 2018-08-18 04:48:52Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/x86.h>
25#include <iprt/asm-amd64-x86.h>
26#include <iprt/thread.h>
27
28#include <VBox/vmm/pdmapi.h>
29#include <VBox/vmm/dbgf.h>
30#include <VBox/vmm/iem.h>
31#include <VBox/vmm/iom.h>
32#include <VBox/vmm/selm.h>
33#include <VBox/vmm/tm.h>
34#include <VBox/vmm/em.h>
35#include <VBox/vmm/gim.h>
36#include <VBox/vmm/apic.h>
37#ifdef VBOX_WITH_REM
38# include <VBox/vmm/rem.h>
39#endif
40#include "HMInternal.h"
41#include <VBox/vmm/vm.h>
42#include "HMVMXR0.h"
43#include "dtrace/VBoxVMM.h"
44
45#ifdef DEBUG_ramshankar
46# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
47# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
49# define HMVMX_ALWAYS_CHECK_GUEST_STATE
50# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
51# define HMVMX_ALWAYS_TRAP_PF
52# define HMVMX_ALWAYS_FLUSH_TLB
53# define HMVMX_ALWAYS_SWAP_EFER
54#endif
55
56
57/*********************************************************************************************************************************
58* Defined Constants And Macros *
59*********************************************************************************************************************************/
60/** Use the function table. */
61#define HMVMX_USE_FUNCTION_TABLE
62
63/** Determine which tagged-TLB flush handler to use. */
64#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
65#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
66#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
67#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
68
69/** @name HMVMX_READ_XXX
70 * Flags to skip redundant reads of some common VMCS fields that are not part of
71 * the guest-CPU or VCPU state but are needed while handling VM-exits.
72 */
73#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
74#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
75#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
76#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
77#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
78#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
79#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
80/** @} */
81
82/**
83 * States of the VMCS.
84 *
85 * This does not reflect all possible VMCS states but currently only those
86 * needed for maintaining the VMCS consistently even when thread-context hooks
87 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
88 */
89#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
90#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
91#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
92
93/**
94 * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
95 * guest using hardware-assisted VMX.
96 *
97 * This excludes state like GPRs (other than RSP) which are always are
98 * swapped and restored across the world-switch and also registers like EFER,
99 * MSR which cannot be modified by the guest without causing a VM-exit.
100 */
101#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
102 | CPUMCTX_EXTRN_RFLAGS \
103 | CPUMCTX_EXTRN_RSP \
104 | CPUMCTX_EXTRN_SREG_MASK \
105 | CPUMCTX_EXTRN_TABLE_MASK \
106 | CPUMCTX_EXTRN_KERNEL_GS_BASE \
107 | CPUMCTX_EXTRN_SYSCALL_MSRS \
108 | CPUMCTX_EXTRN_SYSENTER_MSRS \
109 | CPUMCTX_EXTRN_TSC_AUX \
110 | CPUMCTX_EXTRN_OTHER_MSRS \
111 | CPUMCTX_EXTRN_CR0 \
112 | CPUMCTX_EXTRN_CR3 \
113 | CPUMCTX_EXTRN_CR4 \
114 | CPUMCTX_EXTRN_DR7 \
115 | CPUMCTX_EXTRN_HM_VMX_MASK)
116
117/**
118 * Exception bitmap mask for real-mode guests (real-on-v86).
119 *
120 * We need to intercept all exceptions manually except:
121 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
122 * due to bugs in Intel CPUs.
123 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
124 * support.
125 */
126#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
127 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
128 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
129 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
130 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
131 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
132 | RT_BIT(X86_XCPT_XF))
133
134/** Maximum VM-instruction error number. */
135#define HMVMX_INSTR_ERROR_MAX 28
136
137/** Profiling macro. */
138#ifdef HM_PROFILE_EXIT_DISPATCH
139# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
140# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
141#else
142# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
143# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
144#endif
145
146/** Assert that preemption is disabled or covered by thread-context hooks. */
147#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
148 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
149
150/** Assert that we haven't migrated CPUs when thread-context hooks are not
151 * used. */
152#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
153 || (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
154 ("Illegal migration! Entered on CPU %u Current %u\n", \
155 (a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
156
157/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
158 * context. */
159#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
160 ("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
161 (a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
162
163/** Helper macro for VM-exit handlers called unexpectedly. */
164#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
165 do { \
166 (a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
167 return VERR_VMX_UNEXPECTED_EXIT; \
168 } while (0)
169
170/** Macro for importing segment registers to the VMCS from the guest-CPU context. */
171#ifdef VMX_USE_CACHED_VMCS_ACCESSES
172# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
173 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
174 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
175#else
176# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
177 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
178 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
179#endif
180
181/** Macro for exporting segment registers to the VMCS from the guest-CPU context. */
182# define HMVMX_EXPORT_SREG(Sel, a_pCtxSelReg) \
183 hmR0VmxExportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
184 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
185
186
187/*********************************************************************************************************************************
188* Structures and Typedefs *
189*********************************************************************************************************************************/
190/**
191 * VMX transient state.
192 *
193 * A state structure for holding miscellaneous information across
194 * VMX non-root operation and restored after the transition.
195 */
196typedef struct VMXTRANSIENT
197{
198 /** The host's rflags/eflags. */
199 RTCCUINTREG fEFlags;
200#if HC_ARCH_BITS == 32
201 uint32_t u32Alignment0;
202#endif
203 /** The guest's TPR value used for TPR shadowing. */
204 uint8_t u8GuestTpr;
205 /** Alignment. */
206 uint8_t abAlignment0[7];
207
208 /** The basic VM-exit reason. */
209 uint16_t uExitReason;
210 /** Alignment. */
211 uint16_t u16Alignment0;
212 /** The VM-exit interruption error code. */
213 uint32_t uExitIntErrorCode;
214 /** The VM-exit exit code qualification. */
215 uint64_t uExitQual;
216
217 /** The VM-exit interruption-information field. */
218 uint32_t uExitIntInfo;
219 /** The VM-exit instruction-length field. */
220 uint32_t cbInstr;
221 /** The VM-exit instruction-information field. */
222 VMXEXITINSTRINFO ExitInstrInfo;
223 /** Whether the VM-entry failed or not. */
224 bool fVMEntryFailed;
225 /** Alignment. */
226 uint8_t abAlignment1[3];
227
228 /** The VM-entry interruption-information field. */
229 uint32_t uEntryIntInfo;
230 /** The VM-entry exception error code field. */
231 uint32_t uEntryXcptErrorCode;
232 /** The VM-entry instruction length field. */
233 uint32_t cbEntryInstr;
234
235 /** IDT-vectoring information field. */
236 uint32_t uIdtVectoringInfo;
237 /** IDT-vectoring error code. */
238 uint32_t uIdtVectoringErrorCode;
239
240 /** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
241 uint32_t fVmcsFieldsRead;
242
243 /** Whether the guest debug state was active at the time of VM-exit. */
244 bool fWasGuestDebugStateActive;
245 /** Whether the hyper debug state was active at the time of VM-exit. */
246 bool fWasHyperDebugStateActive;
247 /** Whether TSC-offsetting should be setup before VM-entry. */
248 bool fUpdateTscOffsettingAndPreemptTimer;
249 /** Whether the VM-exit was caused by a page-fault during delivery of a
250 * contributory exception or a page-fault. */
251 bool fVectoringDoublePF;
252 /** Whether the VM-exit was caused by a page-fault during delivery of an
253 * external interrupt or NMI. */
254 bool fVectoringPF;
255} VMXTRANSIENT;
256AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
257AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
258AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
259AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
260AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
261/** Pointer to VMX transient state. */
262typedef VMXTRANSIENT *PVMXTRANSIENT;
263
264
265/**
266 * MSR-bitmap read permissions.
267 */
268typedef enum VMXMSREXITREAD
269{
270 /** Reading this MSR causes a VM-exit. */
271 VMXMSREXIT_INTERCEPT_READ = 0xb,
272 /** Reading this MSR does not cause a VM-exit. */
273 VMXMSREXIT_PASSTHRU_READ
274} VMXMSREXITREAD;
275/** Pointer to MSR-bitmap read permissions. */
276typedef VMXMSREXITREAD* PVMXMSREXITREAD;
277
278/**
279 * MSR-bitmap write permissions.
280 */
281typedef enum VMXMSREXITWRITE
282{
283 /** Writing to this MSR causes a VM-exit. */
284 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
285 /** Writing to this MSR does not cause a VM-exit. */
286 VMXMSREXIT_PASSTHRU_WRITE
287} VMXMSREXITWRITE;
288/** Pointer to MSR-bitmap write permissions. */
289typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
290
291
292/**
293 * VMX VM-exit handler.
294 *
295 * @returns Strict VBox status code (i.e. informational status codes too).
296 * @param pVCpu The cross context virtual CPU structure.
297 * @param pVmxTransient Pointer to the VMX-transient structure.
298 */
299#ifndef HMVMX_USE_FUNCTION_TABLE
300typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
301#else
302typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
303/** Pointer to VM-exit handler. */
304typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
305#endif
306
307/**
308 * VMX VM-exit handler, non-strict status code.
309 *
310 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
311 *
312 * @returns VBox status code, no informational status code returned.
313 * @param pVCpu The cross context virtual CPU structure.
314 * @param pVmxTransient Pointer to the VMX-transient structure.
315 *
316 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
317 * use of that status code will be replaced with VINF_EM_SOMETHING
318 * later when switching over to IEM.
319 */
320#ifndef HMVMX_USE_FUNCTION_TABLE
321typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
322#else
323typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
324#endif
325
326
327/*********************************************************************************************************************************
328* Internal Functions *
329*********************************************************************************************************************************/
330static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush);
331static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr);
332static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
333static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat);
334static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
335 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState);
336#if HC_ARCH_BITS == 32
337static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
338#endif
339#ifndef HMVMX_USE_FUNCTION_TABLE
340DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
341# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
342# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
343#else
344# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
345# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
346#endif
347
348
349/** @name VM-exit handlers.
350 * @{
351 */
352static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
353static FNVMXEXITHANDLER hmR0VmxExitExtInt;
354static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
355static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
356static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
357static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
358static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
359static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
360static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
361static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
362static FNVMXEXITHANDLER hmR0VmxExitCpuid;
363static FNVMXEXITHANDLER hmR0VmxExitGetsec;
364static FNVMXEXITHANDLER hmR0VmxExitHlt;
365static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
366static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
367static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
368static FNVMXEXITHANDLER hmR0VmxExitVmcall;
369#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
370static FNVMXEXITHANDLER hmR0VmxExitVmclear;
371static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
372static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
373static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
374static FNVMXEXITHANDLER hmR0VmxExitVmread;
375static FNVMXEXITHANDLER hmR0VmxExitVmresume;
376static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
377static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
378static FNVMXEXITHANDLER hmR0VmxExitVmxon;
379#endif
380static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
381static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
382static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
383static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
384static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
385static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
386static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
387static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
388static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
389static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
390static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
391static FNVMXEXITHANDLER hmR0VmxExitMwait;
392static FNVMXEXITHANDLER hmR0VmxExitMtf;
393static FNVMXEXITHANDLER hmR0VmxExitMonitor;
394static FNVMXEXITHANDLER hmR0VmxExitPause;
395static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
396static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
397static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
398static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
399static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
400static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
401static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
402static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
403static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
404static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
405static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
406static FNVMXEXITHANDLER hmR0VmxExitRdrand;
407static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
408/** @} */
409
410static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
411static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
412static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
413static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
414static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
415static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
416static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
417static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu);
418
419
420/*********************************************************************************************************************************
421* Global Variables *
422*********************************************************************************************************************************/
423#ifdef HMVMX_USE_FUNCTION_TABLE
424
425/**
426 * VMX_EXIT dispatch table.
427 */
428static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
429{
430 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
431 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
432 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
433 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
434 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
435 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
436 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
437 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
438 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
439 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
440 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
441 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
442 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
443 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
444 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
445 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
446 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
447 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
448 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
449#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
450 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
451 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
452 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
453 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
454 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
455 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
456 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
457 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
458 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
459#else
460 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
461 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
462 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
463 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
464 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
465 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
466 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
467 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
468 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
469#endif
470 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
471 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
472 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
473 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
474 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
475 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
476 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
477 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
478 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
479 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
480 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
481 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
482 /* 40 UNDEFINED */ hmR0VmxExitPause,
483 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
484 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
485 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
486 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
487 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
488 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
489 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
490 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
491 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
492 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
493 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
494 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
495 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
496 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
497 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
498 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
499 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
500 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
501 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
502 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
503 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
504 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
505 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
506 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
507};
508#endif /* HMVMX_USE_FUNCTION_TABLE */
509
510#ifdef VBOX_STRICT
511static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
512{
513 /* 0 */ "(Not Used)",
514 /* 1 */ "VMCALL executed in VMX root operation.",
515 /* 2 */ "VMCLEAR with invalid physical address.",
516 /* 3 */ "VMCLEAR with VMXON pointer.",
517 /* 4 */ "VMLAUNCH with non-clear VMCS.",
518 /* 5 */ "VMRESUME with non-launched VMCS.",
519 /* 6 */ "VMRESUME after VMXOFF",
520 /* 7 */ "VM-entry with invalid control fields.",
521 /* 8 */ "VM-entry with invalid host state fields.",
522 /* 9 */ "VMPTRLD with invalid physical address.",
523 /* 10 */ "VMPTRLD with VMXON pointer.",
524 /* 11 */ "VMPTRLD with incorrect revision identifier.",
525 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
526 /* 13 */ "VMWRITE to read-only VMCS component.",
527 /* 14 */ "(Not Used)",
528 /* 15 */ "VMXON executed in VMX root operation.",
529 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
530 /* 17 */ "VM-entry with non-launched executing VMCS.",
531 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
532 /* 19 */ "VMCALL with non-clear VMCS.",
533 /* 20 */ "VMCALL with invalid VM-exit control fields.",
534 /* 21 */ "(Not Used)",
535 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
536 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
537 /* 24 */ "VMCALL with invalid SMM-monitor features.",
538 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
539 /* 26 */ "VM-entry with events blocked by MOV SS.",
540 /* 27 */ "(Not Used)",
541 /* 28 */ "Invalid operand to INVEPT/INVVPID."
542};
543#endif /* VBOX_STRICT */
544
545
546
547/**
548 * Updates the VM's last error record.
549 *
550 * If there was a VMX instruction error, reads the error data from the VMCS and
551 * updates VCPU's last error record as well.
552 *
553 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
554 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
555 * VERR_VMX_INVALID_VMCS_FIELD.
556 * @param rc The error code.
557 */
558static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
559{
560 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
561 || rc == VERR_VMX_UNABLE_TO_START_VM)
562 {
563 AssertPtrReturnVoid(pVCpu);
564 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
565 }
566 pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
567}
568
569
570/**
571 * Reads the VM-entry interruption-information field from the VMCS into the VMX
572 * transient structure.
573 *
574 * @returns VBox status code.
575 * @param pVmxTransient Pointer to the VMX transient structure.
576 *
577 * @remarks No-long-jump zone!!!
578 */
579DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
580{
581 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
582 AssertRCReturn(rc, rc);
583 return VINF_SUCCESS;
584}
585
586#ifdef VBOX_STRICT
587/**
588 * Reads the VM-entry exception error code field from the VMCS into
589 * the VMX transient structure.
590 *
591 * @returns VBox status code.
592 * @param pVmxTransient Pointer to the VMX transient structure.
593 *
594 * @remarks No-long-jump zone!!!
595 */
596DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
597{
598 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
599 AssertRCReturn(rc, rc);
600 return VINF_SUCCESS;
601}
602
603
604/**
605 * Reads the VM-entry exception error code field from the VMCS into
606 * the VMX transient structure.
607 *
608 * @returns VBox status code.
609 * @param pVmxTransient Pointer to the VMX transient structure.
610 *
611 * @remarks No-long-jump zone!!!
612 */
613DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
614{
615 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
616 AssertRCReturn(rc, rc);
617 return VINF_SUCCESS;
618}
619#endif /* VBOX_STRICT */
620
621
622/**
623 * Reads the VM-exit interruption-information field from the VMCS into the VMX
624 * transient structure.
625 *
626 * @returns VBox status code.
627 * @param pVmxTransient Pointer to the VMX transient structure.
628 */
629DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
630{
631 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
632 {
633 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
634 AssertRCReturn(rc,rc);
635 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_INFO;
636 }
637 return VINF_SUCCESS;
638}
639
640
641/**
642 * Reads the VM-exit interruption error code from the VMCS into the VMX
643 * transient structure.
644 *
645 * @returns VBox status code.
646 * @param pVmxTransient Pointer to the VMX transient structure.
647 */
648DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
649{
650 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
651 {
652 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
653 AssertRCReturn(rc, rc);
654 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
655 }
656 return VINF_SUCCESS;
657}
658
659
660/**
661 * Reads the VM-exit instruction length field from the VMCS into the VMX
662 * transient structure.
663 *
664 * @returns VBox status code.
665 * @param pVmxTransient Pointer to the VMX transient structure.
666 */
667DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
668{
669 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
670 {
671 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
672 AssertRCReturn(rc, rc);
673 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_LEN;
674 }
675 return VINF_SUCCESS;
676}
677
678
679/**
680 * Reads the VM-exit instruction-information field from the VMCS into
681 * the VMX transient structure.
682 *
683 * @returns VBox status code.
684 * @param pVmxTransient Pointer to the VMX transient structure.
685 */
686DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
687{
688 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
689 {
690 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
691 AssertRCReturn(rc, rc);
692 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_INFO;
693 }
694 return VINF_SUCCESS;
695}
696
697
698/**
699 * Reads the exit code qualification from the VMCS into the VMX transient
700 * structure.
701 *
702 * @returns VBox status code.
703 * @param pVCpu The cross context virtual CPU structure of the
704 * calling EMT. (Required for the VMCS cache case.)
705 * @param pVmxTransient Pointer to the VMX transient structure.
706 */
707DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
708{
709 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
710 {
711 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
712 AssertRCReturn(rc, rc);
713 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION;
714 }
715 return VINF_SUCCESS;
716}
717
718
719/**
720 * Reads the IDT-vectoring information field from the VMCS into the VMX
721 * transient structure.
722 *
723 * @returns VBox status code.
724 * @param pVmxTransient Pointer to the VMX transient structure.
725 *
726 * @remarks No-long-jump zone!!!
727 */
728DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
729{
730 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
731 {
732 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
733 AssertRCReturn(rc, rc);
734 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_INFO;
735 }
736 return VINF_SUCCESS;
737}
738
739
740/**
741 * Reads the IDT-vectoring error code from the VMCS into the VMX
742 * transient structure.
743 *
744 * @returns VBox status code.
745 * @param pVmxTransient Pointer to the VMX transient structure.
746 */
747DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
748{
749 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
750 {
751 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
752 AssertRCReturn(rc, rc);
753 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
754 }
755 return VINF_SUCCESS;
756}
757
758
759/**
760 * Enters VMX root mode operation on the current CPU.
761 *
762 * @returns VBox status code.
763 * @param pVM The cross context VM structure. Can be
764 * NULL, after a resume.
765 * @param HCPhysCpuPage Physical address of the VMXON region.
766 * @param pvCpuPage Pointer to the VMXON region.
767 */
768static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
769{
770 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
771 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
772 Assert(pvCpuPage);
773 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
774
775 if (pVM)
776 {
777 /* Write the VMCS revision dword to the VMXON region. */
778 *(uint32_t *)pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
779 }
780
781 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
782 RTCCUINTREG fEFlags = ASMIntDisableFlags();
783
784 /* Enable the VMX bit in CR4 if necessary. */
785 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
786
787 /* Enter VMX root mode. */
788 int rc = VMXEnable(HCPhysCpuPage);
789 if (RT_FAILURE(rc))
790 {
791 if (!(uOldCr4 & X86_CR4_VMXE))
792 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
793
794 if (pVM)
795 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
796 }
797
798 /* Restore interrupts. */
799 ASMSetFlags(fEFlags);
800 return rc;
801}
802
803
804/**
805 * Exits VMX root mode operation on the current CPU.
806 *
807 * @returns VBox status code.
808 */
809static int hmR0VmxLeaveRootMode(void)
810{
811 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
812
813 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
814 RTCCUINTREG fEFlags = ASMIntDisableFlags();
815
816 /* If we're for some reason not in VMX root mode, then don't leave it. */
817 RTCCUINTREG uHostCR4 = ASMGetCR4();
818
819 int rc;
820 if (uHostCR4 & X86_CR4_VMXE)
821 {
822 /* Exit VMX root mode and clear the VMX bit in CR4. */
823 VMXDisable();
824 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
825 rc = VINF_SUCCESS;
826 }
827 else
828 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
829
830 /* Restore interrupts. */
831 ASMSetFlags(fEFlags);
832 return rc;
833}
834
835
836/**
837 * Allocates and maps one physically contiguous page. The allocated page is
838 * zero'd out. (Used by various VT-x structures).
839 *
840 * @returns IPRT status code.
841 * @param pMemObj Pointer to the ring-0 memory object.
842 * @param ppVirt Where to store the virtual address of the
843 * allocation.
844 * @param pHCPhys Where to store the physical address of the
845 * allocation.
846 */
847static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
848{
849 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
850 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
851 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
852
853 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
854 if (RT_FAILURE(rc))
855 return rc;
856 *ppVirt = RTR0MemObjAddress(*pMemObj);
857 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
858 ASMMemZero32(*ppVirt, PAGE_SIZE);
859 return VINF_SUCCESS;
860}
861
862
863/**
864 * Frees and unmaps an allocated physical page.
865 *
866 * @param pMemObj Pointer to the ring-0 memory object.
867 * @param ppVirt Where to re-initialize the virtual address of
868 * allocation as 0.
869 * @param pHCPhys Where to re-initialize the physical address of the
870 * allocation as 0.
871 */
872static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
873{
874 AssertPtr(pMemObj);
875 AssertPtr(ppVirt);
876 AssertPtr(pHCPhys);
877 if (*pMemObj != NIL_RTR0MEMOBJ)
878 {
879 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
880 AssertRC(rc);
881 *pMemObj = NIL_RTR0MEMOBJ;
882 *ppVirt = 0;
883 *pHCPhys = 0;
884 }
885}
886
887
888/**
889 * Worker function to free VT-x related structures.
890 *
891 * @returns IPRT status code.
892 * @param pVM The cross context VM structure.
893 */
894static void hmR0VmxStructsFree(PVM pVM)
895{
896 for (VMCPUID i = 0; i < pVM->cCpus; i++)
897 {
898 PVMCPU pVCpu = &pVM->aCpus[i];
899 AssertPtr(pVCpu);
900
901 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
902 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
903
904 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
905 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
906
907 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
908 }
909
910 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
911#ifdef VBOX_WITH_CRASHDUMP_MAGIC
912 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
913#endif
914}
915
916
917/**
918 * Worker function to allocate VT-x related VM structures.
919 *
920 * @returns IPRT status code.
921 * @param pVM The cross context VM structure.
922 */
923static int hmR0VmxStructsAlloc(PVM pVM)
924{
925 /*
926 * Initialize members up-front so we can cleanup properly on allocation failure.
927 */
928#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
929 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
930 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
931 pVM->hm.s.vmx.HCPhys##a_Name = 0;
932
933#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
934 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
935 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
936 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
937
938#ifdef VBOX_WITH_CRASHDUMP_MAGIC
939 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
940#endif
941 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
942
943 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
944 for (VMCPUID i = 0; i < pVM->cCpus; i++)
945 {
946 PVMCPU pVCpu = &pVM->aCpus[i];
947 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
948 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
949 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
950 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
951 }
952#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
953#undef VMXLOCAL_INIT_VM_MEMOBJ
954
955 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
956 AssertReturnStmt(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE) <= PAGE_SIZE,
957 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
958 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
959
960 /*
961 * Allocate all the VT-x structures.
962 */
963 int rc = VINF_SUCCESS;
964#ifdef VBOX_WITH_CRASHDUMP_MAGIC
965 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
966 if (RT_FAILURE(rc))
967 goto cleanup;
968 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
969 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
970#endif
971
972 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
973 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
974 {
975 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
976 &pVM->hm.s.vmx.HCPhysApicAccess);
977 if (RT_FAILURE(rc))
978 goto cleanup;
979 }
980
981 /*
982 * Initialize per-VCPU VT-x structures.
983 */
984 for (VMCPUID i = 0; i < pVM->cCpus; i++)
985 {
986 PVMCPU pVCpu = &pVM->aCpus[i];
987 AssertPtr(pVCpu);
988
989 /* Allocate the VM control structure (VMCS). */
990 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
991 if (RT_FAILURE(rc))
992 goto cleanup;
993
994 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
995 if ( PDMHasApic(pVM)
996 && (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
997 {
998 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
999 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1000 if (RT_FAILURE(rc))
1001 goto cleanup;
1002 }
1003
1004 /*
1005 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1006 * transparent accesses of specific MSRs.
1007 *
1008 * If the condition for enabling MSR bitmaps changes here, don't forget to
1009 * update HMAreMsrBitmapsAvailable().
1010 */
1011 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1012 {
1013 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1014 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1015 if (RT_FAILURE(rc))
1016 goto cleanup;
1017 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1018 }
1019
1020 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1021 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1022 if (RT_FAILURE(rc))
1023 goto cleanup;
1024
1025 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1026 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1027 if (RT_FAILURE(rc))
1028 goto cleanup;
1029 }
1030
1031 return VINF_SUCCESS;
1032
1033cleanup:
1034 hmR0VmxStructsFree(pVM);
1035 return rc;
1036}
1037
1038
1039/**
1040 * Does global VT-x initialization (called during module initialization).
1041 *
1042 * @returns VBox status code.
1043 */
1044VMMR0DECL(int) VMXR0GlobalInit(void)
1045{
1046#ifdef HMVMX_USE_FUNCTION_TABLE
1047 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1048# ifdef VBOX_STRICT
1049 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1050 Assert(g_apfnVMExitHandlers[i]);
1051# endif
1052#endif
1053 return VINF_SUCCESS;
1054}
1055
1056
1057/**
1058 * Does global VT-x termination (called during module termination).
1059 */
1060VMMR0DECL(void) VMXR0GlobalTerm()
1061{
1062 /* Nothing to do currently. */
1063}
1064
1065
1066/**
1067 * Sets up and activates VT-x on the current CPU.
1068 *
1069 * @returns VBox status code.
1070 * @param pHostCpu Pointer to the global CPU info struct.
1071 * @param pVM The cross context VM structure. Can be
1072 * NULL after a host resume operation.
1073 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1074 * fEnabledByHost is @c true).
1075 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1076 * @a fEnabledByHost is @c true).
1077 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1078 * enable VT-x on the host.
1079 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1080 */
1081VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1082 void *pvMsrs)
1083{
1084 Assert(pHostCpu);
1085 Assert(pvMsrs);
1086 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1087
1088 /* Enable VT-x if it's not already enabled by the host. */
1089 if (!fEnabledByHost)
1090 {
1091 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1092 if (RT_FAILURE(rc))
1093 return rc;
1094 }
1095
1096 /*
1097 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
1098 * using EPTPs) so we don't retain any stale guest-physical mappings which won't get
1099 * invalidated when flushing by VPID.
1100 */
1101 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1102 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1103 {
1104 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
1105 pHostCpu->fFlushAsidBeforeUse = false;
1106 }
1107 else
1108 pHostCpu->fFlushAsidBeforeUse = true;
1109
1110 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1111 ++pHostCpu->cTlbFlushes;
1112
1113 return VINF_SUCCESS;
1114}
1115
1116
1117/**
1118 * Deactivates VT-x on the current CPU.
1119 *
1120 * @returns VBox status code.
1121 * @param pHostCpu Pointer to the global CPU info struct.
1122 * @param pvCpuPage Pointer to the VMXON region.
1123 * @param HCPhysCpuPage Physical address of the VMXON region.
1124 *
1125 * @remarks This function should never be called when SUPR0EnableVTx() or
1126 * similar was used to enable VT-x on the host.
1127 */
1128VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1129{
1130 RT_NOREF3(pHostCpu, pvCpuPage, HCPhysCpuPage);
1131
1132 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1133 return hmR0VmxLeaveRootMode();
1134}
1135
1136
1137/**
1138 * Sets the permission bits for the specified MSR in the MSR bitmap.
1139 *
1140 * @param pVCpu The cross context virtual CPU structure.
1141 * @param uMsr The MSR value.
1142 * @param enmRead Whether reading this MSR causes a VM-exit.
1143 * @param enmWrite Whether writing this MSR causes a VM-exit.
1144 */
1145static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1146{
1147 int32_t iBit;
1148 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1149
1150 /*
1151 * Layout:
1152 * 0x000 - 0x3ff - Low MSR read bits
1153 * 0x400 - 0x7ff - High MSR read bits
1154 * 0x800 - 0xbff - Low MSR write bits
1155 * 0xc00 - 0xfff - High MSR write bits
1156 */
1157 if (uMsr <= 0x00001fff)
1158 iBit = uMsr;
1159 else if (uMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1160 {
1161 iBit = uMsr - UINT32_C(0xc0000000);
1162 pbMsrBitmap += 0x400;
1163 }
1164 else
1165 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1166
1167 Assert(iBit <= 0x1fff);
1168 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1169 ASMBitSet(pbMsrBitmap, iBit);
1170 else
1171 ASMBitClear(pbMsrBitmap, iBit);
1172
1173 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1174 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1175 else
1176 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1177}
1178
1179
1180#ifdef VBOX_STRICT
1181/**
1182 * Gets the permission bits for the specified MSR in the MSR bitmap.
1183 *
1184 * @returns VBox status code.
1185 * @retval VINF_SUCCESS if the specified MSR is found.
1186 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1187 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1188 *
1189 * @param pVCpu The cross context virtual CPU structure.
1190 * @param uMsr The MSR.
1191 * @param penmRead Where to store the read permissions.
1192 * @param penmWrite Where to store the write permissions.
1193 */
1194static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1195{
1196 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1197 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1198 int32_t iBit;
1199 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1200
1201 /* See hmR0VmxSetMsrPermission() for the layout. */
1202 if (uMsr <= 0x00001fff)
1203 iBit = uMsr;
1204 else if ( uMsr >= 0xc0000000
1205 && uMsr <= 0xc0001fff)
1206 {
1207 iBit = (uMsr - 0xc0000000);
1208 pbMsrBitmap += 0x400;
1209 }
1210 else
1211 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1212
1213 Assert(iBit <= 0x1fff);
1214 if (ASMBitTest(pbMsrBitmap, iBit))
1215 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1216 else
1217 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1218
1219 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1220 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1221 else
1222 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1223 return VINF_SUCCESS;
1224}
1225#endif /* VBOX_STRICT */
1226
1227
1228/**
1229 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1230 * area.
1231 *
1232 * @returns VBox status code.
1233 * @param pVCpu The cross context virtual CPU structure.
1234 * @param cMsrs The number of MSRs.
1235 */
1236static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1237{
1238 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1239 uint64_t const uVmxMiscMsr = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc;
1240 uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(uVmxMiscMsr);
1241 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1242 {
1243 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1244 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1245 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1246 }
1247
1248 /* Update number of guest MSRs to load/store across the world-switch. */
1249 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1250 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1251
1252 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1253 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1254 AssertRCReturn(rc, rc);
1255
1256 /* Update the VCPU's copy of the MSR count. */
1257 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1258
1259 return VINF_SUCCESS;
1260}
1261
1262
1263/**
1264 * Adds a new (or updates the value of an existing) guest/host MSR
1265 * pair to be swapped during the world-switch as part of the
1266 * auto-load/store MSR area in the VMCS.
1267 *
1268 * @returns VBox status code.
1269 * @param pVCpu The cross context virtual CPU structure.
1270 * @param uMsr The MSR.
1271 * @param uGuestMsrValue Value of the guest MSR.
1272 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1273 * necessary.
1274 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1275 * its value was updated. Optional, can be NULL.
1276 */
1277static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1278 bool *pfAddedAndUpdated)
1279{
1280 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1281 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1282 uint32_t i;
1283 for (i = 0; i < cMsrs; i++)
1284 {
1285 if (pGuestMsr->u32Msr == uMsr)
1286 break;
1287 pGuestMsr++;
1288 }
1289
1290 bool fAdded = false;
1291 if (i == cMsrs)
1292 {
1293 ++cMsrs;
1294 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1295 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1296
1297 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1298 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1299 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1300
1301 fAdded = true;
1302 }
1303
1304 /* Update the MSR values in the auto-load/store MSR area. */
1305 pGuestMsr->u32Msr = uMsr;
1306 pGuestMsr->u64Value = uGuestMsrValue;
1307
1308 /* Create/update the MSR slot in the host MSR area. */
1309 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1310 pHostMsr += i;
1311 pHostMsr->u32Msr = uMsr;
1312
1313 /*
1314 * Update the host MSR only when requested by the caller AND when we're
1315 * adding it to the auto-load/store area. Otherwise, it would have been
1316 * updated by hmR0VmxExportHostMsrs(). We do this for performance reasons.
1317 */
1318 bool fUpdatedMsrValue = false;
1319 if ( fAdded
1320 && fUpdateHostMsr)
1321 {
1322 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1323 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1324 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1325 fUpdatedMsrValue = true;
1326 }
1327
1328 if (pfAddedAndUpdated)
1329 *pfAddedAndUpdated = fUpdatedMsrValue;
1330 return VINF_SUCCESS;
1331}
1332
1333
1334/**
1335 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1336 * auto-load/store MSR area in the VMCS.
1337 *
1338 * @returns VBox status code.
1339 * @param pVCpu The cross context virtual CPU structure.
1340 * @param uMsr The MSR.
1341 */
1342static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1343{
1344 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1345 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1346 for (uint32_t i = 0; i < cMsrs; i++)
1347 {
1348 /* Find the MSR. */
1349 if (pGuestMsr->u32Msr == uMsr)
1350 {
1351 /* If it's the last MSR, simply reduce the count. */
1352 if (i == cMsrs - 1)
1353 {
1354 --cMsrs;
1355 break;
1356 }
1357
1358 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1359 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1360 pLastGuestMsr += cMsrs - 1;
1361 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1362 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1363
1364 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1365 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1366 pLastHostMsr += cMsrs - 1;
1367 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1368 pHostMsr->u64Value = pLastHostMsr->u64Value;
1369 --cMsrs;
1370 break;
1371 }
1372 pGuestMsr++;
1373 }
1374
1375 /* Update the VMCS if the count changed (meaning the MSR was found). */
1376 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1377 {
1378 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1379 AssertRCReturn(rc, rc);
1380
1381 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1382 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1383 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1384
1385 Log4Func(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1386 return VINF_SUCCESS;
1387 }
1388
1389 return VERR_NOT_FOUND;
1390}
1391
1392
1393/**
1394 * Checks if the specified guest MSR is part of the auto-load/store area in
1395 * the VMCS.
1396 *
1397 * @returns true if found, false otherwise.
1398 * @param pVCpu The cross context virtual CPU structure.
1399 * @param uMsr The MSR to find.
1400 */
1401static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1402{
1403 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1404 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1405
1406 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1407 {
1408 if (pGuestMsr->u32Msr == uMsr)
1409 return true;
1410 }
1411 return false;
1412}
1413
1414
1415/**
1416 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1417 *
1418 * @param pVCpu The cross context virtual CPU structure.
1419 *
1420 * @remarks No-long-jump zone!!!
1421 */
1422static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1423{
1424 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1425 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1426 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1427 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1428
1429 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1430 {
1431 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1432
1433 /*
1434 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1435 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1436 */
1437 if (pHostMsr->u32Msr == MSR_K6_EFER)
1438 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1439 else
1440 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1441 }
1442
1443 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1444}
1445
1446
1447/**
1448 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1449 * perform lazy restoration of the host MSRs while leaving VT-x.
1450 *
1451 * @param pVCpu The cross context virtual CPU structure.
1452 *
1453 * @remarks No-long-jump zone!!!
1454 */
1455static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1456{
1457 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1458
1459 /*
1460 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1461 */
1462 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1463 {
1464 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1465#if HC_ARCH_BITS == 64
1466 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1467 {
1468 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1469 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1470 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1471 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1472 }
1473#endif
1474 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1475 }
1476}
1477
1478
1479/**
1480 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1481 * lazily while leaving VT-x.
1482 *
1483 * @returns true if it does, false otherwise.
1484 * @param pVCpu The cross context virtual CPU structure.
1485 * @param uMsr The MSR to check.
1486 */
1487static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1488{
1489 NOREF(pVCpu);
1490#if HC_ARCH_BITS == 64
1491 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1492 {
1493 switch (uMsr)
1494 {
1495 case MSR_K8_LSTAR:
1496 case MSR_K6_STAR:
1497 case MSR_K8_SF_MASK:
1498 case MSR_K8_KERNEL_GS_BASE:
1499 return true;
1500 }
1501 }
1502#else
1503 RT_NOREF(pVCpu, uMsr);
1504#endif
1505 return false;
1506}
1507
1508
1509/**
1510 * Loads a set of guests MSRs to allow read/passthru to the guest.
1511 *
1512 * The name of this function is slightly confusing. This function does NOT
1513 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1514 * common prefix for functions dealing with "lazy restoration" of the shared
1515 * MSRs.
1516 *
1517 * @param pVCpu The cross context virtual CPU structure.
1518 *
1519 * @remarks No-long-jump zone!!!
1520 */
1521static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1522{
1523 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1524 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1525
1526 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1527#if HC_ARCH_BITS == 64
1528 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1529 {
1530 /*
1531 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1532 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1533 * we can skip a few MSR writes.
1534 *
1535 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1536 * guest MSR values in the guest-CPU context might be different to what's currently
1537 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1538 * CPU, see @bugref{8728}.
1539 */
1540 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1541 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1542 && pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1543 && pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1544 && pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1545 && pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1546 {
1547#ifdef VBOX_STRICT
1548 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1549 Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1550 Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
1551 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
1552#endif
1553 }
1554 else
1555 {
1556 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1557 ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
1558 ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
1559 ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
1560 }
1561 }
1562#endif
1563 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1564}
1565
1566
1567/**
1568 * Performs lazy restoration of the set of host MSRs if they were previously
1569 * loaded with guest MSR values.
1570 *
1571 * @param pVCpu The cross context virtual CPU structure.
1572 *
1573 * @remarks No-long-jump zone!!!
1574 * @remarks The guest MSRs should have been saved back into the guest-CPU
1575 * context by hmR0VmxImportGuestState()!!!
1576 */
1577static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1578{
1579 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1580 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1581
1582 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1583 {
1584 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1585#if HC_ARCH_BITS == 64
1586 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1587 {
1588 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1589 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1590 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1591 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1592 }
1593#endif
1594 }
1595 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1596}
1597
1598
1599/**
1600 * Verifies that our cached values of the VMCS fields are all consistent with
1601 * what's actually present in the VMCS.
1602 *
1603 * @returns VBox status code.
1604 * @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
1605 * @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
1606 * VMCS content. HMCPU error-field is
1607 * updated, see VMX_VCI_XXX.
1608 * @param pVCpu The cross context virtual CPU structure.
1609 */
1610static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1611{
1612 uint32_t u32Val;
1613 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1614 AssertRCReturn(rc, rc);
1615 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32EntryCtls == u32Val,
1616 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1617 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
1618 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1619
1620 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1621 AssertRCReturn(rc, rc);
1622 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ExitCtls == u32Val,
1623 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1624 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
1625 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1626
1627 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1628 AssertRCReturn(rc, rc);
1629 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32PinCtls == u32Val,
1630 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1631 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
1632 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1633
1634 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1635 AssertRCReturn(rc, rc);
1636 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls == u32Val,
1637 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1638 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
1639 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1640
1641 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
1642 {
1643 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1644 AssertRCReturn(rc, rc);
1645 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1646 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1647 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
1648 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1649 }
1650
1651 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
1652 AssertRCReturn(rc, rc);
1653 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32XcptBitmap == u32Val,
1654 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap, u32Val),
1655 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
1656 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1657
1658 uint64_t u64Val;
1659 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
1660 AssertRCReturn(rc, rc);
1661 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u64TscOffset == u64Val,
1662 ("Cache=%#RX64 VMCS=%#RX64\n", pVCpu->hm.s.vmx.u64TscOffset, u64Val),
1663 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
1664 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1665
1666 return VINF_SUCCESS;
1667}
1668
1669
1670#ifdef VBOX_STRICT
1671/**
1672 * Verifies that our cached host EFER value has not changed
1673 * since we cached it.
1674 *
1675 * @param pVCpu The cross context virtual CPU structure.
1676 */
1677static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1678{
1679 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1680
1681 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
1682 {
1683 uint64_t u64Val;
1684 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1685 AssertRC(rc);
1686
1687 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1688 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1689 }
1690}
1691
1692
1693/**
1694 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1695 * VMCS are correct.
1696 *
1697 * @param pVCpu The cross context virtual CPU structure.
1698 */
1699static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1700{
1701 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1702
1703 /* Verify MSR counts in the VMCS are what we think it should be. */
1704 uint32_t cMsrs;
1705 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1706 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1707
1708 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1709 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1710
1711 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1712 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1713
1714 PCVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1715 PCVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1716 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1717 {
1718 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1719 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1720 pGuestMsr->u32Msr, cMsrs));
1721
1722 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1723 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1724 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1725
1726 /* Verify that the permissions are as expected in the MSR bitmap. */
1727 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1728 {
1729 VMXMSREXITREAD enmRead;
1730 VMXMSREXITWRITE enmWrite;
1731 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1732 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1733 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1734 {
1735 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1736 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1737 }
1738 else
1739 {
1740 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1741 pGuestMsr->u32Msr, cMsrs));
1742 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1743 pGuestMsr->u32Msr, cMsrs));
1744 }
1745 }
1746 }
1747}
1748#endif /* VBOX_STRICT */
1749
1750
1751/**
1752 * Flushes the TLB using EPT.
1753 *
1754 * @returns VBox status code.
1755 * @param pVCpu The cross context virtual CPU structure of the calling
1756 * EMT. Can be NULL depending on @a enmTlbFlush.
1757 * @param enmTlbFlush Type of flush.
1758 *
1759 * @remarks Caller is responsible for making sure this function is called only
1760 * when NestedPaging is supported and providing @a enmTlbFlush that is
1761 * supported by the CPU.
1762 * @remarks Can be called with interrupts disabled.
1763 */
1764static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush)
1765{
1766 uint64_t au64Descriptor[2];
1767 if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
1768 au64Descriptor[0] = 0;
1769 else
1770 {
1771 Assert(pVCpu);
1772 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1773 }
1774 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1775
1776 int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
1777 AssertMsg(rc == VINF_SUCCESS,
1778 ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0, rc));
1779
1780 if ( RT_SUCCESS(rc)
1781 && pVCpu)
1782 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1783}
1784
1785
1786/**
1787 * Flushes the TLB using VPID.
1788 *
1789 * @returns VBox status code.
1790 * @param pVCpu The cross context virtual CPU structure of the calling
1791 * EMT. Can be NULL depending on @a enmTlbFlush.
1792 * @param enmTlbFlush Type of flush.
1793 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1794 * on @a enmTlbFlush).
1795 *
1796 * @remarks Can be called with interrupts disabled.
1797 */
1798static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
1799{
1800 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
1801
1802 uint64_t au64Descriptor[2];
1803 if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
1804 {
1805 au64Descriptor[0] = 0;
1806 au64Descriptor[1] = 0;
1807 }
1808 else
1809 {
1810 AssertPtr(pVCpu);
1811 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1812 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1813 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1814 au64Descriptor[1] = GCPtr;
1815 }
1816
1817 int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
1818 AssertMsg(rc == VINF_SUCCESS,
1819 ("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1820
1821 if ( RT_SUCCESS(rc)
1822 && pVCpu)
1823 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1824 NOREF(rc);
1825}
1826
1827
1828/**
1829 * Invalidates a guest page by guest virtual address. Only relevant for
1830 * EPT/VPID, otherwise there is nothing really to invalidate.
1831 *
1832 * @returns VBox status code.
1833 * @param pVCpu The cross context virtual CPU structure.
1834 * @param GCVirt Guest virtual address of the page to invalidate.
1835 */
1836VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
1837{
1838 AssertPtr(pVCpu);
1839 LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
1840
1841 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1842 if (!fFlushPending)
1843 {
1844 /*
1845 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1846 * the EPT case. See @bugref{6043} and @bugref{6177}.
1847 *
1848 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1849 * as this function maybe called in a loop with individual addresses.
1850 */
1851 PVM pVM = pVCpu->CTX_SUFF(pVM);
1852 if (pVM->hm.s.vmx.fVpid)
1853 {
1854 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1855
1856#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1857 /*
1858 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1859 * where executing INVVPID outside 64-bit mode does not flush translations of
1860 * 64-bit linear addresses, see @bugref{6208#c72}.
1861 */
1862 if (RT_HI_U32(GCVirt))
1863 fVpidFlush = false;
1864#endif
1865
1866 if (fVpidFlush)
1867 {
1868 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
1869 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1870 }
1871 else
1872 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1873 }
1874 else if (pVM->hm.s.fNestedPaging)
1875 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1876 }
1877
1878 return VINF_SUCCESS;
1879}
1880
1881
1882/**
1883 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1884 * case where neither EPT nor VPID is supported by the CPU.
1885 *
1886 * @param pVCpu The cross context virtual CPU structure.
1887 * @param pCpu Pointer to the global HM struct.
1888 *
1889 * @remarks Called with interrupts disabled.
1890 */
1891static void hmR0VmxFlushTaggedTlbNone(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1892{
1893 AssertPtr(pVCpu);
1894 AssertPtr(pCpu);
1895
1896 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1897
1898 Assert(pCpu->idCpu != NIL_RTCPUID);
1899 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1900 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1901 pVCpu->hm.s.fForceTLBFlush = false;
1902 return;
1903}
1904
1905
1906/**
1907 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1908 *
1909 * @param pVCpu The cross context virtual CPU structure.
1910 * @param pCpu Pointer to the global HM CPU struct.
1911 *
1912 * @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
1913 * nomenclature. The reason is, to avoid confusion in compare statements
1914 * since the host-CPU copies are named "ASID".
1915 *
1916 * @remarks Called with interrupts disabled.
1917 */
1918static void hmR0VmxFlushTaggedTlbBoth(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1919{
1920#ifdef VBOX_WITH_STATISTICS
1921 bool fTlbFlushed = false;
1922# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1923# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1924 if (!fTlbFlushed) \
1925 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1926 } while (0)
1927#else
1928# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1929# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1930#endif
1931
1932 AssertPtr(pCpu);
1933 AssertPtr(pVCpu);
1934 Assert(pCpu->idCpu != NIL_RTCPUID);
1935
1936 PVM pVM = pVCpu->CTX_SUFF(pVM);
1937 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1938 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1939 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1940
1941 /*
1942 * Force a TLB flush for the first world-switch if the current CPU differs from the one we
1943 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1944 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1945 * cannot reuse the current ASID anymore.
1946 */
1947 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1948 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1949 {
1950 ++pCpu->uCurrentAsid;
1951 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1952 {
1953 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1954 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1955 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1956 }
1957
1958 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1959 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1960 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1961
1962 /*
1963 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1964 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1965 */
1966 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
1967 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1968 HMVMX_SET_TAGGED_TLB_FLUSHED();
1969 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1970 }
1971 else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
1972 {
1973 /*
1974 * Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
1975 * creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
1976 * tables when EPT is in use. Flushing-by-VPID will only flush linear (only
1977 * VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
1978 * mappings, see @bugref{6568}.
1979 *
1980 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
1981 */
1982 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
1983 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1984 HMVMX_SET_TAGGED_TLB_FLUSHED();
1985 }
1986
1987 pVCpu->hm.s.fForceTLBFlush = false;
1988 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1989
1990 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1991 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1992 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1993 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1994 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1995 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
1996 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
1997 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1998 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1999
2000 /* Update VMCS with the VPID. */
2001 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2002 AssertRC(rc);
2003
2004#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2005}
2006
2007
2008/**
2009 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2010 *
2011 * @returns VBox status code.
2012 * @param pVCpu The cross context virtual CPU structure.
2013 * @param pCpu Pointer to the global HM CPU struct.
2014 *
2015 * @remarks Called with interrupts disabled.
2016 */
2017static void hmR0VmxFlushTaggedTlbEpt(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2018{
2019 AssertPtr(pVCpu);
2020 AssertPtr(pCpu);
2021 Assert(pCpu->idCpu != NIL_RTCPUID);
2022 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2023 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2024
2025 /*
2026 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2027 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2028 */
2029 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2030 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2031 {
2032 pVCpu->hm.s.fForceTLBFlush = true;
2033 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2034 }
2035
2036 /* Check for explicit TLB flushes. */
2037 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2038 {
2039 pVCpu->hm.s.fForceTLBFlush = true;
2040 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2041 }
2042
2043 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2044 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2045
2046 if (pVCpu->hm.s.fForceTLBFlush)
2047 {
2048 hmR0VmxFlushEpt(pVCpu, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2049 pVCpu->hm.s.fForceTLBFlush = false;
2050 }
2051}
2052
2053
2054/**
2055 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2056 *
2057 * @returns VBox status code.
2058 * @param pVCpu The cross context virtual CPU structure.
2059 * @param pCpu Pointer to the global HM CPU struct.
2060 *
2061 * @remarks Called with interrupts disabled.
2062 */
2063static void hmR0VmxFlushTaggedTlbVpid(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2064{
2065 AssertPtr(pVCpu);
2066 AssertPtr(pCpu);
2067 Assert(pCpu->idCpu != NIL_RTCPUID);
2068 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2069 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2070
2071 /*
2072 * Force a TLB flush for the first world switch if the current CPU differs from the one we
2073 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2074 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2075 * cannot reuse the current ASID anymore.
2076 */
2077 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2078 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2079 {
2080 pVCpu->hm.s.fForceTLBFlush = true;
2081 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2082 }
2083
2084 /* Check for explicit TLB flushes. */
2085 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2086 {
2087 /*
2088 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2089 * hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2090 * fExplicitFlush = true here and change the pCpu->fFlushAsidBeforeUse check below to
2091 * include fExplicitFlush's too) - an obscure corner case.
2092 */
2093 pVCpu->hm.s.fForceTLBFlush = true;
2094 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2095 }
2096
2097 PVM pVM = pVCpu->CTX_SUFF(pVM);
2098 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2099 if (pVCpu->hm.s.fForceTLBFlush)
2100 {
2101 ++pCpu->uCurrentAsid;
2102 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2103 {
2104 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2105 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2106 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2107 }
2108
2109 pVCpu->hm.s.fForceTLBFlush = false;
2110 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2111 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2112 if (pCpu->fFlushAsidBeforeUse)
2113 {
2114 if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2115 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2116 else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2117 {
2118 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2119 pCpu->fFlushAsidBeforeUse = false;
2120 }
2121 else
2122 {
2123 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2124 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2125 }
2126 }
2127 }
2128
2129 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2130 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2131 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2132 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2133 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2134 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2135 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2136
2137 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2138 AssertRC(rc);
2139}
2140
2141
2142/**
2143 * Flushes the guest TLB entry based on CPU capabilities.
2144 *
2145 * @param pVCpu The cross context virtual CPU structure.
2146 * @param pCpu Pointer to the global HM CPU struct.
2147 */
2148DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2149{
2150#ifdef HMVMX_ALWAYS_FLUSH_TLB
2151 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2152#endif
2153 PVM pVM = pVCpu->CTX_SUFF(pVM);
2154 switch (pVM->hm.s.vmx.enmTlbFlushType)
2155 {
2156 case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVCpu, pCpu); break;
2157 case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pVCpu, pCpu); break;
2158 case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pVCpu, pCpu); break;
2159 case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pVCpu, pCpu); break;
2160 default:
2161 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2162 break;
2163 }
2164 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2165}
2166
2167
2168/**
2169 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2170 * TLB entries from the host TLB before VM-entry.
2171 *
2172 * @returns VBox status code.
2173 * @param pVM The cross context VM structure.
2174 */
2175static int hmR0VmxSetupTaggedTlb(PVM pVM)
2176{
2177 /*
2178 * Determine optimal flush type for Nested Paging.
2179 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2180 * guest execution (see hmR3InitFinalizeR0()).
2181 */
2182 if (pVM->hm.s.fNestedPaging)
2183 {
2184 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2185 {
2186 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2187 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2188 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2189 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2190 else
2191 {
2192 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2193 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2194 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2195 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2196 }
2197
2198 /* Make sure the write-back cacheable memory type for EPT is supported. */
2199 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2200 {
2201 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2202 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2203 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2204 }
2205
2206 /* EPT requires a page-walk length of 4. */
2207 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2208 {
2209 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2210 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2211 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2212 }
2213 }
2214 else
2215 {
2216 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2217 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2218 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2219 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2220 }
2221 }
2222
2223 /*
2224 * Determine optimal flush type for VPID.
2225 */
2226 if (pVM->hm.s.vmx.fVpid)
2227 {
2228 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2229 {
2230 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2231 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2232 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2233 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2234 else
2235 {
2236 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2237 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2238 LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2239 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2240 LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2241 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2242 pVM->hm.s.vmx.fVpid = false;
2243 }
2244 }
2245 else
2246 {
2247 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2248 Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2249 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2250 pVM->hm.s.vmx.fVpid = false;
2251 }
2252 }
2253
2254 /*
2255 * Setup the handler for flushing tagged-TLBs.
2256 */
2257 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2258 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2259 else if (pVM->hm.s.fNestedPaging)
2260 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2261 else if (pVM->hm.s.vmx.fVpid)
2262 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2263 else
2264 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2265 return VINF_SUCCESS;
2266}
2267
2268
2269/**
2270 * Sets up pin-based VM-execution controls in the VMCS.
2271 *
2272 * @returns VBox status code.
2273 * @param pVCpu The cross context virtual CPU structure.
2274 *
2275 * @remarks We don't really care about optimizing vmwrites here as it's done only
2276 * once per VM and hence we don't care about VMCS-field cache comparisons.
2277 */
2278static int hmR0VmxSetupPinCtls(PVMCPU pVCpu)
2279{
2280 PVM pVM = pVCpu->CTX_SUFF(pVM);
2281 uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.disallowed0; /* Bits set here must always be set. */
2282 uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2283
2284 fVal |= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2285 | VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2286
2287 if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2288 fVal |= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2289
2290 /* Enable the VMX preemption timer. */
2291 if (pVM->hm.s.vmx.fUsePreemptTimer)
2292 {
2293 Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2294 fVal |= VMX_PIN_CTLS_PREEMPT_TIMER;
2295 }
2296
2297#if 0
2298 /* Enable posted-interrupt processing. */
2299 if (pVM->hm.s.fPostedIntrs)
2300 {
2301 Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2302 Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2303 fVal |= VMX_PIN_CTL_POSTED_INT;
2304 }
2305#endif
2306
2307 if ((fVal & fZap) != fVal)
2308 {
2309 LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2310 pVM->hm.s.vmx.Msrs.PinCtls.n.disallowed0, fVal, fZap));
2311 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2312 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2313 }
2314
2315 /* Commit it to the VMCS and update our cache. */
2316 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2317 AssertRCReturn(rc, rc);
2318 pVCpu->hm.s.vmx.u32PinCtls = fVal;
2319
2320 return VINF_SUCCESS;
2321}
2322
2323
2324/**
2325 * Sets up secondary processor-based VM-execution controls in the VMCS.
2326 *
2327 * @returns VBox status code.
2328 * @param pVCpu The cross context virtual CPU structure.
2329 *
2330 * @remarks We don't really care about optimizing vmwrites here as it's done only
2331 * once per VM and hence we don't care about VMCS-field cache comparisons.
2332 */
2333static int hmR0VmxSetupProcCtls2(PVMCPU pVCpu)
2334{
2335 PVM pVM = pVCpu->CTX_SUFF(pVM);
2336 uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2337 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2338
2339 /* WBINVD causes a VM-exit. */
2340 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2341 fVal |= VMX_PROC_CTLS2_WBINVD_EXIT;
2342
2343 /* Enable EPT (aka nested-paging). */
2344 if (pVM->hm.s.fNestedPaging)
2345 fVal |= VMX_PROC_CTLS2_EPT;
2346
2347 /*
2348 * Enable the INVPCID instruction if supported by the hardware and we expose
2349 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2350 */
2351 if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID)
2352 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2353 fVal |= VMX_PROC_CTLS2_INVPCID;
2354
2355 /* Enable VPID. */
2356 if (pVM->hm.s.vmx.fVpid)
2357 fVal |= VMX_PROC_CTLS2_VPID;
2358
2359 /* Enable Unrestricted guest execution. */
2360 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2361 fVal |= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
2362
2363#if 0
2364 if (pVM->hm.s.fVirtApicRegs)
2365 {
2366 /* Enable APIC-register virtualization. */
2367 Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
2368 fVal |= VMX_PROC_CTLS2_APIC_REG_VIRT;
2369
2370 /* Enable virtual-interrupt delivery. */
2371 Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
2372 fVal |= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
2373 }
2374#endif
2375
2376 /* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is where the TPR shadow resides. */
2377 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2378 * done dynamically. */
2379 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
2380 {
2381 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2382 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2383 fVal |= VMX_PROC_CTLS2_VIRT_APIC_ACCESS; /* Virtualize APIC accesses. */
2384 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2385 AssertRCReturn(rc, rc);
2386 }
2387
2388 /* Enable RDTSCP. */
2389 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP)
2390 fVal |= VMX_PROC_CTLS2_RDTSCP;
2391
2392 /* Enable Pause-Loop exiting. */
2393 if ( pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT
2394 && pVM->hm.s.vmx.cPleGapTicks
2395 && pVM->hm.s.vmx.cPleWindowTicks)
2396 {
2397 fVal |= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
2398
2399 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2400 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2401 AssertRCReturn(rc, rc);
2402 }
2403
2404 if ((fVal & fZap) != fVal)
2405 {
2406 LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2407 pVM->hm.s.vmx.Msrs.ProcCtls2.n.disallowed0, fVal, fZap));
2408 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2409 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2410 }
2411
2412 /* Commit it to the VMCS and update our cache. */
2413 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2414 AssertRCReturn(rc, rc);
2415 pVCpu->hm.s.vmx.u32ProcCtls2 = fVal;
2416
2417 return VINF_SUCCESS;
2418}
2419
2420
2421/**
2422 * Sets up processor-based VM-execution controls in the VMCS.
2423 *
2424 * @returns VBox status code.
2425 * @param pVCpu The cross context virtual CPU structure.
2426 *
2427 * @remarks We don't really care about optimizing vmwrites here as it's done only
2428 * once per VM and hence we don't care about VMCS-field cache comparisons.
2429 */
2430static int hmR0VmxSetupProcCtls(PVMCPU pVCpu)
2431{
2432 PVM pVM = pVCpu->CTX_SUFF(pVM);
2433 uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2434 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2435
2436 fVal |= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
2437 | VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2438 | VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2439 | VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2440 | VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2441 | VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2442 | VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2443
2444 /* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2445 if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
2446 || (pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
2447 {
2448 LogRelFunc(("Unsupported VMX_PROC_CTLS_MOV_DR_EXIT combo!"));
2449 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2450 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2451 }
2452
2453 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2454 if (!pVM->hm.s.fNestedPaging)
2455 {
2456 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2457 fVal |= VMX_PROC_CTLS_INVLPG_EXIT
2458 | VMX_PROC_CTLS_CR3_LOAD_EXIT
2459 | VMX_PROC_CTLS_CR3_STORE_EXIT;
2460 }
2461
2462 /* Use TPR shadowing if supported by the CPU. */
2463 if ( PDMHasApic(pVM)
2464 && pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
2465 {
2466 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2467 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2468 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2469 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2470 AssertRCReturn(rc, rc);
2471
2472 fVal |= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2473 /* CR8 writes cause a VM-exit based on TPR threshold. */
2474 Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
2475 Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
2476 }
2477 else
2478 {
2479 /*
2480 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2481 * Set this control only for 64-bit guests.
2482 */
2483 if (pVM->hm.s.fAllow64BitGuests)
2484 {
2485 fVal |= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2486 | VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2487 }
2488 }
2489
2490 /* Use MSR-bitmaps if supported by the CPU. */
2491 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2492 {
2493 fVal |= VMX_PROC_CTLS_USE_MSR_BITMAPS;
2494
2495 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2496 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2497 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2498 AssertRCReturn(rc, rc);
2499
2500 /*
2501 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2502 * automatically using dedicated fields in the VMCS.
2503 */
2504 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2505 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2506 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2507 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2508 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2509#if HC_ARCH_BITS == 64
2510 /*
2511 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2512 */
2513 if (pVM->hm.s.fAllow64BitGuests)
2514 {
2515 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2516 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2517 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2518 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2519 }
2520#endif
2521 /*
2522 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2523 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2524 */
2525 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2526 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2527
2528 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2529 }
2530
2531 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2532 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2533 fVal |= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
2534
2535 if ((fVal & fZap) != fVal)
2536 {
2537 LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2538 pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0, fVal, fZap));
2539 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2540 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2541 }
2542
2543 /* Commit it to the VMCS and update our cache. */
2544 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2545 AssertRCReturn(rc, rc);
2546 pVCpu->hm.s.vmx.u32ProcCtls = fVal;
2547
2548 /* Set up secondary processor-based VM-execution controls if the CPU supports it. */
2549 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2550 return hmR0VmxSetupProcCtls2(pVCpu);
2551
2552 /* Sanity check, should not really happen. */
2553 if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2554 {
2555 LogRelFunc(("Unrestricted Guest enabled when secondary processor-based VM-execution controls not available\n"));
2556 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2557 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2558 }
2559
2560 /* Old CPUs without secondary processor-based VM-execution controls would end up here. */
2561 return VINF_SUCCESS;
2562}
2563
2564
2565/**
2566 * Sets up miscellaneous (everything other than Pin & Processor-based
2567 * VM-execution) control fields in the VMCS.
2568 *
2569 * @returns VBox status code.
2570 * @param pVCpu The cross context virtual CPU structure.
2571 */
2572static int hmR0VmxSetupMiscCtls(PVMCPU pVCpu)
2573{
2574 AssertPtr(pVCpu);
2575
2576 int rc = VERR_GENERAL_FAILURE;
2577
2578 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2579#if 0
2580 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxExportGuestCR3AndCR4())*/
2581 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2582 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2583
2584 /*
2585 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2586 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2587 * We thus use the exception bitmap to control it rather than use both.
2588 */
2589 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2590 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2591
2592 /* All IO & IOIO instructions cause VM-exits. */
2593 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2594 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2595
2596 /* Initialize the MSR-bitmap area. */
2597 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2598 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2599 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2600 AssertRCReturn(rc, rc);
2601#endif
2602
2603 /* Setup MSR auto-load/store area. */
2604 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2605 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2606 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2607 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2608 AssertRCReturn(rc, rc);
2609
2610 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2611 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2612 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2613 AssertRCReturn(rc, rc);
2614
2615 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2616 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2617 AssertRCReturn(rc, rc);
2618
2619 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2620#if 0
2621 /* Setup debug controls */
2622 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2623 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2624 AssertRCReturn(rc, rc);
2625#endif
2626
2627 return rc;
2628}
2629
2630
2631/**
2632 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2633 *
2634 * We shall setup those exception intercepts that don't change during the
2635 * lifetime of the VM here. The rest are done dynamically while loading the
2636 * guest state.
2637 *
2638 * @returns VBox status code.
2639 * @param pVCpu The cross context virtual CPU structure.
2640 */
2641static int hmR0VmxInitXcptBitmap(PVMCPU pVCpu)
2642{
2643 AssertPtr(pVCpu);
2644
2645 uint32_t uXcptBitmap;
2646
2647 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2648 uXcptBitmap = RT_BIT_32(X86_XCPT_AC);
2649
2650 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2651 and writes, and because recursive #DBs can cause the CPU hang, we must always
2652 intercept #DB. */
2653 uXcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2654
2655 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2656 if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
2657 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
2658
2659 /* Commit it to the VMCS. */
2660 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
2661 AssertRCReturn(rc, rc);
2662
2663 /* Update our cache of the exception bitmap. */
2664 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
2665 return VINF_SUCCESS;
2666}
2667
2668
2669/**
2670 * Does per-VM VT-x initialization.
2671 *
2672 * @returns VBox status code.
2673 * @param pVM The cross context VM structure.
2674 */
2675VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2676{
2677 LogFlowFunc(("pVM=%p\n", pVM));
2678
2679 int rc = hmR0VmxStructsAlloc(pVM);
2680 if (RT_FAILURE(rc))
2681 {
2682 LogRelFunc(("hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2683 return rc;
2684 }
2685
2686 return VINF_SUCCESS;
2687}
2688
2689
2690/**
2691 * Does per-VM VT-x termination.
2692 *
2693 * @returns VBox status code.
2694 * @param pVM The cross context VM structure.
2695 */
2696VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2697{
2698 LogFlowFunc(("pVM=%p\n", pVM));
2699
2700#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2701 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2702 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2703#endif
2704 hmR0VmxStructsFree(pVM);
2705 return VINF_SUCCESS;
2706}
2707
2708
2709/**
2710 * Sets up the VM for execution under VT-x.
2711 * This function is only called once per-VM during initialization.
2712 *
2713 * @returns VBox status code.
2714 * @param pVM The cross context VM structure.
2715 */
2716VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2717{
2718 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2719 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2720
2721 LogFlowFunc(("pVM=%p\n", pVM));
2722
2723 /*
2724 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be
2725 * allocated. We no longer support the highly unlikely case of UnrestrictedGuest without
2726 * pRealModeTSS, see hmR3InitFinalizeR0Intel().
2727 */
2728 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2729 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2730 || !pVM->hm.s.vmx.pRealModeTSS))
2731 {
2732 LogRelFunc(("Invalid real-on-v86 state.\n"));
2733 return VERR_INTERNAL_ERROR;
2734 }
2735
2736 /* Initialize these always, see hmR3InitFinalizeR0().*/
2737 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
2738 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
2739
2740 /* Setup the tagged-TLB flush handlers. */
2741 int rc = hmR0VmxSetupTaggedTlb(pVM);
2742 if (RT_FAILURE(rc))
2743 {
2744 LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2745 return rc;
2746 }
2747
2748 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2749 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2750#if HC_ARCH_BITS == 64
2751 if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
2752 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2753 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
2754 {
2755 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2756 }
2757#endif
2758
2759 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2760 RTCCUINTREG const uHostCR4 = ASMGetCR4();
2761 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2762 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2763
2764 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2765 {
2766 PVMCPU pVCpu = &pVM->aCpus[i];
2767 AssertPtr(pVCpu);
2768 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2769
2770 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2771 Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2772
2773 /* Set revision dword at the beginning of the VMCS structure. */
2774 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
2775
2776 /* Load this VMCS as the current VMCS. */
2777 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2778 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc\n", rc),
2779 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2780
2781 rc = hmR0VmxSetupPinCtls(pVCpu);
2782 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc\n", rc),
2783 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2784
2785 rc = hmR0VmxSetupProcCtls(pVCpu);
2786 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc\n", rc),
2787 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2788
2789 rc = hmR0VmxSetupMiscCtls(pVCpu);
2790 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc\n", rc),
2791 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2792
2793 rc = hmR0VmxInitXcptBitmap(pVCpu);
2794 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc\n", rc),
2795 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2796
2797#if HC_ARCH_BITS == 32
2798 rc = hmR0VmxInitVmcsReadCache(pVCpu);
2799 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc\n", rc),
2800 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2801#endif
2802
2803 /* Sync the CPU's internal data into our VMCS memory region & set the launch state to "clear". */
2804 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2805 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc\n", rc),
2806 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2807
2808 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2809
2810 hmR0VmxUpdateErrorRecord(pVCpu, rc);
2811 }
2812
2813 return VINF_SUCCESS;
2814}
2815
2816
2817/**
2818 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2819 * the VMCS.
2820 *
2821 * @returns VBox status code.
2822 */
2823static int hmR0VmxExportHostControlRegs(void)
2824{
2825 RTCCUINTREG uReg = ASMGetCR0();
2826 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2827 AssertRCReturn(rc, rc);
2828
2829 uReg = ASMGetCR3();
2830 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2831 AssertRCReturn(rc, rc);
2832
2833 uReg = ASMGetCR4();
2834 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2835 AssertRCReturn(rc, rc);
2836 return rc;
2837}
2838
2839
2840/**
2841 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2842 * the host-state area in the VMCS.
2843 *
2844 * @returns VBox status code.
2845 * @param pVCpu The cross context virtual CPU structure.
2846 */
2847static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
2848{
2849#if HC_ARCH_BITS == 64
2850/**
2851 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2852 * requirements. See hmR0VmxExportHostSegmentRegs().
2853 */
2854# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2855 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2856 { \
2857 bool fValidSelector = true; \
2858 if ((selValue) & X86_SEL_LDT) \
2859 { \
2860 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2861 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2862 } \
2863 if (fValidSelector) \
2864 { \
2865 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2866 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2867 } \
2868 (selValue) = 0; \
2869 }
2870
2871 /*
2872 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2873 * should -not- save the messed up state without restoring the original host-state,
2874 * see @bugref{7240}.
2875 *
2876 * This apparently can happen (most likely the FPU changes), deal with it rather than
2877 * asserting. Was observed booting Solaris 10u10 32-bit guest.
2878 */
2879 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2880 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2881 {
2882 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2883 pVCpu->idCpu));
2884 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2885 }
2886 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2887#else
2888 RT_NOREF(pVCpu);
2889#endif
2890
2891 /*
2892 * Host DS, ES, FS and GS segment registers.
2893 */
2894#if HC_ARCH_BITS == 64
2895 RTSEL uSelDS = ASMGetDS();
2896 RTSEL uSelES = ASMGetES();
2897 RTSEL uSelFS = ASMGetFS();
2898 RTSEL uSelGS = ASMGetGS();
2899#else
2900 RTSEL uSelDS = 0;
2901 RTSEL uSelES = 0;
2902 RTSEL uSelFS = 0;
2903 RTSEL uSelGS = 0;
2904#endif
2905
2906 /*
2907 * Host CS and SS segment registers.
2908 */
2909 RTSEL uSelCS = ASMGetCS();
2910 RTSEL uSelSS = ASMGetSS();
2911
2912 /*
2913 * Host TR segment register.
2914 */
2915 RTSEL uSelTR = ASMGetTR();
2916
2917#if HC_ARCH_BITS == 64
2918 /*
2919 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
2920 * gain VM-entry and restore them before we get preempted.
2921 *
2922 * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2923 */
2924 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2925 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2926 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2927 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2928# undef VMXLOCAL_ADJUST_HOST_SEG
2929#endif
2930
2931 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2932 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2933 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2934 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2935 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2936 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2937 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2938 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2939 Assert(uSelCS);
2940 Assert(uSelTR);
2941
2942 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2943#if 0
2944 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE))
2945 Assert(uSelSS != 0);
2946#endif
2947
2948 /* Write these host selector fields into the host-state area in the VMCS. */
2949 int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2950 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2951#if HC_ARCH_BITS == 64
2952 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2953 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2954 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2955 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2956#else
2957 NOREF(uSelDS);
2958 NOREF(uSelES);
2959 NOREF(uSelFS);
2960 NOREF(uSelGS);
2961#endif
2962 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2963 AssertRCReturn(rc, rc);
2964
2965 /*
2966 * Host GDTR and IDTR.
2967 */
2968 RTGDTR Gdtr;
2969 RTIDTR Idtr;
2970 RT_ZERO(Gdtr);
2971 RT_ZERO(Idtr);
2972 ASMGetGDTR(&Gdtr);
2973 ASMGetIDTR(&Idtr);
2974 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2975 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2976 AssertRCReturn(rc, rc);
2977
2978#if HC_ARCH_BITS == 64
2979 /*
2980 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
2981 * them to the maximum limit (0xffff) on every VM-exit.
2982 */
2983 if (Gdtr.cbGdt != 0xffff)
2984 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2985
2986 /*
2987 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2988 * Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
2989 * as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
2990 * However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
2991 * uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
2992 * alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
2993 * at 0xffff on hosts where we are sure it won't cause trouble.
2994 */
2995# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
2996 if (Idtr.cbIdt < 0x0fff)
2997# else
2998 if (Idtr.cbIdt != 0xffff)
2999# endif
3000 {
3001 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3002 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3003 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3004 }
3005#endif
3006
3007 /*
3008 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3009 * and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3010 * RPL should be too in most cases.
3011 */
3012 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3013 ("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3014
3015 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3016#if HC_ARCH_BITS == 64
3017 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3018
3019 /*
3020 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3021 * all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3022 * restoration if the host has something else. Task switching is not supported in 64-bit
3023 * mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3024 * limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3025 *
3026 * [1] See Intel spec. 3.5 "System Descriptor Types".
3027 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3028 */
3029 PVM pVM = pVCpu->CTX_SUFF(pVM);
3030 Assert(pDesc->System.u4Type == 11);
3031 if ( pDesc->System.u16LimitLow != 0x67
3032 || pDesc->System.u4LimitHigh)
3033 {
3034 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3035 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3036 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3037 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3038 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3039 }
3040
3041 /*
3042 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3043 */
3044 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3045 {
3046 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3047 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3048 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3049 {
3050 /* The GDT is read-only but the writable GDT is available. */
3051 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3052 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3053 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3054 AssertRCReturn(rc, rc);
3055 }
3056 }
3057#else
3058 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3059#endif
3060 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3061 AssertRCReturn(rc, rc);
3062
3063 /*
3064 * Host FS base and GS base.
3065 */
3066#if HC_ARCH_BITS == 64
3067 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3068 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3069 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3070 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3071 AssertRCReturn(rc, rc);
3072
3073 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3074 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3075 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3076 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3077 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3078#endif
3079 return VINF_SUCCESS;
3080}
3081
3082
3083/**
3084 * Exports certain host MSRs in the VM-exit MSR-load area and some in the
3085 * host-state area of the VMCS.
3086 *
3087 * Theses MSRs will be automatically restored on the host after every successful
3088 * VM-exit.
3089 *
3090 * @returns VBox status code.
3091 * @param pVCpu The cross context virtual CPU structure.
3092 *
3093 * @remarks No-long-jump zone!!!
3094 */
3095static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
3096{
3097 AssertPtr(pVCpu);
3098 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3099
3100 /*
3101 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3102 * rather than swapping them on every VM-entry.
3103 */
3104 hmR0VmxLazySaveHostMsrs(pVCpu);
3105
3106 /*
3107 * Host Sysenter MSRs.
3108 */
3109 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3110#if HC_ARCH_BITS == 32
3111 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3112 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3113#else
3114 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3115 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3116#endif
3117 AssertRCReturn(rc, rc);
3118
3119 /*
3120 * Host EFER MSR.
3121 *
3122 * If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3123 * done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3124 */
3125 PVM pVM = pVCpu->CTX_SUFF(pVM);
3126 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3127 {
3128 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3129 AssertRCReturn(rc, rc);
3130 }
3131
3132 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see hmR0VmxExportGuestExitCtls(). */
3133
3134 return VINF_SUCCESS;
3135}
3136
3137
3138/**
3139 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3140 *
3141 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3142 * these two bits are handled by VM-entry, see hmR0VmxExportGuestExitCtls() and
3143 * hmR0VMxExportGuestEntryCtls().
3144 *
3145 * @returns true if we need to load guest EFER, false otherwise.
3146 * @param pVCpu The cross context virtual CPU structure.
3147 *
3148 * @remarks Requires EFER, CR4.
3149 * @remarks No-long-jump zone!!!
3150 */
3151static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu)
3152{
3153#ifdef HMVMX_ALWAYS_SWAP_EFER
3154 return true;
3155#endif
3156
3157 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3158#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3159 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3160 if (CPUMIsGuestInLongModeEx(pCtx))
3161 return false;
3162#endif
3163
3164 PVM pVM = pVCpu->CTX_SUFF(pVM);
3165 uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3166 uint64_t const u64GuestEfer = pCtx->msrEFER;
3167
3168 /*
3169 * For 64-bit guests, if EFER.SCE bit differs, we need to swap EFER to ensure that the
3170 * guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3171 */
3172 if ( CPUMIsGuestInLongModeEx(pCtx)
3173 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3174 {
3175 return true;
3176 }
3177
3178 /*
3179 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3180 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3181 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3182 */
3183 if ( (pCtx->cr4 & X86_CR4_PAE)
3184 && (pCtx->cr0 & X86_CR0_PG)
3185 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3186 {
3187 /* Assert that host is NX capable. */
3188 Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
3189 return true;
3190 }
3191
3192 return false;
3193}
3194
3195
3196/**
3197 * Exports the guest state with appropriate VM-entry controls in the VMCS.
3198 *
3199 * These controls can affect things done on VM-exit; e.g. "load debug controls",
3200 * see Intel spec. 24.8.1 "VM-entry controls".
3201 *
3202 * @returns VBox status code.
3203 * @param pVCpu The cross context virtual CPU structure.
3204 *
3205 * @remarks Requires EFER.
3206 * @remarks No-long-jump zone!!!
3207 */
3208static int hmR0VmxExportGuestEntryCtls(PVMCPU pVCpu)
3209{
3210 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_CTLS)
3211 {
3212 PVM pVM = pVCpu->CTX_SUFF(pVM);
3213 uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
3214 uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3215
3216 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3217 fVal |= VMX_ENTRY_CTLS_LOAD_DEBUG;
3218
3219 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3220 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
3221 {
3222 fVal |= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
3223 Log4Func(("VMX_ENTRY_CTLS_IA32E_MODE_GUEST\n"));
3224 }
3225 else
3226 Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
3227
3228 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3229 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3230 && hmR0VmxShouldSwapEferMsr(pVCpu))
3231 {
3232 fVal |= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
3233 Log4Func(("VMX_ENTRY_CTLS_LOAD_EFER_MSR\n"));
3234 }
3235
3236 /*
3237 * The following should -not- be set (since we're not in SMM mode):
3238 * - VMX_ENTRY_CTLS_ENTRY_TO_SMM
3239 * - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
3240 */
3241
3242 /** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
3243 * VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
3244
3245 if ((fVal & fZap) != fVal)
3246 {
3247 Log4Func(("Invalid VM-entry controls combo! Cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3248 pVM->hm.s.vmx.Msrs.EntryCtls.n.disallowed0, fVal, fZap));
3249 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3250 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3251 }
3252
3253 /* Commit it to the VMCS and update our cache. */
3254 if (pVCpu->hm.s.vmx.u32EntryCtls != fVal)
3255 {
3256 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
3257 AssertRCReturn(rc, rc);
3258 pVCpu->hm.s.vmx.u32EntryCtls = fVal;
3259 }
3260
3261 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_CTLS);
3262 }
3263 return VINF_SUCCESS;
3264}
3265
3266
3267/**
3268 * Exports the guest state with appropriate VM-exit controls in the VMCS.
3269 *
3270 * @returns VBox status code.
3271 * @param pVCpu The cross context virtual CPU structure.
3272 *
3273 * @remarks Requires EFER.
3274 */
3275static int hmR0VmxExportGuestExitCtls(PVMCPU pVCpu)
3276{
3277 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_EXIT_CTLS)
3278 {
3279 PVM pVM = pVCpu->CTX_SUFF(pVM);
3280 uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
3281 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3282
3283 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3284 fVal |= VMX_EXIT_CTLS_SAVE_DEBUG;
3285
3286 /*
3287 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3288 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in
3289 * hmR0VmxExportHostMsrs().
3290 */
3291#if HC_ARCH_BITS == 64
3292 fVal |= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
3293 Log4Func(("VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE\n"));
3294#else
3295 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3296 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3297 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3298 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3299 {
3300 /* The switcher returns to long mode, EFER is managed by the switcher. */
3301 fVal |= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
3302 Log4Func(("VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE\n"));
3303 }
3304 else
3305 Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
3306#endif
3307
3308 /* If the newer VMCS fields for managing EFER exists, use it. */
3309 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3310 && hmR0VmxShouldSwapEferMsr(pVCpu))
3311 {
3312 fVal |= VMX_EXIT_CTLS_SAVE_EFER_MSR
3313 | VMX_EXIT_CTLS_LOAD_EFER_MSR;
3314 Log4Func(("VMX_EXIT_CTLS_SAVE_EFER_MSR and VMX_EXIT_CTLS_LOAD_EFER_MSR\n"));
3315 }
3316
3317 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3318 Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
3319
3320 /** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
3321 * VMX_EXIT_CTLS_SAVE_PAT_MSR,
3322 * VMX_EXIT_CTLS_LOAD_PAT_MSR. */
3323
3324 /* Enable saving of the VMX preemption timer value on VM-exit. */
3325 if ( pVM->hm.s.vmx.fUsePreemptTimer
3326 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_VMX_PREEMPT_TIMER))
3327 fVal |= VMX_EXIT_CTLS_SAVE_VMX_PREEMPT_TIMER;
3328
3329 if ((fVal & fZap) != fVal)
3330 {
3331 LogRelFunc(("Invalid VM-exit controls combo! cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3332 pVM->hm.s.vmx.Msrs.ExitCtls.n.disallowed0, fVal, fZap));
3333 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3334 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3335 }
3336
3337 /* Commit it to the VMCS and update our cache. */
3338 if (pVCpu->hm.s.vmx.u32ExitCtls != fVal)
3339 {
3340 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
3341 AssertRCReturn(rc, rc);
3342 pVCpu->hm.s.vmx.u32ExitCtls = fVal;
3343 }
3344
3345 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_EXIT_CTLS);
3346 }
3347 return VINF_SUCCESS;
3348}
3349
3350
3351/**
3352 * Sets the TPR threshold in the VMCS.
3353 *
3354 * @returns VBox status code.
3355 * @param pVCpu The cross context virtual CPU structure.
3356 * @param u32TprThreshold The TPR threshold (task-priority class only).
3357 */
3358DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3359{
3360 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3361 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3362 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3363}
3364
3365
3366/**
3367 * Exports the guest APIC TPR state into the VMCS.
3368 *
3369 * @returns VBox status code.
3370 * @param pVCpu The cross context virtual CPU structure.
3371 *
3372 * @remarks No-long-jump zone!!!
3373 */
3374static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu)
3375{
3376 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
3377 {
3378 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
3379
3380 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3381 && APICIsEnabled(pVCpu))
3382 {
3383 /*
3384 * Setup TPR shadowing.
3385 */
3386 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
3387 {
3388 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3389
3390 bool fPendingIntr = false;
3391 uint8_t u8Tpr = 0;
3392 uint8_t u8PendingIntr = 0;
3393 int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3394 AssertRCReturn(rc, rc);
3395
3396 /*
3397 * If there are interrupts pending but masked by the TPR, instruct VT-x to
3398 * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
3399 * priority of the pending interrupt so we can deliver the interrupt. If there
3400 * are no interrupts pending, set threshold to 0 to not cause any
3401 * TPR-below-threshold VM-exits.
3402 */
3403 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3404 uint32_t u32TprThreshold = 0;
3405 if (fPendingIntr)
3406 {
3407 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3408 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3409 const uint8_t u8TprPriority = u8Tpr >> 4;
3410 if (u8PendingPriority <= u8TprPriority)
3411 u32TprThreshold = u8PendingPriority;
3412 }
3413
3414 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3415 AssertRCReturn(rc, rc);
3416 }
3417 }
3418 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
3419 }
3420 return VINF_SUCCESS;
3421}
3422
3423
3424/**
3425 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3426 *
3427 * @returns Guest's interruptibility-state.
3428 * @param pVCpu The cross context virtual CPU structure.
3429 *
3430 * @remarks No-long-jump zone!!!
3431 */
3432static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu)
3433{
3434 /*
3435 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3436 */
3437 uint32_t fIntrState = 0;
3438 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3439 {
3440 /* If inhibition is active, RIP & RFLAGS should've been accessed
3441 (i.e. read previously from the VMCS or from ring-3). */
3442 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3443#ifdef VBOX_STRICT
3444 uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
3445 AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
3446#endif
3447 if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3448 {
3449 if (pCtx->eflags.Bits.u1IF)
3450 fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
3451 else
3452 fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
3453 }
3454 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3455 {
3456 /*
3457 * We can clear the inhibit force flag as even if we go back to the recompiler
3458 * without executing guest code in VT-x, the flag's condition to be cleared is
3459 * met and thus the cleared state is correct.
3460 */
3461 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3462 }
3463 }
3464
3465 /*
3466 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3467 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3468 * setting this would block host-NMIs and IRET will not clear the blocking.
3469 *
3470 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3471 */
3472 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3473 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
3474 {
3475 fIntrState |= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
3476 }
3477
3478 return fIntrState;
3479}
3480
3481
3482/**
3483 * Exports the guest's interruptibility-state into the guest-state area in the
3484 * VMCS.
3485 *
3486 * @returns VBox status code.
3487 * @param pVCpu The cross context virtual CPU structure.
3488 * @param fIntrState The interruptibility-state to set.
3489 */
3490static int hmR0VmxExportGuestIntrState(PVMCPU pVCpu, uint32_t fIntrState)
3491{
3492 NOREF(pVCpu);
3493 AssertMsg(!(fIntrState & 0xfffffff0), ("%#x\n", fIntrState)); /* Bits 31:4 MBZ. */
3494 Assert((fIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3495 return VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
3496}
3497
3498
3499/**
3500 * Exports the exception intercepts required for guest execution in the VMCS.
3501 *
3502 * @returns VBox status code.
3503 * @param pVCpu The cross context virtual CPU structure.
3504 *
3505 * @remarks No-long-jump zone!!!
3506 */
3507static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu)
3508{
3509 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
3510 {
3511 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3512
3513 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxExportSharedCR0(). */
3514 if (pVCpu->hm.s.fGIMTrapXcptUD)
3515 uXcptBitmap |= RT_BIT(X86_XCPT_UD);
3516#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3517 else
3518 uXcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3519#endif
3520
3521 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_AC));
3522 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_DB));
3523
3524 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3525 {
3526 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3527 AssertRCReturn(rc, rc);
3528 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3529 }
3530
3531 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3532 Log4Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64\n", uXcptBitmap));
3533 }
3534 return VINF_SUCCESS;
3535}
3536
3537
3538/**
3539 * Exports the guest's RIP into the guest-state area in the VMCS.
3540 *
3541 * @returns VBox status code.
3542 * @param pVCpu The cross context virtual CPU structure.
3543 *
3544 * @remarks No-long-jump zone!!!
3545 */
3546static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
3547{
3548 int rc = VINF_SUCCESS;
3549 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
3550 {
3551 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
3552
3553 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
3554 AssertRCReturn(rc, rc);
3555
3556 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3557 Log4Func(("RIP=%#RX64\n", pVCpu->cpum.GstCtx.rip));
3558 }
3559 return rc;
3560}
3561
3562
3563/**
3564 * Exports the guest's RSP into the guest-state area in the VMCS.
3565 *
3566 * @returns VBox status code.
3567 * @param pVCpu The cross context virtual CPU structure.
3568 *
3569 * @remarks No-long-jump zone!!!
3570 */
3571static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
3572{
3573 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3574 {
3575 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
3576
3577 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
3578 AssertRCReturn(rc, rc);
3579
3580 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3581 }
3582 return VINF_SUCCESS;
3583}
3584
3585
3586/**
3587 * Exports the guest's RFLAGS into the guest-state area in the VMCS.
3588 *
3589 * @returns VBox status code.
3590 * @param pVCpu The cross context virtual CPU structure.
3591 *
3592 * @remarks No-long-jump zone!!!
3593 */
3594static int hmR0VmxExportGuestRflags(PVMCPU pVCpu)
3595{
3596 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3597 {
3598 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
3599
3600 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3601 Let us assert it as such and use 32-bit VMWRITE. */
3602 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
3603 X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
3604 Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3605 Assert(!(fEFlags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3606
3607 /*
3608 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3609 * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3610 * can run the real-mode guest code under Virtual 8086 mode.
3611 */
3612 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3613 {
3614 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3615 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3616 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3617 fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3618 fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3619 }
3620
3621 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3622 AssertRCReturn(rc, rc);
3623
3624 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3625 Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
3626 }
3627 return VINF_SUCCESS;
3628}
3629
3630
3631/**
3632 * Exports the guest CR0 control register into the guest-state area in the VMCS.
3633 *
3634 * The guest FPU state is always pre-loaded hence we don't need to bother about
3635 * sharing FPU related CR0 bits between the guest and host.
3636 *
3637 * @returns VBox status code.
3638 * @param pVCpu The cross context virtual CPU structure.
3639 *
3640 * @remarks No-long-jump zone!!!
3641 */
3642static int hmR0VmxExportGuestCR0(PVMCPU pVCpu)
3643{
3644 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
3645 {
3646 PVM pVM = pVCpu->CTX_SUFF(pVM);
3647 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
3648 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.cr0));
3649
3650 uint32_t const u32ShadowCr0 = pVCpu->cpum.GstCtx.cr0;
3651 uint32_t u32GuestCr0 = pVCpu->cpum.GstCtx.cr0;
3652
3653 /*
3654 * Setup VT-x's view of the guest CR0.
3655 * Minimize VM-exits due to CR3 changes when we have NestedPaging.
3656 */
3657 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
3658 if (pVM->hm.s.fNestedPaging)
3659 {
3660 if (CPUMIsGuestPagingEnabled(pVCpu))
3661 {
3662 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3663 uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
3664 | VMX_PROC_CTLS_CR3_STORE_EXIT);
3665 }
3666 else
3667 {
3668 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3669 uProcCtls |= VMX_PROC_CTLS_CR3_LOAD_EXIT
3670 | VMX_PROC_CTLS_CR3_STORE_EXIT;
3671 }
3672
3673 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3674 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3675 uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
3676 }
3677 else
3678 {
3679 /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3680 u32GuestCr0 |= X86_CR0_WP;
3681 }
3682
3683 /*
3684 * Guest FPU bits.
3685 *
3686 * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3687 * using CR0.TS.
3688 *
3689 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3690 * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3691 */
3692 u32GuestCr0 |= X86_CR0_NE;
3693
3694 /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3695 bool const fInterceptMF = !(u32ShadowCr0 & X86_CR0_NE);
3696
3697 /*
3698 * Update exception intercepts.
3699 */
3700 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3701 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3702 {
3703 Assert(PDMVmmDevHeapIsEnabled(pVM));
3704 Assert(pVM->hm.s.vmx.pRealModeTSS);
3705 uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3706 }
3707 else
3708 {
3709 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3710 uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3711 if (fInterceptMF)
3712 uXcptBitmap |= RT_BIT(X86_XCPT_MF);
3713 }
3714
3715 /* Additional intercepts for debugging, define these yourself explicitly. */
3716#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3717 uXcptBitmap |= 0
3718 | RT_BIT(X86_XCPT_BP)
3719 | RT_BIT(X86_XCPT_DE)
3720 | RT_BIT(X86_XCPT_NM)
3721 | RT_BIT(X86_XCPT_TS)
3722 | RT_BIT(X86_XCPT_UD)
3723 | RT_BIT(X86_XCPT_NP)
3724 | RT_BIT(X86_XCPT_SS)
3725 | RT_BIT(X86_XCPT_GP)
3726 | RT_BIT(X86_XCPT_PF)
3727 | RT_BIT(X86_XCPT_MF)
3728 ;
3729#elif defined(HMVMX_ALWAYS_TRAP_PF)
3730 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
3731#endif
3732 Assert(pVM->hm.s.fNestedPaging || (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
3733
3734 /*
3735 * Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW).
3736 */
3737 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3738 uint32_t fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3739 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3740 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
3741 else
3742 Assert((fSetCr0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3743
3744 u32GuestCr0 |= fSetCr0;
3745 u32GuestCr0 &= fZapCr0;
3746 u32GuestCr0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3747
3748 /*
3749 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3750 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3751 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3752 */
3753 uint32_t u32Cr0Mask = X86_CR0_PE
3754 | X86_CR0_NE
3755 | (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
3756 | X86_CR0_PG
3757 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3758 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3759 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3760
3761 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3762 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3763 * and @bugref{6944}. */
3764#if 0
3765 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3766 u32Cr0Mask &= ~X86_CR0_PE;
3767#endif
3768 /*
3769 * Finally, update VMCS fields with the CR0 values and the exception bitmap.
3770 */
3771 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCr0);
3772 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32ShadowCr0);
3773 if (u32Cr0Mask != pVCpu->hm.s.vmx.u32Cr0Mask)
3774 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32Cr0Mask);
3775 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
3776 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3777 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3778 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3779 AssertRCReturn(rc, rc);
3780
3781 /* Update our caches. */
3782 pVCpu->hm.s.vmx.u32Cr0Mask = u32Cr0Mask;
3783 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
3784 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3785
3786 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3787
3788 Log4Func(("u32Cr0Mask=%#RX32 u32ShadowCr0=%#RX32 u32GuestCr0=%#RX32 (fSetCr0=%#RX32 fZapCr0=%#RX32\n", u32Cr0Mask,
3789 u32ShadowCr0, u32GuestCr0, fSetCr0, fZapCr0));
3790 }
3791
3792 return VINF_SUCCESS;
3793}
3794
3795
3796/**
3797 * Exports the guest control registers (CR3, CR4) into the guest-state area
3798 * in the VMCS.
3799 *
3800 * @returns VBox strict status code.
3801 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3802 * without unrestricted guest access and the VMMDev is not presently
3803 * mapped (e.g. EFI32).
3804 *
3805 * @param pVCpu The cross context virtual CPU structure.
3806 *
3807 * @remarks No-long-jump zone!!!
3808 */
3809static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu)
3810{
3811 int rc = VINF_SUCCESS;
3812 PVM pVM = pVCpu->CTX_SUFF(pVM);
3813
3814 /*
3815 * Guest CR2.
3816 * It's always loaded in the assembler code. Nothing to do here.
3817 */
3818
3819 /*
3820 * Guest CR3.
3821 */
3822 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
3823 {
3824 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
3825
3826 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3827 if (pVM->hm.s.fNestedPaging)
3828 {
3829 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3830
3831 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3832 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3833 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3834 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3835
3836 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3837 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3838 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3839
3840 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3841 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3842 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3843 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3844 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3845 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3846 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3847
3848 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3849 AssertRCReturn(rc, rc);
3850
3851 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3852 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3853 || CPUMIsGuestPagingEnabledEx(pCtx))
3854 {
3855 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3856 if (CPUMIsGuestInPAEModeEx(pCtx))
3857 {
3858 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3859 AssertRCReturn(rc, rc);
3860 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3861 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3862 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3863 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3864 AssertRCReturn(rc, rc);
3865 }
3866
3867 /*
3868 * The guest's view of its CR3 is unblemished with Nested Paging when the
3869 * guest is using paging or we have unrestricted guest execution to handle
3870 * the guest when it's not using paging.
3871 */
3872 GCPhysGuestCR3 = pCtx->cr3;
3873 }
3874 else
3875 {
3876 /*
3877 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
3878 * thinks it accesses physical memory directly, we use our identity-mapped
3879 * page table to map guest-linear to guest-physical addresses. EPT takes care
3880 * of translating it to host-physical addresses.
3881 */
3882 RTGCPHYS GCPhys;
3883 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3884
3885 /* We obtain it here every time as the guest could have relocated this PCI region. */
3886 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3887 if (RT_SUCCESS(rc))
3888 { /* likely */ }
3889 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3890 {
3891 Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3892 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3893 }
3894 else
3895 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3896
3897 GCPhysGuestCR3 = GCPhys;
3898 }
3899
3900 Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
3901 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3902 AssertRCReturn(rc, rc);
3903 }
3904 else
3905 {
3906 /* Non-nested paging case, just use the hypervisor's CR3. */
3907 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3908
3909 Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
3910 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3911 AssertRCReturn(rc, rc);
3912 }
3913
3914 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3915 }
3916
3917 /*
3918 * Guest CR4.
3919 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3920 */
3921 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
3922 {
3923 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3924 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
3925 Assert(!RT_HI_U32(pCtx->cr4));
3926
3927 uint32_t u32GuestCr4 = pCtx->cr4;
3928 uint32_t const u32ShadowCr4 = pCtx->cr4;
3929
3930 /*
3931 * Setup VT-x's view of the guest CR4.
3932 *
3933 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3934 * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3935 * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3936 *
3937 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3938 */
3939 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3940 {
3941 Assert(pVM->hm.s.vmx.pRealModeTSS);
3942 Assert(PDMVmmDevHeapIsEnabled(pVM));
3943 u32GuestCr4 &= ~X86_CR4_VME;
3944 }
3945
3946 if (pVM->hm.s.fNestedPaging)
3947 {
3948 if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3949 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3950 {
3951 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3952 u32GuestCr4 |= X86_CR4_PSE;
3953 /* Our identity mapping is a 32-bit page directory. */
3954 u32GuestCr4 &= ~X86_CR4_PAE;
3955 }
3956 /* else use guest CR4.*/
3957 }
3958 else
3959 {
3960 /*
3961 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3962 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3963 */
3964 switch (pVCpu->hm.s.enmShadowMode)
3965 {
3966 case PGMMODE_REAL: /* Real-mode. */
3967 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3968 case PGMMODE_32_BIT: /* 32-bit paging. */
3969 {
3970 u32GuestCr4 &= ~X86_CR4_PAE;
3971 break;
3972 }
3973
3974 case PGMMODE_PAE: /* PAE paging. */
3975 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3976 {
3977 u32GuestCr4 |= X86_CR4_PAE;
3978 break;
3979 }
3980
3981 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3982 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3983#ifdef VBOX_ENABLE_64_BITS_GUESTS
3984 break;
3985#endif
3986 default:
3987 AssertFailed();
3988 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3989 }
3990 }
3991
3992 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3993 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3994 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3995 u32GuestCr4 |= fSetCr4;
3996 u32GuestCr4 &= fZapCr4;
3997
3998 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them,
3999 that would cause a VM-exit. */
4000 uint32_t u32Cr4Mask = X86_CR4_VME
4001 | X86_CR4_PAE
4002 | X86_CR4_PGE
4003 | X86_CR4_PSE
4004 | X86_CR4_VMXE;
4005 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4006 u32Cr4Mask |= X86_CR4_OSXSAVE;
4007 if (pVM->cpum.ro.GuestFeatures.fPcid)
4008 u32Cr4Mask |= X86_CR4_PCIDE;
4009
4010 /* Write VT-x's view of the guest CR4, the CR4 modify mask and the read-only CR4 shadow
4011 into the VMCS and update our cache. */
4012 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCr4);
4013 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32ShadowCr4);
4014 if (pVCpu->hm.s.vmx.u32Cr4Mask != u32Cr4Mask)
4015 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32Cr4Mask);
4016 AssertRCReturn(rc, rc);
4017 pVCpu->hm.s.vmx.u32Cr4Mask = u32Cr4Mask;
4018
4019 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4020 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
4021
4022 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
4023
4024 Log4Func(("u32GuestCr4=%#RX32 u32ShadowCr4=%#RX32 (fSetCr4=%#RX32 fZapCr4=%#RX32)\n", u32GuestCr4, u32ShadowCr4, fSetCr4,
4025 fZapCr4));
4026 }
4027 return rc;
4028}
4029
4030
4031/**
4032 * Exports the guest debug registers into the guest-state area in the VMCS.
4033 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4034 *
4035 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4036 *
4037 * @returns VBox status code.
4038 * @param pVCpu The cross context virtual CPU structure.
4039 *
4040 * @remarks No-long-jump zone!!!
4041 */
4042static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu)
4043{
4044 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4045
4046#ifdef VBOX_STRICT
4047 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4048 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
4049 {
4050 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4051 Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0);
4052 Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
4053 }
4054#endif
4055
4056 bool fSteppingDB = false;
4057 bool fInterceptMovDRx = false;
4058 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
4059 if (pVCpu->hm.s.fSingleInstruction)
4060 {
4061 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4062 PVM pVM = pVCpu->CTX_SUFF(pVM);
4063 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
4064 {
4065 uProcCtls |= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
4066 Assert(fSteppingDB == false);
4067 }
4068 else
4069 {
4070 pVCpu->cpum.GstCtx.eflags.u32 |= X86_EFL_TF;
4071 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_RFLAGS;
4072 pVCpu->hm.s.fClearTrapFlag = true;
4073 fSteppingDB = true;
4074 }
4075 }
4076
4077 uint32_t u32GuestDr7;
4078 if ( fSteppingDB
4079 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4080 {
4081 /*
4082 * Use the combined guest and host DRx values found in the hypervisor register set
4083 * because the debugger has breakpoints active or someone is single stepping on the
4084 * host side without a monitor trap flag.
4085 *
4086 * Note! DBGF expects a clean DR6 state before executing guest code.
4087 */
4088#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4089 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4090 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4091 {
4092 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4093 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4094 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4095 }
4096 else
4097#endif
4098 if (!CPUMIsHyperDebugStateActive(pVCpu))
4099 {
4100 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4101 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4102 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4103 }
4104
4105 /* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
4106 u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
4107 pVCpu->hm.s.fUsingHyperDR7 = true;
4108 fInterceptMovDRx = true;
4109 }
4110 else
4111 {
4112 /*
4113 * If the guest has enabled debug registers, we need to load them prior to
4114 * executing guest code so they'll trigger at the right time.
4115 */
4116 if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD))
4117 {
4118#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4119 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4120 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4121 {
4122 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4123 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4124 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4125 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4126 }
4127 else
4128#endif
4129 if (!CPUMIsGuestDebugStateActive(pVCpu))
4130 {
4131 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4132 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4133 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4134 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4135 }
4136 Assert(!fInterceptMovDRx);
4137 }
4138 /*
4139 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4140 * must intercept #DB in order to maintain a correct DR6 guest value, and
4141 * because we need to intercept it to prevent nested #DBs from hanging the
4142 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4143 */
4144#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4145 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4146 && !CPUMIsGuestDebugStateActive(pVCpu))
4147#else
4148 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4149#endif
4150 {
4151 fInterceptMovDRx = true;
4152 }
4153
4154 /* Update DR7 with the actual guest value. */
4155 u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
4156 pVCpu->hm.s.fUsingHyperDR7 = false;
4157 }
4158
4159 if (fInterceptMovDRx)
4160 uProcCtls |= VMX_PROC_CTLS_MOV_DR_EXIT;
4161 else
4162 uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
4163
4164 /*
4165 * Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
4166 * monitor-trap flag and update our cache.
4167 */
4168 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
4169 {
4170 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4171 AssertRCReturn(rc2, rc2);
4172 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
4173 }
4174
4175 /*
4176 * Update guest DR7.
4177 */
4178 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
4179 AssertRCReturn(rc, rc);
4180
4181 return VINF_SUCCESS;
4182}
4183
4184
4185#ifdef VBOX_STRICT
4186/**
4187 * Strict function to validate segment registers.
4188 *
4189 * @param pVCpu The cross context virtual CPU structure.
4190 *
4191 * @remarks Will import guest CR0 on strict builds during validation of
4192 * segments.
4193 */
4194static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu)
4195{
4196 /*
4197 * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
4198 *
4199 * The reason we check for attribute value 0 in this function and not just the unusable bit is
4200 * because hmR0VmxExportGuestSegmentReg() only updates the VMCS' copy of the value with the unusable bit
4201 * and doesn't change the guest-context value.
4202 */
4203 PVM pVM = pVCpu->CTX_SUFF(pVM);
4204 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4205 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
4206 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4207 && ( !CPUMIsGuestInRealModeEx(pCtx)
4208 && !CPUMIsGuestInV86ModeEx(pCtx)))
4209 {
4210 /* Protected mode checks */
4211 /* CS */
4212 Assert(pCtx->cs.Attr.n.u1Present);
4213 Assert(!(pCtx->cs.Attr.u & 0xf00));
4214 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4215 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4216 || !(pCtx->cs.Attr.n.u1Granularity));
4217 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4218 || (pCtx->cs.Attr.n.u1Granularity));
4219 /* CS cannot be loaded with NULL in protected mode. */
4220 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4221 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4222 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4223 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4224 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4225 else
4226 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4227 /* SS */
4228 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4229 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4230 if ( !(pCtx->cr0 & X86_CR0_PE)
4231 || pCtx->cs.Attr.n.u4Type == 3)
4232 {
4233 Assert(!pCtx->ss.Attr.n.u2Dpl);
4234 }
4235 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4236 {
4237 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4238 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4239 Assert(pCtx->ss.Attr.n.u1Present);
4240 Assert(!(pCtx->ss.Attr.u & 0xf00));
4241 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4242 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4243 || !(pCtx->ss.Attr.n.u1Granularity));
4244 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4245 || (pCtx->ss.Attr.n.u1Granularity));
4246 }
4247 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmentReg(). */
4248 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4249 {
4250 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4251 Assert(pCtx->ds.Attr.n.u1Present);
4252 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4253 Assert(!(pCtx->ds.Attr.u & 0xf00));
4254 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4255 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4256 || !(pCtx->ds.Attr.n.u1Granularity));
4257 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4258 || (pCtx->ds.Attr.n.u1Granularity));
4259 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4260 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4261 }
4262 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4263 {
4264 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4265 Assert(pCtx->es.Attr.n.u1Present);
4266 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4267 Assert(!(pCtx->es.Attr.u & 0xf00));
4268 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4269 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4270 || !(pCtx->es.Attr.n.u1Granularity));
4271 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4272 || (pCtx->es.Attr.n.u1Granularity));
4273 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4274 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4275 }
4276 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4277 {
4278 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4279 Assert(pCtx->fs.Attr.n.u1Present);
4280 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4281 Assert(!(pCtx->fs.Attr.u & 0xf00));
4282 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4283 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4284 || !(pCtx->fs.Attr.n.u1Granularity));
4285 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4286 || (pCtx->fs.Attr.n.u1Granularity));
4287 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4288 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4289 }
4290 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4291 {
4292 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4293 Assert(pCtx->gs.Attr.n.u1Present);
4294 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4295 Assert(!(pCtx->gs.Attr.u & 0xf00));
4296 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4297 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4298 || !(pCtx->gs.Attr.n.u1Granularity));
4299 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4300 || (pCtx->gs.Attr.n.u1Granularity));
4301 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4302 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4303 }
4304 /* 64-bit capable CPUs. */
4305# if HC_ARCH_BITS == 64
4306 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4307 Assert(!pCtx->ss.Attr.u || !RT_HI_U32(pCtx->ss.u64Base));
4308 Assert(!pCtx->ds.Attr.u || !RT_HI_U32(pCtx->ds.u64Base));
4309 Assert(!pCtx->es.Attr.u || !RT_HI_U32(pCtx->es.u64Base));
4310# endif
4311 }
4312 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4313 || ( CPUMIsGuestInRealModeEx(pCtx)
4314 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4315 {
4316 /* Real and v86 mode checks. */
4317 /* hmR0VmxExportGuestSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4318 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4319 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4320 {
4321 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4322 }
4323 else
4324 {
4325 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4326 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4327 }
4328
4329 /* CS */
4330 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4331 Assert(pCtx->cs.u32Limit == 0xffff);
4332 Assert(u32CSAttr == 0xf3);
4333 /* SS */
4334 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4335 Assert(pCtx->ss.u32Limit == 0xffff);
4336 Assert(u32SSAttr == 0xf3);
4337 /* DS */
4338 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4339 Assert(pCtx->ds.u32Limit == 0xffff);
4340 Assert(u32DSAttr == 0xf3);
4341 /* ES */
4342 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4343 Assert(pCtx->es.u32Limit == 0xffff);
4344 Assert(u32ESAttr == 0xf3);
4345 /* FS */
4346 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4347 Assert(pCtx->fs.u32Limit == 0xffff);
4348 Assert(u32FSAttr == 0xf3);
4349 /* GS */
4350 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4351 Assert(pCtx->gs.u32Limit == 0xffff);
4352 Assert(u32GSAttr == 0xf3);
4353 /* 64-bit capable CPUs. */
4354# if HC_ARCH_BITS == 64
4355 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4356 Assert(!u32SSAttr || !RT_HI_U32(pCtx->ss.u64Base));
4357 Assert(!u32DSAttr || !RT_HI_U32(pCtx->ds.u64Base));
4358 Assert(!u32ESAttr || !RT_HI_U32(pCtx->es.u64Base));
4359# endif
4360 }
4361}
4362#endif /* VBOX_STRICT */
4363
4364
4365/**
4366 * Exports a guest segment register into the guest-state area in the VMCS.
4367 *
4368 * @returns VBox status code.
4369 * @param pVCpu The cross context virtual CPU structure.
4370 * @param idxSel Index of the selector in the VMCS.
4371 * @param idxLimit Index of the segment limit in the VMCS.
4372 * @param idxBase Index of the segment base in the VMCS.
4373 * @param idxAccess Index of the access rights of the segment in the VMCS.
4374 * @param pSelReg Pointer to the segment selector.
4375 *
4376 * @remarks No-long-jump zone!!!
4377 */
4378static int hmR0VmxExportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
4379 PCCPUMSELREG pSelReg)
4380{
4381 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4382 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4383 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4384 AssertRCReturn(rc, rc);
4385
4386 uint32_t u32Access = pSelReg->Attr.u;
4387 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4388 {
4389 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4390 u32Access = 0xf3;
4391 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4392 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4393 }
4394 else
4395 {
4396 /*
4397 * The way to differentiate between whether this is really a null selector or was just
4398 * a selector loaded with 0 in real-mode is using the segment attributes. A selector
4399 * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4400 * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4401 * NULL selectors loaded in protected-mode have their attribute as 0.
4402 */
4403 if (!u32Access)
4404 u32Access = X86DESCATTR_UNUSABLE;
4405 }
4406
4407 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4408 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4409 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4410
4411 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4412 AssertRCReturn(rc, rc);
4413 return rc;
4414}
4415
4416
4417/**
4418 * Exports the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4419 * into the guest-state area in the VMCS.
4420 *
4421 * @returns VBox status code.
4422 * @param pVCpu The cross context virtual CPU structure.
4423 *
4424 * @remarks Will import guest CR0 on strict builds during validation of
4425 * segments.
4426 * @remarks No-long-jump zone!!!
4427 */
4428static int hmR0VmxExportGuestSegmentRegs(PVMCPU pVCpu)
4429{
4430 int rc = VERR_INTERNAL_ERROR_5;
4431 PVM pVM = pVCpu->CTX_SUFF(pVM);
4432 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4433
4434 /*
4435 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4436 */
4437 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4438 {
4439#ifdef VBOX_WITH_REM
4440 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4441 {
4442 Assert(pVM->hm.s.vmx.pRealModeTSS);
4443 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4444 if ( pVCpu->hm.s.vmx.fWasInRealMode
4445 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4446 {
4447 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4448 in real-mode (e.g. OpenBSD 4.0) */
4449 REMFlushTBs(pVM);
4450 Log4Func(("Switch to protected mode detected!\n"));
4451 pVCpu->hm.s.vmx.fWasInRealMode = false;
4452 }
4453 }
4454#endif
4455 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
4456 {
4457 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
4458 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4459 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pCtx->cs.Attr.u;
4460 rc = HMVMX_EXPORT_SREG(CS, &pCtx->cs);
4461 AssertRCReturn(rc, rc);
4462 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
4463 }
4464
4465 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
4466 {
4467 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
4468 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4469 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pCtx->ss.Attr.u;
4470 rc = HMVMX_EXPORT_SREG(SS, &pCtx->ss);
4471 AssertRCReturn(rc, rc);
4472 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
4473 }
4474
4475 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
4476 {
4477 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
4478 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4479 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pCtx->ds.Attr.u;
4480 rc = HMVMX_EXPORT_SREG(DS, &pCtx->ds);
4481 AssertRCReturn(rc, rc);
4482 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
4483 }
4484
4485 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
4486 {
4487 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
4488 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4489 pVCpu->hm.s.vmx.RealMode.AttrES.u = pCtx->es.Attr.u;
4490 rc = HMVMX_EXPORT_SREG(ES, &pCtx->es);
4491 AssertRCReturn(rc, rc);
4492 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
4493 }
4494
4495 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
4496 {
4497 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
4498 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4499 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pCtx->fs.Attr.u;
4500 rc = HMVMX_EXPORT_SREG(FS, &pCtx->fs);
4501 AssertRCReturn(rc, rc);
4502 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
4503 }
4504
4505 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
4506 {
4507 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
4508 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4509 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pCtx->gs.Attr.u;
4510 rc = HMVMX_EXPORT_SREG(GS, &pCtx->gs);
4511 AssertRCReturn(rc, rc);
4512 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
4513 }
4514
4515#ifdef VBOX_STRICT
4516 hmR0VmxValidateSegmentRegs(pVCpu);
4517#endif
4518
4519 Log4Func(("CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pCtx->cs.Sel, pCtx->cs.u64Base,
4520 pCtx->cs.u32Limit, pCtx->cs.Attr.u));
4521 }
4522
4523 /*
4524 * Guest TR.
4525 */
4526 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
4527 {
4528 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
4529
4530 /*
4531 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
4532 * achieved using the interrupt redirection bitmap (all bits cleared to let the guest
4533 * handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4534 */
4535 uint16_t u16Sel = 0;
4536 uint32_t u32Limit = 0;
4537 uint64_t u64Base = 0;
4538 uint32_t u32AccessRights = 0;
4539
4540 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4541 {
4542 u16Sel = pCtx->tr.Sel;
4543 u32Limit = pCtx->tr.u32Limit;
4544 u64Base = pCtx->tr.u64Base;
4545 u32AccessRights = pCtx->tr.Attr.u;
4546 }
4547 else
4548 {
4549 Assert(pVM->hm.s.vmx.pRealModeTSS);
4550 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
4551
4552 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4553 RTGCPHYS GCPhys;
4554 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4555 AssertRCReturn(rc, rc);
4556
4557 X86DESCATTR DescAttr;
4558 DescAttr.u = 0;
4559 DescAttr.n.u1Present = 1;
4560 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4561
4562 u16Sel = 0;
4563 u32Limit = HM_VTX_TSS_SIZE;
4564 u64Base = GCPhys; /* in real-mode phys = virt. */
4565 u32AccessRights = DescAttr.u;
4566 }
4567
4568 /* Validate. */
4569 Assert(!(u16Sel & RT_BIT(2)));
4570 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4571 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4572 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4573 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4574 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4575 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4576 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4577 Assert( (u32Limit & 0xfff) == 0xfff
4578 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4579 Assert( !(pCtx->tr.u32Limit & 0xfff00000)
4580 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4581
4582 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4583 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4584 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4585 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4586 AssertRCReturn(rc, rc);
4587
4588 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
4589 Log4Func(("TR base=%#RX64\n", pCtx->tr.u64Base));
4590 }
4591
4592 /*
4593 * Guest GDTR.
4594 */
4595 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4596 {
4597 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
4598
4599 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
4600 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
4601 AssertRCReturn(rc, rc);
4602
4603 /* Validate. */
4604 Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4605
4606 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4607 Log4Func(("GDTR base=%#RX64\n", pCtx->gdtr.pGdt));
4608 }
4609
4610 /*
4611 * Guest LDTR.
4612 */
4613 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4614 {
4615 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
4616
4617 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4618 uint32_t u32Access = 0;
4619 if (!pCtx->ldtr.Attr.u)
4620 u32Access = X86DESCATTR_UNUSABLE;
4621 else
4622 u32Access = pCtx->ldtr.Attr.u;
4623
4624 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
4625 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
4626 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4627 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
4628 AssertRCReturn(rc, rc);
4629
4630 /* Validate. */
4631 if (!(u32Access & X86DESCATTR_UNUSABLE))
4632 {
4633 Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4634 Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4635 Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4636 Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4637 Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4638 Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4639 Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
4640 || !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4641 Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
4642 || pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4643 }
4644
4645 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4646 Log4Func(("LDTR base=%#RX64\n", pCtx->ldtr.u64Base));
4647 }
4648
4649 /*
4650 * Guest IDTR.
4651 */
4652 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4653 {
4654 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
4655
4656 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
4657 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
4658 AssertRCReturn(rc, rc);
4659
4660 /* Validate. */
4661 Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4662
4663 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4664 Log4Func(("IDTR base=%#RX64\n", pCtx->idtr.pIdt));
4665 }
4666
4667 return VINF_SUCCESS;
4668}
4669
4670
4671/**
4672 * Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4673 * areas.
4674 *
4675 * These MSRs will automatically be loaded to the host CPU on every successful
4676 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4677 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4678 * -not- updated here for performance reasons. See hmR0VmxExportHostMsrs().
4679 *
4680 * Also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4681 *
4682 * @returns VBox status code.
4683 * @param pVCpu The cross context virtual CPU structure.
4684 *
4685 * @remarks No-long-jump zone!!!
4686 */
4687static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu)
4688{
4689 AssertPtr(pVCpu);
4690 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4691
4692 /*
4693 * MSRs that we use the auto-load/store MSR area in the VMCS.
4694 * For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
4695 */
4696 PVM pVM = pVCpu->CTX_SUFF(pVM);
4697 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4698 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4699 {
4700 if (pVM->hm.s.fAllow64BitGuests)
4701 {
4702#if HC_ARCH_BITS == 32
4703 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS | CPUMCTX_EXTRN_KERNEL_GS_BASE);
4704
4705 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pCtx->msrLSTAR, false, NULL);
4706 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pCtx->msrSTAR, false, NULL);
4707 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pCtx->msrSFMASK, false, NULL);
4708 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, false, NULL);
4709 AssertRCReturn(rc, rc);
4710# ifdef LOG_ENABLED
4711 PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4712 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4713 Log4Func(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4714# endif
4715#endif
4716 }
4717 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4718 }
4719
4720 /*
4721 * Guest Sysenter MSRs.
4722 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4723 * VM-exits on WRMSRs for these MSRs.
4724 */
4725 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4726 {
4727 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
4728
4729 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4730 {
4731 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
4732 AssertRCReturn(rc, rc);
4733 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4734 }
4735
4736 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4737 {
4738 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
4739 AssertRCReturn(rc, rc);
4740 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4741 }
4742
4743 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4744 {
4745 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
4746 AssertRCReturn(rc, rc);
4747 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4748 }
4749 }
4750
4751 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4752 {
4753 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
4754
4755 if (hmR0VmxShouldSwapEferMsr(pVCpu))
4756 {
4757 /*
4758 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4759 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4760 */
4761 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4762 {
4763 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
4764 AssertRCReturn(rc,rc);
4765 Log4Func(("EFER=%#RX64\n", pCtx->msrEFER));
4766 }
4767 else
4768 {
4769 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pCtx->msrEFER, false /* fUpdateHostMsr */,
4770 NULL /* pfAddedAndUpdated */);
4771 AssertRCReturn(rc, rc);
4772
4773 /* We need to intercept reads too, see @bugref{7386#c16}. */
4774 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
4775 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4776 Log4Func(("MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pCtx->msrEFER,
4777 pVCpu->hm.s.vmx.cMsrs));
4778 }
4779 }
4780 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4781 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4782 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4783 }
4784
4785 return VINF_SUCCESS;
4786}
4787
4788
4789#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4790/**
4791 * Check if guest state allows safe use of 32-bit switcher again.
4792 *
4793 * Segment bases and protected mode structures must be 32-bit addressable
4794 * because the 32-bit switcher will ignore high dword when writing these VMCS
4795 * fields. See @bugref{8432} for details.
4796 *
4797 * @returns true if safe, false if must continue to use the 64-bit switcher.
4798 * @param pCtx Pointer to the guest-CPU context.
4799 *
4800 * @remarks No-long-jump zone!!!
4801 */
4802static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
4803{
4804 if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4805 if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4806 if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4807 if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4808 if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4809 if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4810 if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4811 if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4812 if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4813 if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4814
4815 /* All good, bases are 32-bit. */
4816 return true;
4817}
4818#endif
4819
4820
4821/**
4822 * Selects up the appropriate function to run guest code.
4823 *
4824 * @returns VBox status code.
4825 * @param pVCpu The cross context virtual CPU structure.
4826 *
4827 * @remarks No-long-jump zone!!!
4828 */
4829static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu)
4830{
4831 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4832 if (CPUMIsGuestInLongModeEx(pCtx))
4833 {
4834#ifndef VBOX_ENABLE_64_BITS_GUESTS
4835 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4836#endif
4837 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4838#if HC_ARCH_BITS == 32
4839 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4840 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4841 {
4842#ifdef VBOX_STRICT
4843 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4844 {
4845 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4846 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4847 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4848 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4849 | HM_CHANGED_VMX_ENTRY_CTLS
4850 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4851 }
4852#endif
4853 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4854
4855 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4856 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4857 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4858 Log4Func(("Selected 64-bit switcher\n"));
4859 }
4860#else
4861 /* 64-bit host. */
4862 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4863#endif
4864 }
4865 else
4866 {
4867 /* Guest is not in long mode, use the 32-bit handler. */
4868#if HC_ARCH_BITS == 32
4869 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4870 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4871 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4872 {
4873# ifdef VBOX_STRICT
4874 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4875 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4876 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4877 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4878 | HM_CHANGED_VMX_ENTRY_CTLS
4879 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4880# endif
4881 }
4882# ifdef VBOX_ENABLE_64_BITS_GUESTS
4883 /*
4884 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
4885 * design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
4886 * switcher flag because now we know the guest is in a sane state where it's safe
4887 * to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4888 * the much faster 32-bit switcher again.
4889 */
4890 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4891 {
4892 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4893 Log4Func(("Selected 32-bit switcher\n"));
4894 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4895 }
4896 else
4897 {
4898 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4899 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4900 || hmR0VmxIs32BitSwitcherSafe(pCtx))
4901 {
4902 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4903 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4904 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
4905 | HM_CHANGED_VMX_ENTRY_CTLS
4906 | HM_CHANGED_VMX_EXIT_CTLS
4907 | HM_CHANGED_HOST_CONTEXT);
4908 Log4Func(("Selected 32-bit switcher (safe)\n"));
4909 }
4910 }
4911# else
4912 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4913# endif
4914#else
4915 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4916#endif
4917 }
4918 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4919 return VINF_SUCCESS;
4920}
4921
4922
4923/**
4924 * Wrapper for running the guest code in VT-x.
4925 *
4926 * @returns VBox status code, no informational status codes.
4927 * @param pVCpu The cross context virtual CPU structure.
4928 *
4929 * @remarks No-long-jump zone!!!
4930 */
4931DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu)
4932{
4933 /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4934 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4935 pCtx->fExtrn |= HMVMX_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
4936
4937 /*
4938 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4939 * floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
4940 * callee-saved and thus the need for this XMM wrapper.
4941 *
4942 * See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
4943 */
4944 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4945 /** @todo Add stats for resume vs launch. */
4946 PVM pVM = pVCpu->CTX_SUFF(pVM);
4947#ifdef VBOX_WITH_KERNEL_USING_XMM
4948 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4949#else
4950 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4951#endif
4952 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4953 return rc;
4954}
4955
4956
4957/**
4958 * Reports world-switch error and dumps some useful debug info.
4959 *
4960 * @param pVCpu The cross context virtual CPU structure.
4961 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4962 * @param pVmxTransient Pointer to the VMX transient structure (only
4963 * exitReason updated).
4964 */
4965static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
4966{
4967 Assert(pVCpu);
4968 Assert(pVmxTransient);
4969 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
4970
4971 Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
4972 switch (rcVMRun)
4973 {
4974 case VERR_VMX_INVALID_VMXON_PTR:
4975 AssertFailed();
4976 break;
4977 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4978 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4979 {
4980 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4981 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4982 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
4983 AssertRC(rc);
4984
4985 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4986 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4987 Cannot do it here as we may have been long preempted. */
4988
4989#ifdef VBOX_STRICT
4990 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4991 pVmxTransient->uExitReason));
4992 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
4993 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4994 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4995 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4996 else
4997 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4998 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4999 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5000
5001 /* VMX control bits. */
5002 uint32_t u32Val;
5003 uint64_t u64Val;
5004 RTHCUINTREG uHCReg;
5005 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5006 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5007 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5008 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5009 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
5010 {
5011 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5012 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5013 }
5014 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5015 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5016 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5017 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5018 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5019 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5020 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5021 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5022 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5023 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5024 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5025 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5026 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5027 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5028 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5029 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5030 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5031 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5032 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5033 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5034 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5035 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5036 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5037 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5038 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5039 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5040 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5041 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5042 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5043 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5044 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5045 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5046 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5047 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5048 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5049 {
5050 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5051 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5052 }
5053
5054 /* Guest bits. */
5055 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5056 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
5057 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5058 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
5059 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5060 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
5061 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
5062 {
5063 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5064 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5065 }
5066
5067 /* Host bits. */
5068 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5069 Log4(("Host CR0 %#RHr\n", uHCReg));
5070 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5071 Log4(("Host CR3 %#RHr\n", uHCReg));
5072 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5073 Log4(("Host CR4 %#RHr\n", uHCReg));
5074
5075 RTGDTR HostGdtr;
5076 PCX86DESCHC pDesc;
5077 ASMGetGDTR(&HostGdtr);
5078 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5079 Log4(("Host CS %#08x\n", u32Val));
5080 if (u32Val < HostGdtr.cbGdt)
5081 {
5082 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5083 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5084 }
5085
5086 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5087 Log4(("Host DS %#08x\n", u32Val));
5088 if (u32Val < HostGdtr.cbGdt)
5089 {
5090 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5091 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5092 }
5093
5094 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5095 Log4(("Host ES %#08x\n", u32Val));
5096 if (u32Val < HostGdtr.cbGdt)
5097 {
5098 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5099 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5100 }
5101
5102 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5103 Log4(("Host FS %#08x\n", u32Val));
5104 if (u32Val < HostGdtr.cbGdt)
5105 {
5106 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5107 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5108 }
5109
5110 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5111 Log4(("Host GS %#08x\n", u32Val));
5112 if (u32Val < HostGdtr.cbGdt)
5113 {
5114 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5115 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5116 }
5117
5118 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5119 Log4(("Host SS %#08x\n", u32Val));
5120 if (u32Val < HostGdtr.cbGdt)
5121 {
5122 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5123 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5124 }
5125
5126 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5127 Log4(("Host TR %#08x\n", u32Val));
5128 if (u32Val < HostGdtr.cbGdt)
5129 {
5130 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5131 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5132 }
5133
5134 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5135 Log4(("Host TR Base %#RHv\n", uHCReg));
5136 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5137 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5138 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5139 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5140 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5141 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5142 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5143 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5144 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5145 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5146 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5147 Log4(("Host RSP %#RHv\n", uHCReg));
5148 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5149 Log4(("Host RIP %#RHv\n", uHCReg));
5150# if HC_ARCH_BITS == 64
5151 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5152 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5153 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5154 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5155 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5156 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5157# endif
5158#endif /* VBOX_STRICT */
5159 break;
5160 }
5161
5162 default:
5163 /* Impossible */
5164 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5165 break;
5166 }
5167}
5168
5169
5170#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5171#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5172# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5173#endif
5174#ifdef VBOX_STRICT
5175static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5176{
5177 switch (idxField)
5178 {
5179 case VMX_VMCS_GUEST_RIP:
5180 case VMX_VMCS_GUEST_RSP:
5181 case VMX_VMCS_GUEST_SYSENTER_EIP:
5182 case VMX_VMCS_GUEST_SYSENTER_ESP:
5183 case VMX_VMCS_GUEST_GDTR_BASE:
5184 case VMX_VMCS_GUEST_IDTR_BASE:
5185 case VMX_VMCS_GUEST_CS_BASE:
5186 case VMX_VMCS_GUEST_DS_BASE:
5187 case VMX_VMCS_GUEST_ES_BASE:
5188 case VMX_VMCS_GUEST_FS_BASE:
5189 case VMX_VMCS_GUEST_GS_BASE:
5190 case VMX_VMCS_GUEST_SS_BASE:
5191 case VMX_VMCS_GUEST_LDTR_BASE:
5192 case VMX_VMCS_GUEST_TR_BASE:
5193 case VMX_VMCS_GUEST_CR3:
5194 return true;
5195 }
5196 return false;
5197}
5198
5199static bool hmR0VmxIsValidReadField(uint32_t idxField)
5200{
5201 switch (idxField)
5202 {
5203 /* Read-only fields. */
5204 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5205 return true;
5206 }
5207 /* Remaining readable fields should also be writable. */
5208 return hmR0VmxIsValidWriteField(idxField);
5209}
5210#endif /* VBOX_STRICT */
5211
5212
5213/**
5214 * Executes the specified handler in 64-bit mode.
5215 *
5216 * @returns VBox status code (no informational status codes).
5217 * @param pVCpu The cross context virtual CPU structure.
5218 * @param enmOp The operation to perform.
5219 * @param cParams Number of parameters.
5220 * @param paParam Array of 32-bit parameters.
5221 */
5222VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
5223{
5224 PVM pVM = pVCpu->CTX_SUFF(pVM);
5225 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5226 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5227 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5228 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5229
5230#ifdef VBOX_STRICT
5231 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5232 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5233
5234 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5235 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5236#endif
5237
5238 /* Disable interrupts. */
5239 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5240
5241#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5242 RTCPUID idHostCpu = RTMpCpuId();
5243 CPUMR0SetLApic(pVCpu, idHostCpu);
5244#endif
5245
5246 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5247 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5248
5249 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5250 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5251 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5252
5253 /* Leave VMX Root Mode. */
5254 VMXDisable();
5255
5256 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5257
5258 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5259 CPUMSetHyperEIP(pVCpu, enmOp);
5260 for (int i = (int)cParams - 1; i >= 0; i--)
5261 CPUMPushHyper(pVCpu, paParam[i]);
5262
5263 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5264
5265 /* Call the switcher. */
5266 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
5267 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5268
5269 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5270 /* Make sure the VMX instructions don't cause #UD faults. */
5271 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5272
5273 /* Re-enter VMX Root Mode */
5274 int rc2 = VMXEnable(HCPhysCpuPage);
5275 if (RT_FAILURE(rc2))
5276 {
5277 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5278 ASMSetFlags(fOldEFlags);
5279 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5280 return rc2;
5281 }
5282
5283 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5284 AssertRC(rc2);
5285 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5286 Assert(!(ASMGetFlags() & X86_EFL_IF));
5287 ASMSetFlags(fOldEFlags);
5288 return rc;
5289}
5290
5291
5292/**
5293 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5294 * supporting 64-bit guests.
5295 *
5296 * @returns VBox status code.
5297 * @param fResume Whether to VMLAUNCH or VMRESUME.
5298 * @param pCtx Pointer to the guest-CPU context.
5299 * @param pCache Pointer to the VMCS cache.
5300 * @param pVM The cross context VM structure.
5301 * @param pVCpu The cross context virtual CPU structure.
5302 */
5303DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5304{
5305 NOREF(fResume);
5306
5307 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5308 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5309
5310#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5311 pCache->uPos = 1;
5312 pCache->interPD = PGMGetInterPaeCR3(pVM);
5313 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5314#endif
5315
5316#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5317 pCache->TestIn.HCPhysCpuPage = 0;
5318 pCache->TestIn.HCPhysVmcs = 0;
5319 pCache->TestIn.pCache = 0;
5320 pCache->TestOut.HCPhysVmcs = 0;
5321 pCache->TestOut.pCache = 0;
5322 pCache->TestOut.pCtx = 0;
5323 pCache->TestOut.eflags = 0;
5324#else
5325 NOREF(pCache);
5326#endif
5327
5328 uint32_t aParam[10];
5329 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5330 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5331 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5332 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5333 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5334 aParam[5] = 0;
5335 aParam[6] = VM_RC_ADDR(pVM, pVM);
5336 aParam[7] = 0;
5337 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5338 aParam[9] = 0;
5339
5340#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5341 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5342 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5343#endif
5344 int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5345
5346#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5347 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5348 Assert(pCtx->dr[4] == 10);
5349 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5350#endif
5351
5352#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5353 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5354 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5355 pVCpu->hm.s.vmx.HCPhysVmcs));
5356 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5357 pCache->TestOut.HCPhysVmcs));
5358 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5359 pCache->TestOut.pCache));
5360 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5361 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5362 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5363 pCache->TestOut.pCtx));
5364 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5365#endif
5366 NOREF(pCtx);
5367 return rc;
5368}
5369
5370
5371/**
5372 * Initialize the VMCS-Read cache.
5373 *
5374 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5375 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5376 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5377 * (those that have a 32-bit FULL & HIGH part).
5378 *
5379 * @returns VBox status code.
5380 * @param pVCpu The cross context virtual CPU structure.
5381 */
5382static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
5383{
5384#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5385 do { \
5386 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5387 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5388 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5389 ++cReadFields; \
5390 } while (0)
5391
5392 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5393 uint32_t cReadFields = 0;
5394
5395 /*
5396 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5397 * and serve to indicate exceptions to the rules.
5398 */
5399
5400 /* Guest-natural selector base fields. */
5401#if 0
5402 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5403 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5404 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5405#endif
5406 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5407 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5408 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5409 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5410 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5411 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5412 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5413 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5414 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5415 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5416 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5417 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5418#if 0
5419 /* Unused natural width guest-state fields. */
5420 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5421 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5422#endif
5423 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5424 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5425
5426 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
5427 these 64-bit fields (using "FULL" and "HIGH" fields). */
5428#if 0
5429 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5430 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5431 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5432 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5433 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5434 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5435 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5436 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5437 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5438#endif
5439
5440 /* Natural width guest-state fields. */
5441 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5442#if 0
5443 /* Currently unused field. */
5444 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5445#endif
5446
5447 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5448 {
5449 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5450 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5451 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5452 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5453 }
5454 else
5455 {
5456 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5457 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5458 }
5459
5460#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5461 return VINF_SUCCESS;
5462}
5463
5464
5465/**
5466 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5467 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5468 * darwin, running 64-bit guests).
5469 *
5470 * @returns VBox status code.
5471 * @param pVCpu The cross context virtual CPU structure.
5472 * @param idxField The VMCS field encoding.
5473 * @param u64Val 16, 32 or 64-bit value.
5474 */
5475VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5476{
5477 int rc;
5478 switch (idxField)
5479 {
5480 /*
5481 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5482 */
5483 /* 64-bit Control fields. */
5484 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5485 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5486 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5487 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5488 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5489 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5490 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5491 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5492 case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
5493 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5494 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5495 case VMX_VMCS64_CTRL_EPTP_FULL:
5496 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5497 /* 64-bit Guest-state fields. */
5498 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5499 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5500 case VMX_VMCS64_GUEST_PAT_FULL:
5501 case VMX_VMCS64_GUEST_EFER_FULL:
5502 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5503 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5504 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5505 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5506 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5507 /* 64-bit Host-state fields. */
5508 case VMX_VMCS64_HOST_PAT_FULL:
5509 case VMX_VMCS64_HOST_EFER_FULL:
5510 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5511 {
5512 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5513 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5514 break;
5515 }
5516
5517 /*
5518 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5519 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5520 */
5521 /* Natural-width Guest-state fields. */
5522 case VMX_VMCS_GUEST_CR3:
5523 case VMX_VMCS_GUEST_ES_BASE:
5524 case VMX_VMCS_GUEST_CS_BASE:
5525 case VMX_VMCS_GUEST_SS_BASE:
5526 case VMX_VMCS_GUEST_DS_BASE:
5527 case VMX_VMCS_GUEST_FS_BASE:
5528 case VMX_VMCS_GUEST_GS_BASE:
5529 case VMX_VMCS_GUEST_LDTR_BASE:
5530 case VMX_VMCS_GUEST_TR_BASE:
5531 case VMX_VMCS_GUEST_GDTR_BASE:
5532 case VMX_VMCS_GUEST_IDTR_BASE:
5533 case VMX_VMCS_GUEST_RSP:
5534 case VMX_VMCS_GUEST_RIP:
5535 case VMX_VMCS_GUEST_SYSENTER_ESP:
5536 case VMX_VMCS_GUEST_SYSENTER_EIP:
5537 {
5538 if (!(RT_HI_U32(u64Val)))
5539 {
5540 /* If this field is 64-bit, VT-x will zero out the top bits. */
5541 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5542 }
5543 else
5544 {
5545 /* Assert that only the 32->64 switcher case should ever come here. */
5546 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5547 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5548 }
5549 break;
5550 }
5551
5552 default:
5553 {
5554 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5555 rc = VERR_INVALID_PARAMETER;
5556 break;
5557 }
5558 }
5559 AssertRCReturn(rc, rc);
5560 return rc;
5561}
5562
5563
5564/**
5565 * Queue up a VMWRITE by using the VMCS write cache.
5566 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5567 *
5568 * @param pVCpu The cross context virtual CPU structure.
5569 * @param idxField The VMCS field encoding.
5570 * @param u64Val 16, 32 or 64-bit value.
5571 */
5572VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5573{
5574 AssertPtr(pVCpu);
5575 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5576
5577 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5578 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5579
5580 /* Make sure there are no duplicates. */
5581 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5582 {
5583 if (pCache->Write.aField[i] == idxField)
5584 {
5585 pCache->Write.aFieldVal[i] = u64Val;
5586 return VINF_SUCCESS;
5587 }
5588 }
5589
5590 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5591 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5592 pCache->Write.cValidEntries++;
5593 return VINF_SUCCESS;
5594}
5595#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5596
5597
5598/**
5599 * Sets up the usage of TSC-offsetting and updates the VMCS.
5600 *
5601 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5602 * VMX preemption timer.
5603 *
5604 * @returns VBox status code.
5605 * @param pVCpu The cross context virtual CPU structure.
5606 *
5607 * @remarks No-long-jump zone!!!
5608 */
5609static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5610{
5611 bool fOffsettedTsc;
5612 bool fParavirtTsc;
5613 PVM pVM = pVCpu->CTX_SUFF(pVM);
5614 uint64_t uTscOffset;
5615 if (pVM->hm.s.vmx.fUsePreemptTimer)
5616 {
5617 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
5618
5619 /* Make sure the returned values have sane upper and lower boundaries. */
5620 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5621 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5622 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5623 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5624
5625 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5626 int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
5627 AssertRC(rc);
5628 }
5629 else
5630 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
5631
5632 if (fParavirtTsc)
5633 {
5634 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5635 information before every VM-entry, hence disable it for performance sake. */
5636#if 0
5637 int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5638 AssertRC(rc);
5639#endif
5640 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5641 }
5642
5643 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
5644 if ( fOffsettedTsc
5645 && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5646 {
5647 if (pVCpu->hm.s.vmx.u64TscOffset != uTscOffset)
5648 {
5649 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
5650 AssertRC(rc);
5651 pVCpu->hm.s.vmx.u64TscOffset = uTscOffset;
5652 }
5653
5654 if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
5655 {
5656 uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
5657 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5658 AssertRC(rc);
5659 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5660 }
5661 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5662 }
5663 else
5664 {
5665 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5666 if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
5667 {
5668 uProcCtls |= VMX_PROC_CTLS_RDTSC_EXIT;
5669 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5670 AssertRC(rc);
5671 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5672 }
5673 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5674 }
5675}
5676
5677
5678/**
5679 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5680 * VM-exit interruption info type.
5681 *
5682 * @returns The IEM exception flags.
5683 * @param uVector The event vector.
5684 * @param uVmxVectorType The VMX event type.
5685 *
5686 * @remarks This function currently only constructs flags required for
5687 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5688 * and CR2 aspects of an exception are not included).
5689 */
5690static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5691{
5692 uint32_t fIemXcptFlags;
5693 switch (uVmxVectorType)
5694 {
5695 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5696 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5697 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5698 break;
5699
5700 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5701 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5702 break;
5703
5704 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5705 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5706 break;
5707
5708 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5709 {
5710 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5711 if (uVector == X86_XCPT_BP)
5712 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5713 else if (uVector == X86_XCPT_OF)
5714 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5715 else
5716 {
5717 fIemXcptFlags = 0;
5718 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5719 }
5720 break;
5721 }
5722
5723 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5724 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5725 break;
5726
5727 default:
5728 fIemXcptFlags = 0;
5729 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5730 break;
5731 }
5732 return fIemXcptFlags;
5733}
5734
5735
5736/**
5737 * Sets an event as a pending event to be injected into the guest.
5738 *
5739 * @param pVCpu The cross context virtual CPU structure.
5740 * @param u32IntInfo The VM-entry interruption-information field.
5741 * @param cbInstr The VM-entry instruction length in bytes (for software
5742 * interrupts, exceptions and privileged software
5743 * exceptions).
5744 * @param u32ErrCode The VM-entry exception error code.
5745 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5746 * page-fault.
5747 *
5748 * @remarks Statistics counter assumes this is a guest event being injected or
5749 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5750 * always incremented.
5751 */
5752DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5753 RTGCUINTPTR GCPtrFaultAddress)
5754{
5755 Assert(!pVCpu->hm.s.Event.fPending);
5756 pVCpu->hm.s.Event.fPending = true;
5757 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5758 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5759 pVCpu->hm.s.Event.cbInstr = cbInstr;
5760 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5761}
5762
5763
5764/**
5765 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5766 *
5767 * @param pVCpu The cross context virtual CPU structure.
5768 */
5769DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
5770{
5771 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
5772 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5773 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
5774 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5775 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5776}
5777
5778
5779/**
5780 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
5781 *
5782 * @param pVCpu The cross context virtual CPU structure.
5783 */
5784DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
5785{
5786 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
5787 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5788 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
5789 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5790 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5791}
5792
5793
5794/**
5795 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
5796 *
5797 * @param pVCpu The cross context virtual CPU structure.
5798 */
5799DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
5800{
5801 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
5802 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5803 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
5804 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5805 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5806}
5807
5808
5809#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5810/**
5811 * Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
5812 *
5813 * @param pVCpu The cross context virtual CPU structure.
5814 * @param u32ErrCode The error code for the general-protection exception.
5815 */
5816DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
5817{
5818 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
5819 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5820 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
5821 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5822 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrCode, 0 /* GCPtrFaultAddress */);
5823}
5824
5825
5826/**
5827 * Sets a stack (\#SS) exception as pending-for-injection into the VM.
5828 *
5829 * @param pVCpu The cross context virtual CPU structure.
5830 * @param u32ErrCode The error code for the stack exception.
5831 */
5832DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
5833{
5834 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
5835 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5836 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
5837 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5838 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrCode, 0 /* GCPtrFaultAddress */);
5839}
5840
5841
5842/**
5843 * Decodes the memory operand of a VM-exit due to instruction execution.
5844 *
5845 * For instructions with two operands, the second operand is usually found in the
5846 * VM-exit qualification field.
5847 *
5848 * @returns Strict VBox status code (i.e. informational status codes too).
5849 * @retval VINF_SUCCESS if the operand was successfully decoded.
5850 * @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
5851 * operand.
5852 * @param pVCpu The cross context virtual CPU structure.
5853 * @param pExitInstrInfo Pointer to the VM-exit instruction information.
5854 * @param fIsWrite Whether the operand is a destination memory operand
5855 * (i.e. writeable memory location) or not.
5856 * @param GCPtrDisp The instruction displacement field, if any. For
5857 * RIP-relative addressing pass RIP + displacement here.
5858 * @param pGCPtrMem Where to store the destination memory operand.
5859 */
5860static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, PCVMXEXITINSTRINFO pExitInstrInfo, RTGCPTR GCPtrDisp, bool fIsWrite,
5861 PRTGCPTR pGCPtrMem)
5862{
5863 Assert(pExitInstrInfo);
5864 Assert(pGCPtrMem);
5865 Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
5866
5867 static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
5868 static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
5869 AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
5870
5871 uint8_t const uAddrSize = pExitInstrInfo->All.u3AddrSize;
5872 uint8_t const iSegReg = pExitInstrInfo->All.iSegReg;
5873 bool const fIdxRegValid = !pExitInstrInfo->All.fIdxRegInvalid;
5874 uint8_t const iIdxReg = pExitInstrInfo->All.iIdxReg;
5875 uint8_t const uScale = pExitInstrInfo->All.u2Scaling;
5876 bool const fBaseRegValid = !pExitInstrInfo->All.fBaseRegInvalid;
5877 uint8_t const iBaseReg = pExitInstrInfo->All.iBaseReg;
5878 bool const fIsMemOperand = !pExitInstrInfo->All.fIsRegOperand;
5879 bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
5880
5881 /*
5882 * Validate instruction information.
5883 * This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
5884 */
5885 AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
5886 ("Invalid address size. ExitInstrInfo=%#RX32\n", pExitInstrInfo->u), VERR_VMX_IPE_1);
5887 AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
5888 ("Invalid segment register. ExitInstrInfo=%#RX32\n", pExitInstrInfo->u), VERR_VMX_IPE_2);
5889 AssertLogRelMsgReturn(fIsMemOperand,
5890 ("Expected memory operand. ExitInstrInfo=%#RX32\n", pExitInstrInfo->u), VERR_VMX_IPE_3);
5891
5892 /*
5893 * Compute the complete effective address.
5894 *
5895 * See AMD instruction spec. 1.4.2 "SIB Byte Format"
5896 * See AMD spec. 4.5.2 "Segment Registers".
5897 */
5898 RTGCPTR GCPtrMem = GCPtrDisp;
5899 if (fBaseRegValid)
5900 GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
5901 if (fIdxRegValid)
5902 GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
5903
5904 RTGCPTR const GCPtrOff = GCPtrMem;
5905 if ( !fIsLongMode
5906 || iSegReg >= X86_SREG_FS)
5907 GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
5908 GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
5909
5910 /*
5911 * Validate effective address.
5912 * See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
5913 */
5914 uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
5915 Assert(cbAccess > 0);
5916 if (fIsLongMode)
5917 {
5918 if (X86_IS_CANONICAL(GCPtrMem))
5919 {
5920 *pGCPtrMem = GCPtrMem;
5921 return VINF_SUCCESS;
5922 }
5923
5924 /** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
5925 * "Data Limit Checks in 64-bit Mode". */
5926 Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
5927 hmR0VmxSetPendingXcptGP(pVCpu, 0);
5928 return VINF_HM_PENDING_XCPT;
5929 }
5930
5931 /*
5932 * This is a watered down version of iemMemApplySegment().
5933 * Parts that are not applicable for VMX instructions like real-or-v8086 mode
5934 * and segment CPL/DPL checks are skipped.
5935 */
5936 RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
5937 RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
5938 PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
5939
5940 /* Check if the segment is present and usable. */
5941 if ( pSel->Attr.n.u1Present
5942 && !pSel->Attr.n.u1Unusable)
5943 {
5944 Assert(pSel->Attr.n.u1DescType);
5945 if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
5946 {
5947 /* Check permissions for the data segment. */
5948 if ( fIsWrite
5949 && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
5950 {
5951 Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
5952 hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
5953 return VINF_HM_PENDING_XCPT;
5954 }
5955
5956 /* Check limits if it's a normal data segment. */
5957 if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
5958 {
5959 if ( GCPtrFirst32 > pSel->u32Limit
5960 || GCPtrLast32 > pSel->u32Limit)
5961 {
5962 Log4Func(("Data segment limit exceeded."
5963 "iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
5964 GCPtrLast32, pSel->u32Limit));
5965 if (iSegReg == X86_SREG_SS)
5966 hmR0VmxSetPendingXcptSS(pVCpu, 0);
5967 else
5968 hmR0VmxSetPendingXcptGP(pVCpu, 0);
5969 return VINF_HM_PENDING_XCPT;
5970 }
5971 }
5972 else
5973 {
5974 /* Check limits if it's an expand-down data segment.
5975 Note! The upper boundary is defined by the B bit, not the G bit! */
5976 if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
5977 || GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
5978 {
5979 Log4Func(("Expand-down data segment limit exceeded."
5980 "iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
5981 GCPtrLast32, pSel->u32Limit));
5982 if (iSegReg == X86_SREG_SS)
5983 hmR0VmxSetPendingXcptSS(pVCpu, 0);
5984 else
5985 hmR0VmxSetPendingXcptGP(pVCpu, 0);
5986 return VINF_HM_PENDING_XCPT;
5987 }
5988 }
5989 }
5990 else
5991 {
5992 /* Check permissions for the code segment. */
5993 if ( fIsWrite
5994 || !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ))
5995 {
5996 Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
5997 Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
5998 hmR0VmxSetPendingXcptGP(pVCpu, 0);
5999 return VINF_HM_PENDING_XCPT;
6000 }
6001
6002 /* Check limits for the code segment (normal/expand-down not applicable for code segments). */
6003 if ( GCPtrFirst32 > pSel->u32Limit
6004 || GCPtrLast32 > pSel->u32Limit)
6005 {
6006 Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
6007 GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
6008 if (iSegReg == X86_SREG_SS)
6009 hmR0VmxSetPendingXcptSS(pVCpu, 0);
6010 else
6011 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6012 return VINF_HM_PENDING_XCPT;
6013 }
6014 }
6015 }
6016 else
6017 {
6018 Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6019 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6020 return VINF_HM_PENDING_XCPT;
6021 }
6022
6023 *pGCPtrMem = GCPtrMem;
6024 return VINF_SUCCESS;
6025}
6026
6027
6028/**
6029 * Perform the relevant VMX instruction checks for VM-exits that occurred due to the
6030 * guest attempting to execute a VMX instruction.
6031 *
6032 * @returns Strict VBox status code (i.e. informational status codes too).
6033 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
6034 * @retval VINF_HM_PENDING_XCPT if an exception was raised.
6035 *
6036 * @param pVCpu The cross context virtual CPU structure.
6037 * @param pVmxTransient Pointer to the VMX transient structure.
6038 *
6039 * @todo NstVmx: Document other error codes when VM-exit is implemented.
6040 * @remarks No-long-jump zone!!!
6041 */
6042static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6043{
6044 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS
6045 | CPUMCTX_EXTRN_HWVIRT);
6046
6047 if ( CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx)
6048 || ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6049 && !CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx)))
6050 {
6051 Log4Func(("In real/v86-mode or long-mode outside 64-bit code segment -> #UD\n"));
6052 hmR0VmxSetPendingXcptUD(pVCpu);
6053 return VINF_HM_PENDING_XCPT;
6054 }
6055
6056 if (pVmxTransient->uExitReason == VMX_EXIT_VMXON)
6057 {
6058 /*
6059 * We check CR4.VMXE because it is required to be always set while in VMX operation
6060 * by physical CPUs and our CR4 read shadow is only consulted when executing specific
6061 * instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
6062 * otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
6063 */
6064 if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
6065 {
6066 Log4Func(("CR4.VMXE is not set -> #UD\n"));
6067 hmR0VmxSetPendingXcptUD(pVCpu);
6068 return VINF_HM_PENDING_XCPT;
6069 }
6070 }
6071 else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
6072 {
6073 /*
6074 * The guest has not entered VMX operation but attempted to execute a VMX instruction
6075 * (other than VMXON), we need to raise a #UD.
6076 */
6077 Log4Func(("Not in VMX root mode -> #UD\n"));
6078 hmR0VmxSetPendingXcptUD(pVCpu);
6079 return VINF_HM_PENDING_XCPT;
6080 }
6081
6082 if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
6083 {
6084 /*
6085 * The nested-guest attempted to execute a VMX instruction, cause a VM-exit and let
6086 * the guest hypervisor deal with it.
6087 */
6088 /** @todo NSTVMX: Trigger a VM-exit */
6089 }
6090
6091 /*
6092 * VMX instructions require CPL 0 except in VMX non-root mode where the VM-exit intercept
6093 * (above) takes preceedence over the CPL check.
6094 */
6095 if (CPUMGetGuestCPL(pVCpu) > 0)
6096 {
6097 Log4Func(("CPL > 0 -> #GP(0)\n"));
6098 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6099 return VINF_HM_PENDING_XCPT;
6100 }
6101
6102 return VINF_SUCCESS;
6103}
6104
6105#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
6106
6107
6108/**
6109 * Handle a condition that occurred while delivering an event through the guest
6110 * IDT.
6111 *
6112 * @returns Strict VBox status code (i.e. informational status codes too).
6113 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
6114 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
6115 * to continue execution of the guest which will delivery the \#DF.
6116 * @retval VINF_EM_RESET if we detected a triple-fault condition.
6117 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
6118 *
6119 * @param pVCpu The cross context virtual CPU structure.
6120 * @param pVmxTransient Pointer to the VMX transient structure.
6121 *
6122 * @remarks No-long-jump zone!!!
6123 */
6124static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6125{
6126 uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
6127
6128 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
6129 rc2 |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
6130 AssertRCReturn(rc2, rc2);
6131
6132 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
6133 if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6134 {
6135 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
6136 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
6137
6138 /*
6139 * If the event was a software interrupt (generated with INT n) or a software exception
6140 * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
6141 * can handle the VM-exit and continue guest execution which will re-execute the
6142 * instruction rather than re-injecting the exception, as that can cause premature
6143 * trips to ring-3 before injection and involve TRPM which currently has no way of
6144 * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
6145 * the problem).
6146 */
6147 IEMXCPTRAISE enmRaise;
6148 IEMXCPTRAISEINFO fRaiseInfo;
6149 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6150 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6151 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6152 {
6153 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
6154 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6155 }
6156 else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
6157 {
6158 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
6159 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
6160 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
6161 /** @todo Make AssertMsgReturn as just AssertMsg later. */
6162 AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
6163 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
6164 uExitVectorType), VERR_VMX_IPE_5);
6165
6166 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
6167
6168 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
6169 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
6170 {
6171 pVmxTransient->fVectoringPF = true;
6172 enmRaise = IEMXCPTRAISE_PREV_EVENT;
6173 }
6174 }
6175 else
6176 {
6177 /*
6178 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
6179 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
6180 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
6181 */
6182 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6183 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6184 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
6185 enmRaise = IEMXCPTRAISE_PREV_EVENT;
6186 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6187 }
6188
6189 /*
6190 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
6191 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
6192 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
6193 * subsequent VM-entry would fail.
6194 *
6195 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
6196 */
6197 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
6198 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6199 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
6200 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
6201 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
6202 {
6203 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6204 }
6205
6206 switch (enmRaise)
6207 {
6208 case IEMXCPTRAISE_CURRENT_XCPT:
6209 {
6210 Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
6211 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
6212 Assert(rcStrict == VINF_SUCCESS);
6213 break;
6214 }
6215
6216 case IEMXCPTRAISE_PREV_EVENT:
6217 {
6218 uint32_t u32ErrCode;
6219 if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
6220 {
6221 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6222 AssertRCReturn(rc2, rc2);
6223 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6224 }
6225 else
6226 u32ErrCode = 0;
6227
6228 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
6229 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6230 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6231 0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
6232
6233 Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
6234 pVCpu->hm.s.Event.u32ErrCode));
6235 Assert(rcStrict == VINF_SUCCESS);
6236 break;
6237 }
6238
6239 case IEMXCPTRAISE_REEXEC_INSTR:
6240 Assert(rcStrict == VINF_SUCCESS);
6241 break;
6242
6243 case IEMXCPTRAISE_DOUBLE_FAULT:
6244 {
6245 /*
6246 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
6247 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
6248 */
6249 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6250 {
6251 pVmxTransient->fVectoringDoublePF = true;
6252 Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
6253 pVCpu->cpum.GstCtx.cr2));
6254 rcStrict = VINF_SUCCESS;
6255 }
6256 else
6257 {
6258 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6259 hmR0VmxSetPendingXcptDF(pVCpu);
6260 Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
6261 uIdtVector, uExitVector));
6262 rcStrict = VINF_HM_DOUBLE_FAULT;
6263 }
6264 break;
6265 }
6266
6267 case IEMXCPTRAISE_TRIPLE_FAULT:
6268 {
6269 Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
6270 rcStrict = VINF_EM_RESET;
6271 break;
6272 }
6273
6274 case IEMXCPTRAISE_CPU_HANG:
6275 {
6276 Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
6277 rcStrict = VERR_EM_GUEST_CPU_HANG;
6278 break;
6279 }
6280
6281 default:
6282 {
6283 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6284 rcStrict = VERR_VMX_IPE_2;
6285 break;
6286 }
6287 }
6288 }
6289 else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6290 && VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6291 && uExitVector != X86_XCPT_DF
6292 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
6293 {
6294 /*
6295 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6296 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6297 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6298 */
6299 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6300 {
6301 Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
6302 VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6303 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6304 }
6305 }
6306
6307 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
6308 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
6309 return rcStrict;
6310}
6311
6312
6313/**
6314 * Imports a guest segment register from the current VMCS into
6315 * the guest-CPU context.
6316 *
6317 * @returns VBox status code.
6318 * @param pVCpu The cross context virtual CPU structure.
6319 * @param idxSel Index of the selector in the VMCS.
6320 * @param idxLimit Index of the segment limit in the VMCS.
6321 * @param idxBase Index of the segment base in the VMCS.
6322 * @param idxAccess Index of the access rights of the segment in the VMCS.
6323 * @param pSelReg Pointer to the segment selector.
6324 *
6325 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6326 * do not log!
6327 *
6328 * @remarks Never call this function directly!!! Use the
6329 * HMVMX_IMPORT_SREG() macro as that takes care
6330 * of whether to read from the VMCS cache or not.
6331 */
6332static int hmR0VmxImportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6333 PCPUMSELREG pSelReg)
6334{
6335 NOREF(pVCpu);
6336
6337 uint32_t u32Sel;
6338 uint32_t u32Limit;
6339 uint32_t u32Attr;
6340 uint64_t u64Base;
6341 int rc = VMXReadVmcs32(idxSel, &u32Sel);
6342 rc |= VMXReadVmcs32(idxLimit, &u32Limit);
6343 rc |= VMXReadVmcs32(idxAccess, &u32Attr);
6344 rc |= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
6345 AssertRCReturn(rc, rc);
6346
6347 pSelReg->Sel = (uint16_t)u32Sel;
6348 pSelReg->ValidSel = (uint16_t)u32Sel;
6349 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6350 pSelReg->u32Limit = u32Limit;
6351 pSelReg->u64Base = u64Base;
6352 pSelReg->Attr.u = u32Attr;
6353
6354 /*
6355 * If VT-x marks the segment as unusable, most other bits remain undefined:
6356 * - For CS the L, D and G bits have meaning.
6357 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6358 * - For the remaining data segments no bits are defined.
6359 *
6360 * The present bit and the unusable bit has been observed to be set at the
6361 * same time (the selector was supposed to be invalid as we started executing
6362 * a V8086 interrupt in ring-0).
6363 *
6364 * What should be important for the rest of the VBox code, is that the P bit is
6365 * cleared. Some of the other VBox code recognizes the unusable bit, but
6366 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6367 * safe side here, we'll strip off P and other bits we don't care about. If
6368 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6369 *
6370 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6371 */
6372 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6373 {
6374 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6375
6376 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6377 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6378 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6379#ifdef VBOX_STRICT
6380 VMMRZCallRing3Disable(pVCpu);
6381 Log4Func(("Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Sel, pSelReg->Attr.u));
6382# ifdef DEBUG_bird
6383 AssertMsg((u32Attr & ~X86DESCATTR_P) == pSelReg->Attr.u,
6384 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6385 idxSel, u32Sel, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6386# endif
6387 VMMRZCallRing3Enable(pVCpu);
6388#endif
6389 }
6390 return VINF_SUCCESS;
6391}
6392
6393
6394/**
6395 * Imports the guest RIP from the VMCS back into the guest-CPU context.
6396 *
6397 * @returns VBox status code.
6398 * @param pVCpu The cross context virtual CPU structure.
6399 *
6400 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6401 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6402 * instead!!!
6403 */
6404DECLINLINE(int) hmR0VmxImportGuestRip(PVMCPU pVCpu)
6405{
6406 uint64_t u64Val;
6407 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6408 if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
6409 {
6410 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6411 if (RT_SUCCESS(rc))
6412 {
6413 pCtx->rip = u64Val;
6414 EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
6415 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
6416 }
6417 return rc;
6418 }
6419 return VINF_SUCCESS;
6420}
6421
6422
6423/**
6424 * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
6425 *
6426 * @returns VBox status code.
6427 * @param pVCpu The cross context virtual CPU structure.
6428 *
6429 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6430 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6431 * instead!!!
6432 */
6433DECLINLINE(int) hmR0VmxImportGuestRFlags(PVMCPU pVCpu)
6434{
6435 uint32_t u32Val;
6436 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6437 if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
6438 {
6439 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
6440 if (RT_SUCCESS(rc))
6441 {
6442 pCtx->eflags.u32 = u32Val;
6443
6444 /* Restore eflags for real-on-v86-mode hack. */
6445 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6446 {
6447 pCtx->eflags.Bits.u1VM = 0;
6448 pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6449 }
6450 }
6451 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
6452 return rc;
6453 }
6454 return VINF_SUCCESS;
6455}
6456
6457
6458/**
6459 * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
6460 * context.
6461 *
6462 * @returns VBox status code.
6463 * @param pVCpu The cross context virtual CPU structure.
6464 *
6465 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6466 * do not log!
6467 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6468 * instead!!!
6469 */
6470DECLINLINE(int) hmR0VmxImportGuestIntrState(PVMCPU pVCpu)
6471{
6472 uint32_t u32Val;
6473 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6474 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
6475 if (RT_SUCCESS(rc))
6476 {
6477 /*
6478 * We additionally have a requirement to import RIP, RFLAGS depending on whether we
6479 * might need them in hmR0VmxEvaluatePendingEvent().
6480 */
6481 if (!u32Val)
6482 {
6483 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6484 {
6485 rc = hmR0VmxImportGuestRip(pVCpu);
6486 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6487 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6488 }
6489
6490 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6491 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6492 }
6493 else
6494 {
6495 rc = hmR0VmxImportGuestRip(pVCpu);
6496 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6497
6498 if (u32Val & ( VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
6499 | VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
6500 {
6501 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
6502 }
6503 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6504 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6505
6506 if (u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
6507 {
6508 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6509 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6510 }
6511 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6512 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6513 }
6514 }
6515 return rc;
6516}
6517
6518
6519/**
6520 * Worker for VMXR0ImportStateOnDemand.
6521 *
6522 * @returns VBox status code.
6523 * @param pVCpu The cross context virtual CPU structure.
6524 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6525 */
6526static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
6527{
6528#define VMXLOCAL_BREAK_RC(a_rc) \
6529 if (RT_FAILURE(a_rc)) \
6530 break
6531
6532 int rc = VINF_SUCCESS;
6533 PVM pVM = pVCpu->CTX_SUFF(pVM);
6534 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6535 uint64_t u64Val;
6536 uint32_t u32Val;
6537
6538 Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
6539 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
6540
6541 /*
6542 * We disable interrupts to make the updating of the state and in particular
6543 * the fExtrn modification atomic wrt to preemption hooks.
6544 */
6545 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
6546
6547 fWhat &= pCtx->fExtrn;
6548 if (fWhat)
6549 {
6550 do
6551 {
6552 if (fWhat & CPUMCTX_EXTRN_RIP)
6553 {
6554 rc = hmR0VmxImportGuestRip(pVCpu);
6555 VMXLOCAL_BREAK_RC(rc);
6556 }
6557
6558 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
6559 {
6560 rc = hmR0VmxImportGuestRFlags(pVCpu);
6561 VMXLOCAL_BREAK_RC(rc);
6562 }
6563
6564 if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
6565 {
6566 rc = hmR0VmxImportGuestIntrState(pVCpu);
6567 VMXLOCAL_BREAK_RC(rc);
6568 }
6569
6570 if (fWhat & CPUMCTX_EXTRN_RSP)
6571 {
6572 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6573 VMXLOCAL_BREAK_RC(rc);
6574 pCtx->rsp = u64Val;
6575 }
6576
6577 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
6578 {
6579 if (fWhat & CPUMCTX_EXTRN_CS)
6580 {
6581 rc = HMVMX_IMPORT_SREG(CS, &pCtx->cs);
6582 rc |= hmR0VmxImportGuestRip(pVCpu);
6583 VMXLOCAL_BREAK_RC(rc);
6584 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6585 pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6586 EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true);
6587 }
6588 if (fWhat & CPUMCTX_EXTRN_SS)
6589 {
6590 rc = HMVMX_IMPORT_SREG(SS, &pCtx->ss);
6591 VMXLOCAL_BREAK_RC(rc);
6592 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6593 pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6594 }
6595 if (fWhat & CPUMCTX_EXTRN_DS)
6596 {
6597 rc = HMVMX_IMPORT_SREG(DS, &pCtx->ds);
6598 VMXLOCAL_BREAK_RC(rc);
6599 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6600 pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6601 }
6602 if (fWhat & CPUMCTX_EXTRN_ES)
6603 {
6604 rc = HMVMX_IMPORT_SREG(ES, &pCtx->es);
6605 VMXLOCAL_BREAK_RC(rc);
6606 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6607 pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6608 }
6609 if (fWhat & CPUMCTX_EXTRN_FS)
6610 {
6611 rc = HMVMX_IMPORT_SREG(FS, &pCtx->fs);
6612 VMXLOCAL_BREAK_RC(rc);
6613 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6614 pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6615 }
6616 if (fWhat & CPUMCTX_EXTRN_GS)
6617 {
6618 rc = HMVMX_IMPORT_SREG(GS, &pCtx->gs);
6619 VMXLOCAL_BREAK_RC(rc);
6620 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6621 pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6622 }
6623 }
6624
6625 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
6626 {
6627 if (fWhat & CPUMCTX_EXTRN_LDTR)
6628 {
6629 rc = HMVMX_IMPORT_SREG(LDTR, &pCtx->ldtr);
6630 VMXLOCAL_BREAK_RC(rc);
6631 }
6632
6633 if (fWhat & CPUMCTX_EXTRN_GDTR)
6634 {
6635 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6636 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
6637 VMXLOCAL_BREAK_RC(rc);
6638 pCtx->gdtr.pGdt = u64Val;
6639 pCtx->gdtr.cbGdt = u32Val;
6640 }
6641
6642 /* Guest IDTR. */
6643 if (fWhat & CPUMCTX_EXTRN_IDTR)
6644 {
6645 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6646 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
6647 VMXLOCAL_BREAK_RC(rc);
6648 pCtx->idtr.pIdt = u64Val;
6649 pCtx->idtr.cbIdt = u32Val;
6650 }
6651
6652 /* Guest TR. */
6653 if (fWhat & CPUMCTX_EXTRN_TR)
6654 {
6655 /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
6656 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6657 {
6658 rc = HMVMX_IMPORT_SREG(TR, &pCtx->tr);
6659 VMXLOCAL_BREAK_RC(rc);
6660 }
6661 }
6662 }
6663
6664 if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
6665 {
6666 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
6667 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
6668 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
6669 pCtx->SysEnter.cs = u32Val;
6670 VMXLOCAL_BREAK_RC(rc);
6671 }
6672
6673#if HC_ARCH_BITS == 64
6674 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
6675 {
6676 if ( pVM->hm.s.fAllow64BitGuests
6677 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6678 pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
6679 }
6680
6681 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
6682 {
6683 if ( pVM->hm.s.fAllow64BitGuests
6684 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6685 {
6686 pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
6687 pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
6688 pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
6689 }
6690 }
6691#endif
6692
6693 if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
6694#if HC_ARCH_BITS == 32
6695 || (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS))
6696#endif
6697 )
6698 {
6699 PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6700 uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
6701 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6702 {
6703 switch (pMsr->u32Msr)
6704 {
6705#if HC_ARCH_BITS == 32
6706 case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
6707 case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
6708 case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
6709 case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6710#endif
6711 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6712 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6713 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit */ break;
6714 default:
6715 {
6716 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6717 ASMSetFlags(fEFlags);
6718 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
6719 cMsrs));
6720 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6721 }
6722 }
6723 }
6724 }
6725
6726 if (fWhat & CPUMCTX_EXTRN_DR7)
6727 {
6728 if (!pVCpu->hm.s.fUsingHyperDR7)
6729 {
6730 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6731 rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
6732 VMXLOCAL_BREAK_RC(rc);
6733 pCtx->dr[7] = u32Val;
6734 }
6735 }
6736
6737 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
6738 {
6739 uint32_t u32Shadow;
6740 if (fWhat & CPUMCTX_EXTRN_CR0)
6741 {
6742 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
6743 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
6744 VMXLOCAL_BREAK_RC(rc);
6745 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr0Mask)
6746 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr0Mask);
6747 VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
6748 CPUMSetGuestCR0(pVCpu, u32Val);
6749 VMMRZCallRing3Enable(pVCpu);
6750 }
6751
6752 if (fWhat & CPUMCTX_EXTRN_CR4)
6753 {
6754 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
6755 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
6756 VMXLOCAL_BREAK_RC(rc);
6757 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr4Mask)
6758 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr4Mask);
6759 CPUMSetGuestCR4(pVCpu, u32Val);
6760 }
6761
6762 if (fWhat & CPUMCTX_EXTRN_CR3)
6763 {
6764 /* CR0.PG bit changes are always intercepted, so it's up to date. */
6765 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6766 || ( pVM->hm.s.fNestedPaging
6767 && CPUMIsGuestPagingEnabledEx(pCtx)))
6768 {
6769 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6770 if (pCtx->cr3 != u64Val)
6771 {
6772 CPUMSetGuestCR3(pVCpu, u64Val);
6773 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6774 }
6775
6776 /* If the guest is in PAE mode, sync back the PDPE's into the guest state.
6777 Note: CR4.PAE, CR0.PG, EFER bit changes are always intercepted, so they're up to date. */
6778 if (CPUMIsGuestInPAEModeEx(pCtx))
6779 {
6780 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6781 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6782 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6783 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6784 VMXLOCAL_BREAK_RC(rc);
6785 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6786 }
6787 }
6788 }
6789 }
6790 } while (0);
6791
6792 if (RT_SUCCESS(rc))
6793 {
6794 /* Update fExtrn. */
6795 pCtx->fExtrn &= ~fWhat;
6796
6797 /* If everything has been imported, clear the HM keeper bit. */
6798 if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
6799 {
6800 pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
6801 Assert(!pCtx->fExtrn);
6802 }
6803 }
6804 }
6805 else
6806 AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
6807
6808 ASMSetFlags(fEFlags);
6809
6810 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
6811
6812 /*
6813 * Honor any pending CR3 updates.
6814 *
6815 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6816 * -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6817 * -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
6818 *
6819 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6820 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6821 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6822 * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
6823 *
6824 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6825 */
6826 if (VMMRZCallRing3IsEnabled(pVCpu))
6827 {
6828 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6829 {
6830 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
6831 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6832 }
6833
6834 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6835 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6836
6837 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6838 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6839 }
6840
6841 return VINF_SUCCESS;
6842#undef VMXLOCAL_BREAK_RC
6843}
6844
6845
6846/**
6847 * Saves the guest state from the VMCS into the guest-CPU context.
6848 *
6849 * @returns VBox status code.
6850 * @param pVCpu The cross context virtual CPU structure.
6851 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6852 */
6853VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
6854{
6855 return hmR0VmxImportGuestState(pVCpu, fWhat);
6856}
6857
6858
6859/**
6860 * Check per-VM and per-VCPU force flag actions that require us to go back to
6861 * ring-3 for one reason or another.
6862 *
6863 * @returns Strict VBox status code (i.e. informational status codes too)
6864 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6865 * ring-3.
6866 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6867 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6868 * interrupts)
6869 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6870 * all EMTs to be in ring-3.
6871 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6872 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6873 * to the EM loop.
6874 *
6875 * @param pVCpu The cross context virtual CPU structure.
6876 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6877 */
6878static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
6879{
6880 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6881
6882 /*
6883 * Anything pending? Should be more likely than not if we're doing a good job.
6884 */
6885 PVM pVM = pVCpu->CTX_SUFF(pVM);
6886 if ( !fStepping
6887 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6888 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6889 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6890 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6891 return VINF_SUCCESS;
6892
6893 /* Pending PGM C3 sync. */
6894 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6895 {
6896 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6897 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
6898 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
6899 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6900 if (rcStrict2 != VINF_SUCCESS)
6901 {
6902 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6903 Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6904 return rcStrict2;
6905 }
6906 }
6907
6908 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6909 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6910 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6911 {
6912 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6913 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6914 Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6915 return rc2;
6916 }
6917
6918 /* Pending VM request packets, such as hardware interrupts. */
6919 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6920 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6921 {
6922 Log4Func(("Pending VM request forcing us back to ring-3\n"));
6923 return VINF_EM_PENDING_REQUEST;
6924 }
6925
6926 /* Pending PGM pool flushes. */
6927 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6928 {
6929 Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
6930 return VINF_PGM_POOL_FLUSH_PENDING;
6931 }
6932
6933 /* Pending DMA requests. */
6934 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6935 {
6936 Log4Func(("Pending DMA request forcing us back to ring-3\n"));
6937 return VINF_EM_RAW_TO_R3;
6938 }
6939
6940 return VINF_SUCCESS;
6941}
6942
6943
6944/**
6945 * Converts any TRPM trap into a pending HM event. This is typically used when
6946 * entering from ring-3 (not longjmp returns).
6947 *
6948 * @param pVCpu The cross context virtual CPU structure.
6949 */
6950static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6951{
6952 Assert(TRPMHasTrap(pVCpu));
6953 Assert(!pVCpu->hm.s.Event.fPending);
6954
6955 uint8_t uVector;
6956 TRPMEVENT enmTrpmEvent;
6957 RTGCUINT uErrCode;
6958 RTGCUINTPTR GCPtrFaultAddress;
6959 uint8_t cbInstr;
6960
6961 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6962 AssertRC(rc);
6963
6964 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6965 uint32_t u32IntInfo = uVector | VMX_EXIT_INT_INFO_VALID;
6966 if (enmTrpmEvent == TRPM_TRAP)
6967 {
6968 switch (uVector)
6969 {
6970 case X86_XCPT_NMI:
6971 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6972 break;
6973
6974 case X86_XCPT_BP:
6975 case X86_XCPT_OF:
6976 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6977 break;
6978
6979 case X86_XCPT_PF:
6980 case X86_XCPT_DF:
6981 case X86_XCPT_TS:
6982 case X86_XCPT_NP:
6983 case X86_XCPT_SS:
6984 case X86_XCPT_GP:
6985 case X86_XCPT_AC:
6986 u32IntInfo |= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
6987 RT_FALL_THRU();
6988 default:
6989 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6990 break;
6991 }
6992 }
6993 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6994 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6995 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6996 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6997 else
6998 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6999
7000 rc = TRPMResetTrap(pVCpu);
7001 AssertRC(rc);
7002 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7003 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7004
7005 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7006}
7007
7008
7009/**
7010 * Converts the pending HM event into a TRPM trap.
7011 *
7012 * @param pVCpu The cross context virtual CPU structure.
7013 */
7014static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7015{
7016 Assert(pVCpu->hm.s.Event.fPending);
7017
7018 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7019 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7020 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
7021 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7022
7023 /* If a trap was already pending, we did something wrong! */
7024 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7025
7026 TRPMEVENT enmTrapType;
7027 switch (uVectorType)
7028 {
7029 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7030 enmTrapType = TRPM_HARDWARE_INT;
7031 break;
7032
7033 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7034 enmTrapType = TRPM_SOFTWARE_INT;
7035 break;
7036
7037 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7038 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7039 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7040 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7041 enmTrapType = TRPM_TRAP;
7042 break;
7043
7044 default:
7045 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7046 enmTrapType = TRPM_32BIT_HACK;
7047 break;
7048 }
7049
7050 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7051
7052 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7053 AssertRC(rc);
7054
7055 if (fErrorCodeValid)
7056 TRPMSetErrorCode(pVCpu, uErrorCode);
7057
7058 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7059 && uVector == X86_XCPT_PF)
7060 {
7061 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7062 }
7063 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7064 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7065 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7066 {
7067 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7068 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
7069 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7070 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7071 }
7072
7073 /* Clear any pending events from the VMCS. */
7074 VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7075 VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, 0);
7076
7077 /* We're now done converting the pending event. */
7078 pVCpu->hm.s.Event.fPending = false;
7079}
7080
7081
7082/**
7083 * Does the necessary state syncing before returning to ring-3 for any reason
7084 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7085 *
7086 * @returns VBox status code.
7087 * @param pVCpu The cross context virtual CPU structure.
7088 * @param fImportState Whether to import the guest state from the VMCS back
7089 * to the guest-CPU context.
7090 *
7091 * @remarks No-long-jmp zone!!!
7092 */
7093static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
7094{
7095 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7096 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7097
7098 RTCPUID idCpu = RTMpCpuId();
7099 Log4Func(("HostCpuId=%u\n", idCpu));
7100
7101 /*
7102 * !!! IMPORTANT !!!
7103 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7104 */
7105
7106 /* Save the guest state if necessary. */
7107 if (fImportState)
7108 {
7109 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
7110 AssertRCReturn(rc, rc);
7111 }
7112
7113 /* Restore host FPU state if necessary. We will resync on next R0 reentry. */
7114 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7115 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
7116
7117 /* Restore host debug registers if necessary. We will resync on next R0 reentry. */
7118#ifdef VBOX_STRICT
7119 if (CPUMIsHyperDebugStateActive(pVCpu))
7120 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
7121#endif
7122 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7123 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7124 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7125
7126#if HC_ARCH_BITS == 64
7127 /* Restore host-state bits that VT-x only restores partially. */
7128 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7129 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7130 {
7131 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7132 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7133 }
7134 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7135#endif
7136
7137 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7138 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7139 {
7140 /* We shouldn't restore the host MSRs without saving the guest MSRs first. */
7141 if (!fImportState)
7142 {
7143 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS);
7144 AssertRCReturn(rc, rc);
7145 }
7146 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7147 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7148 }
7149 else
7150 pVCpu->hm.s.vmx.fLazyMsrs = 0;
7151
7152 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7153 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7154
7155 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7156 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
7157 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
7158 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
7159 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
7160 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7161 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7162 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7163 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7164
7165 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7166
7167 /** @todo This partially defeats the purpose of having preemption hooks.
7168 * The problem is, deregistering the hooks should be moved to a place that
7169 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7170 * context.
7171 */
7172 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7173 {
7174 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7175 AssertRCReturn(rc, rc);
7176
7177 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7178 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7179 }
7180 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7181 NOREF(idCpu);
7182
7183 return VINF_SUCCESS;
7184}
7185
7186
7187/**
7188 * Leaves the VT-x session.
7189 *
7190 * @returns VBox status code.
7191 * @param pVCpu The cross context virtual CPU structure.
7192 *
7193 * @remarks No-long-jmp zone!!!
7194 */
7195static int hmR0VmxLeaveSession(PVMCPU pVCpu)
7196{
7197 HM_DISABLE_PREEMPT(pVCpu);
7198 HMVMX_ASSERT_CPU_SAFE(pVCpu);
7199 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7200 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7201
7202 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7203 and done this from the VMXR0ThreadCtxCallback(). */
7204 if (!pVCpu->hm.s.fLeaveDone)
7205 {
7206 int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
7207 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7208 pVCpu->hm.s.fLeaveDone = true;
7209 }
7210 Assert(!pVCpu->cpum.GstCtx.fExtrn);
7211
7212 /*
7213 * !!! IMPORTANT !!!
7214 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7215 */
7216
7217 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7218 /** @todo Deregistering here means we need to VMCLEAR always
7219 * (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
7220 * for calling VMMR0ThreadCtxHookDisable here! */
7221 VMMR0ThreadCtxHookDisable(pVCpu);
7222
7223 /* Leave HM context. This takes care of local init (term). */
7224 int rc = HMR0LeaveCpu(pVCpu);
7225
7226 HM_RESTORE_PREEMPT();
7227 return rc;
7228}
7229
7230
7231/**
7232 * Does the necessary state syncing before doing a longjmp to ring-3.
7233 *
7234 * @returns VBox status code.
7235 * @param pVCpu The cross context virtual CPU structure.
7236 *
7237 * @remarks No-long-jmp zone!!!
7238 */
7239DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
7240{
7241 return hmR0VmxLeaveSession(pVCpu);
7242}
7243
7244
7245/**
7246 * Take necessary actions before going back to ring-3.
7247 *
7248 * An action requires us to go back to ring-3. This function does the necessary
7249 * steps before we can safely return to ring-3. This is not the same as longjmps
7250 * to ring-3, this is voluntary and prepares the guest so it may continue
7251 * executing outside HM (recompiler/IEM).
7252 *
7253 * @returns VBox status code.
7254 * @param pVCpu The cross context virtual CPU structure.
7255 * @param rcExit The reason for exiting to ring-3. Can be
7256 * VINF_VMM_UNKNOWN_RING3_CALL.
7257 */
7258static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
7259{
7260 Assert(pVCpu);
7261 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7262
7263 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7264 {
7265 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7266 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7267 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7268 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7269 }
7270
7271 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7272 VMMRZCallRing3Disable(pVCpu);
7273 Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
7274
7275 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7276 if (pVCpu->hm.s.Event.fPending)
7277 {
7278 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7279 Assert(!pVCpu->hm.s.Event.fPending);
7280 }
7281
7282 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7283 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7284
7285 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7286 and if we're injecting an event we should have a TRPM trap pending. */
7287 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7288#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
7289 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7290#endif
7291
7292 /* Save guest state and restore host state bits. */
7293 int rc = hmR0VmxLeaveSession(pVCpu);
7294 AssertRCReturn(rc, rc);
7295 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7296 /* Thread-context hooks are unregistered at this point!!! */
7297
7298 /* Sync recompiler state. */
7299 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7300 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7301 | CPUM_CHANGED_LDTR
7302 | CPUM_CHANGED_GDTR
7303 | CPUM_CHANGED_IDTR
7304 | CPUM_CHANGED_TR
7305 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7306 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
7307 && CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
7308 {
7309 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7310 }
7311
7312 Assert(!pVCpu->hm.s.fClearTrapFlag);
7313
7314 /* Update the exit-to-ring 3 reason. */
7315 pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
7316
7317 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7318 if (rcExit != VINF_EM_RAW_INTERRUPT)
7319 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7320
7321 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7322
7323 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7324 VMMRZCallRing3RemoveNotification(pVCpu);
7325 VMMRZCallRing3Enable(pVCpu);
7326
7327 return rc;
7328}
7329
7330
7331/**
7332 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7333 * longjump to ring-3 and possibly get preempted.
7334 *
7335 * @returns VBox status code.
7336 * @param pVCpu The cross context virtual CPU structure.
7337 * @param enmOperation The operation causing the ring-3 longjump.
7338 * @param pvUser User argument, currently unused, NULL.
7339 */
7340static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7341{
7342 RT_NOREF(pvUser);
7343 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7344 {
7345 /*
7346 * !!! IMPORTANT !!!
7347 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7348 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7349 */
7350 VMMRZCallRing3RemoveNotification(pVCpu);
7351 VMMRZCallRing3Disable(pVCpu);
7352 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7353 RTThreadPreemptDisable(&PreemptState);
7354
7355 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
7356 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7357 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7358
7359#if HC_ARCH_BITS == 64
7360 /* Restore host-state bits that VT-x only restores partially. */
7361 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7362 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7363 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7364 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7365#endif
7366
7367 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7368 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7369 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7370
7371 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7372 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7373 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7374 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7375 {
7376 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7377 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7378 }
7379
7380 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7381 VMMR0ThreadCtxHookDisable(pVCpu);
7382 HMR0LeaveCpu(pVCpu);
7383 RTThreadPreemptRestore(&PreemptState);
7384 return VINF_SUCCESS;
7385 }
7386
7387 Assert(pVCpu);
7388 Assert(pvUser);
7389 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7390 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7391
7392 VMMRZCallRing3Disable(pVCpu);
7393 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7394
7395 Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
7396
7397 int rc = hmR0VmxLongJmpToRing3(pVCpu);
7398 AssertRCReturn(rc, rc);
7399
7400 VMMRZCallRing3Enable(pVCpu);
7401 return VINF_SUCCESS;
7402}
7403
7404
7405/**
7406 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7407 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7408 *
7409 * @param pVCpu The cross context virtual CPU structure.
7410 */
7411DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7412{
7413 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7414 {
7415 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7416 {
7417 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_PROC_CTLS_INT_WINDOW_EXIT;
7418 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7419 AssertRC(rc);
7420 Log4Func(("Setup interrupt-window exiting\n"));
7421 }
7422 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7423}
7424
7425
7426/**
7427 * Clears the interrupt-window exiting control in the VMCS.
7428 *
7429 * @param pVCpu The cross context virtual CPU structure.
7430 */
7431DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7432{
7433 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT);
7434 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
7435 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7436 AssertRC(rc);
7437 Log4Func(("Cleared interrupt-window exiting\n"));
7438}
7439
7440
7441/**
7442 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7443 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7444 *
7445 * @param pVCpu The cross context virtual CPU structure.
7446 */
7447DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7448{
7449 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7450 {
7451 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7452 {
7453 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7454 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7455 AssertRC(rc);
7456 Log4Func(("Setup NMI-window exiting\n"));
7457 }
7458 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7459}
7460
7461
7462/**
7463 * Clears the NMI-window exiting control in the VMCS.
7464 *
7465 * @param pVCpu The cross context virtual CPU structure.
7466 */
7467DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7468{
7469 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT);
7470 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7471 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7472 AssertRC(rc);
7473 Log4Func(("Cleared NMI-window exiting\n"));
7474}
7475
7476
7477/**
7478 * Evaluates the event to be delivered to the guest and sets it as the pending
7479 * event.
7480 *
7481 * @returns The VT-x guest-interruptibility state.
7482 * @param pVCpu The cross context virtual CPU structure.
7483 */
7484static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu)
7485{
7486 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7487 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7488 uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu);
7489 bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7490 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7491 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7492
7493 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7494 Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7495 Assert(!fBlockSti || pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7496 Assert(!TRPMHasTrap(pVCpu));
7497
7498 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7499 APICUpdatePendingInterrupts(pVCpu);
7500
7501 /*
7502 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7503 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7504 */
7505 /** @todo SMI. SMIs take priority over NMIs. */
7506 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7507 {
7508 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7509 if ( !pVCpu->hm.s.Event.fPending
7510 && !fBlockNmi
7511 && !fBlockSti
7512 && !fBlockMovSS)
7513 {
7514 Log4Func(("Pending NMI\n"));
7515 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INT_INFO_VALID;
7516 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7517
7518 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7519 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7520 }
7521 else
7522 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7523 }
7524 /*
7525 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7526 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7527 */
7528 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7529 && !pVCpu->hm.s.fSingleInstruction)
7530 {
7531 Assert(!DBGFIsStepping(pVCpu));
7532 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7533 AssertRCReturn(rc, 0);
7534 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7535 if ( !pVCpu->hm.s.Event.fPending
7536 && !fBlockInt
7537 && !fBlockSti
7538 && !fBlockMovSS)
7539 {
7540 uint8_t u8Interrupt;
7541 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7542 if (RT_SUCCESS(rc))
7543 {
7544 Log4Func(("Pending external interrupt u8Interrupt=%#x\n", u8Interrupt));
7545 uint32_t u32IntInfo = u8Interrupt
7546 | VMX_EXIT_INT_INFO_VALID
7547 | (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7548
7549 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7550 }
7551 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7552 {
7553 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
7554 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7555 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7556
7557 /*
7558 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7559 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7560 * need to re-set this force-flag here.
7561 */
7562 }
7563 else
7564 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7565 }
7566 else
7567 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7568 }
7569
7570 return fIntrState;
7571}
7572
7573
7574/**
7575 * Sets a pending-debug exception to be delivered to the guest if the guest is
7576 * single-stepping in the VMCS.
7577 *
7578 * @param pVCpu The cross context virtual CPU structure.
7579 */
7580DECLINLINE(int) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7581{
7582 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7583 RT_NOREF(pVCpu);
7584 return VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
7585}
7586
7587
7588/**
7589 * Injects any pending events into the guest if the guest is in a state to
7590 * receive them.
7591 *
7592 * @returns Strict VBox status code (i.e. informational status codes too).
7593 * @param pVCpu The cross context virtual CPU structure.
7594 * @param fIntrState The VT-x guest-interruptibility state.
7595 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7596 * return VINF_EM_DBG_STEPPED if the event was
7597 * dispatched directly.
7598 */
7599static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, uint32_t fIntrState, bool fStepping)
7600{
7601 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7602 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7603
7604 bool fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7605 bool fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7606
7607 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7608 Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7609 Assert(!fBlockSti || pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7610 Assert(!TRPMHasTrap(pVCpu));
7611
7612 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7613 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7614 if (pVCpu->hm.s.Event.fPending)
7615 {
7616 /*
7617 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7618 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7619 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7620 *
7621 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7622 */
7623 uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7624#ifdef VBOX_STRICT
7625 if (uIntType == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
7626 {
7627 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7628 Assert(!fBlockInt);
7629 Assert(!fBlockSti);
7630 Assert(!fBlockMovSS);
7631 }
7632 else if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
7633 {
7634 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7635 Assert(!fBlockSti);
7636 Assert(!fBlockMovSS);
7637 Assert(!fBlockNmi);
7638 }
7639#endif
7640 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7641 uIntType));
7642 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7643 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping,
7644 &fIntrState);
7645 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7646
7647 /* Update the interruptibility-state as it could have been changed by
7648 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7649 fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7650 fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7651
7652 if (uIntType == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
7653 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7654 else
7655 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7656 }
7657
7658 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7659 if ( fBlockSti
7660 || fBlockMovSS)
7661 {
7662 if (!pVCpu->hm.s.fSingleInstruction)
7663 {
7664 /*
7665 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7666 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7667 * See Intel spec. 27.3.4 "Saving Non-Register State".
7668 */
7669 Assert(!DBGFIsStepping(pVCpu));
7670 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7671 AssertRCReturn(rc, rc);
7672 if (pCtx->eflags.Bits.u1TF)
7673 {
7674 int rc2 = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7675 AssertRCReturn(rc2, rc2);
7676 }
7677 }
7678 else if (pCtx->eflags.Bits.u1TF)
7679 {
7680 /*
7681 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7682 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7683 */
7684 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
7685 fIntrState = 0;
7686 }
7687 }
7688
7689 /*
7690 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7691 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7692 */
7693 int rc3 = hmR0VmxExportGuestIntrState(pVCpu, fIntrState);
7694 AssertRCReturn(rc3, rc3);
7695
7696 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7697 NOREF(fBlockMovSS); NOREF(fBlockSti);
7698 return rcStrict;
7699}
7700
7701
7702/**
7703 * Injects a double-fault (\#DF) exception into the VM.
7704 *
7705 * @returns Strict VBox status code (i.e. informational status codes too).
7706 * @param pVCpu The cross context virtual CPU structure.
7707 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7708 * and should return VINF_EM_DBG_STEPPED if the event
7709 * is injected directly (register modified by us, not
7710 * by hardware on VM-entry).
7711 * @param pfIntrState Pointer to the current guest interruptibility-state.
7712 * This interruptibility-state will be updated if
7713 * necessary. This cannot not be NULL.
7714 */
7715DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, bool fStepping, uint32_t *pfIntrState)
7716{
7717 uint32_t const u32IntInfo = X86_XCPT_DF | VMX_EXIT_INT_INFO_VALID
7718 | (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT)
7719 | VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7720 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */, fStepping,
7721 pfIntrState);
7722}
7723
7724
7725/**
7726 * Injects a general-protection (\#GP) fault into the VM.
7727 *
7728 * @returns Strict VBox status code (i.e. informational status codes too).
7729 * @param pVCpu The cross context virtual CPU structure.
7730 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7731 * mode, i.e. in real-mode it's not valid).
7732 * @param u32ErrorCode The error code associated with the \#GP.
7733 * @param fStepping Whether we're running in
7734 * hmR0VmxRunGuestCodeStep() and should return
7735 * VINF_EM_DBG_STEPPED if the event is injected
7736 * directly (register modified by us, not by
7737 * hardware on VM-entry).
7738 * @param pfIntrState Pointer to the current guest interruptibility-state.
7739 * This interruptibility-state will be updated if
7740 * necessary. This cannot not be NULL.
7741 */
7742DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, bool fErrorCodeValid, uint32_t u32ErrorCode, bool fStepping,
7743 uint32_t *pfIntrState)
7744{
7745 uint32_t const u32IntInfo = X86_XCPT_GP | VMX_EXIT_INT_INFO_VALID
7746 | (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT)
7747 | (fErrorCodeValid ? VMX_EXIT_INT_INFO_ERROR_CODE_VALID : 0);
7748 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */, fStepping,
7749 pfIntrState);
7750}
7751
7752
7753/**
7754 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7755 * stack.
7756 *
7757 * @returns Strict VBox status code (i.e. informational status codes too).
7758 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7759 * @param pVCpu The cross context virtual CPU structure.
7760 * @param uValue The value to push to the guest stack.
7761 */
7762static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
7763{
7764 /*
7765 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7766 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7767 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7768 */
7769 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7770 if (pCtx->sp == 1)
7771 return VINF_EM_RESET;
7772 pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7773 int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
7774 AssertRC(rc);
7775 return rc;
7776}
7777
7778
7779/**
7780 * Injects an event into the guest upon VM-entry by updating the relevant fields
7781 * in the VM-entry area in the VMCS.
7782 *
7783 * @returns Strict VBox status code (i.e. informational status codes too).
7784 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7785 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7786 *
7787 * @param pVCpu The cross context virtual CPU structure.
7788 * @param u64IntInfo The VM-entry interruption-information field.
7789 * @param cbInstr The VM-entry instruction length in bytes (for
7790 * software interrupts, exceptions and privileged
7791 * software exceptions).
7792 * @param u32ErrCode The VM-entry exception error code.
7793 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7794 * @param pfIntrState Pointer to the current guest interruptibility-state.
7795 * This interruptibility-state will be updated if
7796 * necessary. This cannot not be NULL.
7797 * @param fStepping Whether we're running in
7798 * hmR0VmxRunGuestCodeStep() and should return
7799 * VINF_EM_DBG_STEPPED if the event is injected
7800 * directly (register modified by us, not by
7801 * hardware on VM-entry).
7802 */
7803static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7804 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState)
7805{
7806 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7807 AssertMsg(!RT_HI_U32(u64IntInfo), ("%#RX64\n", u64IntInfo));
7808 Assert(pfIntrState);
7809
7810 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7811 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7812 uint32_t const uVector = VMX_EXIT_INT_INFO_VECTOR(u32IntInfo);
7813 uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(u32IntInfo);
7814
7815#ifdef VBOX_STRICT
7816 /*
7817 * Validate the error-code-valid bit for hardware exceptions.
7818 * No error codes for exceptions in real-mode.
7819 *
7820 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7821 */
7822 if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
7823 && !CPUMIsGuestInRealModeEx(pCtx))
7824 {
7825 switch (uVector)
7826 {
7827 case X86_XCPT_PF:
7828 case X86_XCPT_DF:
7829 case X86_XCPT_TS:
7830 case X86_XCPT_NP:
7831 case X86_XCPT_SS:
7832 case X86_XCPT_GP:
7833 case X86_XCPT_AC:
7834 AssertMsg(VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
7835 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7836 RT_FALL_THRU();
7837 default:
7838 break;
7839 }
7840 }
7841#endif
7842
7843 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7844 Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
7845 || !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
7846
7847 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7848
7849 /*
7850 * Hardware interrupts & exceptions cannot be delivered through the software interrupt
7851 * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
7852 * interrupt handler in the (real-mode) guest.
7853 *
7854 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
7855 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7856 */
7857 if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
7858 {
7859 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
7860 {
7861 /*
7862 * For unrestricted execution enabled CPUs running real-mode guests, we must not
7863 * set the deliver-error-code bit.
7864 *
7865 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7866 */
7867 u32IntInfo &= ~VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7868 }
7869 else
7870 {
7871 PVM pVM = pVCpu->CTX_SUFF(pVM);
7872 Assert(PDMVmmDevHeapIsEnabled(pVM));
7873 Assert(pVM->hm.s.vmx.pRealModeTSS);
7874
7875 /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
7876 int rc2 = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_TABLE_MASK | CPUMCTX_EXTRN_RIP
7877 | CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_RFLAGS);
7878 AssertRCReturn(rc2, rc2);
7879
7880 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7881 size_t const cbIdtEntry = sizeof(X86IDTR16);
7882 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
7883 {
7884 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7885 if (uVector == X86_XCPT_DF)
7886 return VINF_EM_RESET;
7887
7888 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7889 if (uVector == X86_XCPT_GP)
7890 return hmR0VmxInjectXcptDF(pVCpu, fStepping, pfIntrState);
7891
7892 /*
7893 * If we're injecting an event with no valid IDT entry, inject a #GP.
7894 * No error codes for exceptions in real-mode.
7895 *
7896 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7897 */
7898 return hmR0VmxInjectXcptGP(pVCpu, false /* fErrCodeValid */, 0 /* u32ErrCode */, fStepping, pfIntrState);
7899 }
7900
7901 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7902 uint16_t uGuestIp = pCtx->ip;
7903 if (uIntType == VMX_EXIT_INT_INFO_TYPE_SW_XCPT)
7904 {
7905 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7906 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7907 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7908 }
7909 else if (uIntType == VMX_EXIT_INT_INFO_TYPE_SW_INT)
7910 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7911
7912 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7913 X86IDTR16 IdtEntry;
7914 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
7915 rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7916 AssertRCReturn(rc2, rc2);
7917
7918 /* Construct the stack frame for the interrupt/exception handler. */
7919 VBOXSTRICTRC rcStrict;
7920 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
7921 if (rcStrict == VINF_SUCCESS)
7922 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
7923 if (rcStrict == VINF_SUCCESS)
7924 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
7925
7926 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7927 if (rcStrict == VINF_SUCCESS)
7928 {
7929 pCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7930 pCtx->rip = IdtEntry.offSel;
7931 pCtx->cs.Sel = IdtEntry.uSel;
7932 pCtx->cs.ValidSel = IdtEntry.uSel;
7933 pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7934 if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
7935 && uVector == X86_XCPT_PF)
7936 pCtx->cr2 = GCPtrFaultAddress;
7937
7938 /* If any other guest-state bits are changed here, make sure to update
7939 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7940 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS | HM_CHANGED_GUEST_CR2
7941 | HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
7942 | HM_CHANGED_GUEST_RSP);
7943
7944 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7945 if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
7946 {
7947 Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
7948 && uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
7949 Log4Func(("Clearing inhibition due to STI\n"));
7950 *pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
7951 }
7952 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7953 u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
7954
7955 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7956 it, if we are returning to ring-3 before executing guest code. */
7957 pVCpu->hm.s.Event.fPending = false;
7958
7959 /* Make hmR0VmxPreRunGuest() return if we're stepping since we've changed cs:rip. */
7960 if (fStepping)
7961 rcStrict = VINF_EM_DBG_STEPPED;
7962 }
7963 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7964 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7965 return rcStrict;
7966 }
7967 }
7968
7969 /* Validate. */
7970 Assert(VMX_EXIT_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7971 Assert(!VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7972 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7973
7974 /* Inject. */
7975 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7976 if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
7977 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7978 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7979 AssertRCReturn(rc, rc);
7980
7981 /* Update CR2. */
7982 if ( VMX_EXIT_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
7983 && uVector == X86_XCPT_PF)
7984 pCtx->cr2 = GCPtrFaultAddress;
7985
7986 Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
7987
7988 return VINF_SUCCESS;
7989}
7990
7991
7992/**
7993 * Clears the interrupt-window exiting control in the VMCS and if necessary
7994 * clears the current event in the VMCS as well.
7995 *
7996 * @returns VBox status code.
7997 * @param pVCpu The cross context virtual CPU structure.
7998 *
7999 * @remarks Use this function only to clear events that have not yet been
8000 * delivered to the guest but are injected in the VMCS!
8001 * @remarks No-long-jump zone!!!
8002 */
8003static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8004{
8005 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8006 {
8007 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8008 Log4Func(("Cleared interrupt widow\n"));
8009 }
8010
8011 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8012 {
8013 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8014 Log4Func(("Cleared interrupt widow\n"));
8015 }
8016}
8017
8018
8019/**
8020 * Enters the VT-x session.
8021 *
8022 * @returns VBox status code.
8023 * @param pVCpu The cross context virtual CPU structure.
8024 * @param pHostCpu Pointer to the global CPU info struct.
8025 */
8026VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu, PHMGLOBALCPUINFO pHostCpu)
8027{
8028 AssertPtr(pVCpu);
8029 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8030 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8031 RT_NOREF(pHostCpu);
8032
8033 LogFlowFunc(("pVCpu=%p\n", pVCpu));
8034 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8035 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8036
8037#ifdef VBOX_STRICT
8038 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8039 RTCCUINTREG uHostCR4 = ASMGetCR4();
8040 if (!(uHostCR4 & X86_CR4_VMXE))
8041 {
8042 LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
8043 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8044 }
8045#endif
8046
8047 /*
8048 * Load the VCPU's VMCS as the current (and active) one.
8049 */
8050 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8051 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8052 if (RT_FAILURE(rc))
8053 return rc;
8054
8055 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8056 pVCpu->hm.s.fLeaveDone = false;
8057 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8058
8059 return VINF_SUCCESS;
8060}
8061
8062
8063/**
8064 * The thread-context callback (only on platforms which support it).
8065 *
8066 * @param enmEvent The thread-context event.
8067 * @param pVCpu The cross context virtual CPU structure.
8068 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8069 * @thread EMT(pVCpu)
8070 */
8071VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8072{
8073 NOREF(fGlobalInit);
8074
8075 switch (enmEvent)
8076 {
8077 case RTTHREADCTXEVENT_OUT:
8078 {
8079 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8080 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8081 VMCPU_ASSERT_EMT(pVCpu);
8082
8083 /* No longjmps (logger flushes, locks) in this fragile context. */
8084 VMMRZCallRing3Disable(pVCpu);
8085 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8086
8087 /*
8088 * Restore host-state (FPU, debug etc.)
8089 */
8090 if (!pVCpu->hm.s.fLeaveDone)
8091 {
8092 /*
8093 * Do -not- import the guest-state here as we might already be in the middle of importing
8094 * it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
8095 */
8096 hmR0VmxLeave(pVCpu, false /* fImportState */);
8097 pVCpu->hm.s.fLeaveDone = true;
8098 }
8099
8100 /* Leave HM context, takes care of local init (term). */
8101 int rc = HMR0LeaveCpu(pVCpu);
8102 AssertRC(rc); NOREF(rc);
8103
8104 /* Restore longjmp state. */
8105 VMMRZCallRing3Enable(pVCpu);
8106 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8107 break;
8108 }
8109
8110 case RTTHREADCTXEVENT_IN:
8111 {
8112 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8113 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8114 VMCPU_ASSERT_EMT(pVCpu);
8115
8116 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8117 VMMRZCallRing3Disable(pVCpu);
8118 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8119
8120 /* Initialize the bare minimum state required for HM. This takes care of
8121 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8122 int rc = hmR0EnterCpu(pVCpu);
8123 AssertRC(rc);
8124 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8125 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8126
8127 /* Load the active VMCS as the current one. */
8128 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8129 {
8130 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8131 AssertRC(rc); NOREF(rc);
8132 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8133 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8134 }
8135 pVCpu->hm.s.fLeaveDone = false;
8136
8137 /* Restore longjmp state. */
8138 VMMRZCallRing3Enable(pVCpu);
8139 break;
8140 }
8141
8142 default:
8143 break;
8144 }
8145}
8146
8147
8148/**
8149 * Exports the host state into the VMCS host-state area.
8150 * Sets up the VM-exit MSR-load area.
8151 *
8152 * The CPU state will be loaded from these fields on every successful VM-exit.
8153 *
8154 * @returns VBox status code.
8155 * @param pVCpu The cross context virtual CPU structure.
8156 *
8157 * @remarks No-long-jump zone!!!
8158 */
8159static int hmR0VmxExportHostState(PVMCPU pVCpu)
8160{
8161 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8162
8163 int rc = VINF_SUCCESS;
8164 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8165 {
8166 rc = hmR0VmxExportHostControlRegs();
8167 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8168
8169 rc = hmR0VmxExportHostSegmentRegs(pVCpu);
8170 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8171
8172 rc = hmR0VmxExportHostMsrs(pVCpu);
8173 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8174
8175 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
8176 }
8177 return rc;
8178}
8179
8180
8181/**
8182 * Saves the host state in the VMCS host-state.
8183 *
8184 * @returns VBox status code.
8185 * @param pVCpu The cross context virtual CPU structure.
8186 *
8187 * @remarks No-long-jump zone!!!
8188 */
8189VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
8190{
8191 AssertPtr(pVCpu);
8192 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8193
8194 /*
8195 * Export the host state here while entering HM context.
8196 * When thread-context hooks are used, we might get preempted and have to re-save the host
8197 * state but most of the time we won't be, so do it here before we disable interrupts.
8198 */
8199 return hmR0VmxExportHostState(pVCpu);
8200}
8201
8202
8203/**
8204 * Exports the guest state into the VMCS guest-state area.
8205 *
8206 * The will typically be done before VM-entry when the guest-CPU state and the
8207 * VMCS state may potentially be out of sync.
8208 *
8209 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8210 * VM-entry controls.
8211 * Sets up the appropriate VMX non-root function to execute guest code based on
8212 * the guest CPU mode.
8213 *
8214 * @returns VBox strict status code.
8215 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8216 * without unrestricted guest access and the VMMDev is not presently
8217 * mapped (e.g. EFI32).
8218 *
8219 * @param pVCpu The cross context virtual CPU structure.
8220 *
8221 * @remarks No-long-jump zone!!!
8222 */
8223static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu)
8224{
8225 AssertPtr(pVCpu);
8226 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8227
8228 LogFlowFunc(("pVCpu=%p\n", pVCpu));
8229
8230 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
8231
8232 /* Determine real-on-v86 mode. */
8233 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8234 if ( !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
8235 && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
8236 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8237
8238 /*
8239 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8240 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8241 */
8242 int rc = hmR0VmxSelectVMRunHandler(pVCpu);
8243 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8244
8245 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8246 rc = hmR0VmxExportGuestEntryCtls(pVCpu);
8247 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8248
8249 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8250 rc = hmR0VmxExportGuestExitCtls(pVCpu);
8251 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8252
8253 rc = hmR0VmxExportGuestCR0(pVCpu);
8254 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8255
8256 VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu);
8257 if (rcStrict == VINF_SUCCESS)
8258 { /* likely */ }
8259 else
8260 {
8261 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8262 return rcStrict;
8263 }
8264
8265 rc = hmR0VmxExportGuestSegmentRegs(pVCpu);
8266 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8267
8268 /* This needs to be done after hmR0VmxExportGuestEntryCtls() and hmR0VmxExportGuestExitCtls() as it
8269 may alter controls if we determine we don't have to swap EFER after all. */
8270 rc = hmR0VmxExportGuestMsrs(pVCpu);
8271 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8272
8273 rc = hmR0VmxExportGuestApicTpr(pVCpu);
8274 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8275
8276 rc = hmR0VmxExportGuestXcptIntercepts(pVCpu);
8277 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8278
8279 /* Exporting RFLAGS here is fine, even though RFLAGS.TF might depend on guest debug state which is
8280 not exported here. It is re-evaluated and updated if necessary in hmR0VmxExportSharedState(). */
8281 rc = hmR0VmxExportGuestRip(pVCpu);
8282 rc |= hmR0VmxExportGuestRsp(pVCpu);
8283 rc |= hmR0VmxExportGuestRflags(pVCpu);
8284 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8285
8286 /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
8287 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
8288 | HM_CHANGED_GUEST_CR2
8289 | (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
8290 | HM_CHANGED_GUEST_X87
8291 | HM_CHANGED_GUEST_SSE_AVX
8292 | HM_CHANGED_GUEST_OTHER_XSAVE
8293 | HM_CHANGED_GUEST_XCRx
8294 | HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
8295 | HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
8296 | HM_CHANGED_GUEST_TSC_AUX
8297 | HM_CHANGED_GUEST_OTHER_MSRS
8298 | HM_CHANGED_GUEST_HWVIRT
8299 | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
8300
8301 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
8302 return rc;
8303}
8304
8305
8306/**
8307 * Exports the state shared between the host and guest into the VMCS.
8308 *
8309 * @param pVCpu The cross context virtual CPU structure.
8310 *
8311 * @remarks No-long-jump zone!!!
8312 */
8313static void hmR0VmxExportSharedState(PVMCPU pVCpu)
8314{
8315 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8316 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8317
8318 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
8319 {
8320 int rc = hmR0VmxExportSharedDebugState(pVCpu);
8321 AssertRC(rc);
8322 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
8323
8324 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8325 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
8326 {
8327 rc = hmR0VmxExportGuestRflags(pVCpu);
8328 AssertRC(rc);
8329 }
8330 }
8331
8332 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
8333 {
8334 hmR0VmxLazyLoadGuestMsrs(pVCpu);
8335 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
8336 }
8337
8338 AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
8339 ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8340}
8341
8342
8343/**
8344 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8345 *
8346 * @returns Strict VBox status code (i.e. informational status codes too).
8347 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8348 * without unrestricted guest access and the VMMDev is not presently
8349 * mapped (e.g. EFI32).
8350 *
8351 * @param pVCpu The cross context virtual CPU structure.
8352 *
8353 * @remarks No-long-jump zone!!!
8354 */
8355static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu)
8356{
8357 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8358 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8359 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8360
8361#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8362 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8363#endif
8364
8365 /*
8366 * For many exits it's only RIP that changes and hence try to export it first
8367 * without going through a lot of change flag checks.
8368 */
8369 VBOXSTRICTRC rcStrict;
8370 uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8371 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8372 if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
8373 {
8374 rcStrict = hmR0VmxExportGuestRip(pVCpu);
8375 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8376 { /* likely */}
8377 else
8378 AssertMsgFailedReturn(("hmR0VmxExportGuestRip failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8379 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
8380 }
8381 else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8382 {
8383 rcStrict = hmR0VmxExportGuestState(pVCpu);
8384 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8385 { /* likely */}
8386 else
8387 {
8388 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("hmR0VmxExportGuestState failed! rc=%Rrc\n",
8389 VBOXSTRICTRC_VAL(rcStrict)));
8390 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8391 return rcStrict;
8392 }
8393 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
8394 }
8395 else
8396 rcStrict = VINF_SUCCESS;
8397
8398#ifdef VBOX_STRICT
8399 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8400 fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8401 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8402 AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
8403 ("fCtxChanged=%#RX64\n", fCtxChanged));
8404#endif
8405 return rcStrict;
8406}
8407
8408
8409/**
8410 * Does the preparations before executing guest code in VT-x.
8411 *
8412 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8413 * recompiler/IEM. We must be cautious what we do here regarding committing
8414 * guest-state information into the VMCS assuming we assuredly execute the
8415 * guest in VT-x mode.
8416 *
8417 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8418 * the common-state (TRPM/forceflags), we must undo those changes so that the
8419 * recompiler/IEM can (and should) use them when it resumes guest execution.
8420 * Otherwise such operations must be done when we can no longer exit to ring-3.
8421 *
8422 * @returns Strict VBox status code (i.e. informational status codes too).
8423 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8424 * have been disabled.
8425 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8426 * double-fault into the guest.
8427 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8428 * dispatched directly.
8429 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8430 *
8431 * @param pVCpu The cross context virtual CPU structure.
8432 * @param pVmxTransient Pointer to the VMX transient structure.
8433 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8434 * us ignore some of the reasons for returning to
8435 * ring-3, and return VINF_EM_DBG_STEPPED if event
8436 * dispatching took place.
8437 */
8438static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
8439{
8440 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8441
8442#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_ONLY_IN_IEM
8443 Log2(("hmR0SvmPreRunGuest: Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
8444 return VINF_EM_RESCHEDULE_REM;
8445#endif
8446
8447#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8448 PGMRZDynMapFlushAutoSet(pVCpu);
8449#endif
8450
8451 /* Check force flag actions that might require us to go back to ring-3. */
8452 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
8453 if (rcStrict == VINF_SUCCESS)
8454 { /* FFs doesn't get set all the time. */ }
8455 else
8456 return rcStrict;
8457
8458 /*
8459 * Setup the virtualized-APIC accesses.
8460 *
8461 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8462 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8463 *
8464 * This is the reason we do it here and not in hmR0VmxExportGuestState().
8465 */
8466 PVM pVM = pVCpu->CTX_SUFF(pVM);
8467 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8468 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
8469 && PDMHasApic(pVM))
8470 {
8471 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8472 Assert(u64MsrApicBase);
8473 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8474
8475 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8476
8477 /* Unalias any existing mapping. */
8478 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8479 AssertRCReturn(rc, rc);
8480
8481 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8482 Log4Func(("Mapped HC APIC-access page at %#RGp\n", GCPhysApicBase));
8483 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8484 AssertRCReturn(rc, rc);
8485
8486 /* Update the per-VCPU cache of the APIC base MSR. */
8487 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8488 }
8489
8490 if (TRPMHasTrap(pVCpu))
8491 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8492 uint32_t fIntrState = hmR0VmxEvaluatePendingEvent(pVCpu);
8493
8494 /*
8495 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
8496 * needs to be done with longjmps or interrupts + preemption enabled. Event injection might
8497 * also result in triple-faulting the VM.
8498 */
8499 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, fIntrState, fStepping);
8500 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8501 { /* likely */ }
8502 else
8503 {
8504 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8505 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8506 return rcStrict;
8507 }
8508
8509 /*
8510 * A longjump might result in importing CR3 even for VM-exits that don't necessarily
8511 * import CR3 themselves. We will need to update them here, as even as late as the above
8512 * hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
8513 * the below force flags to be set.
8514 */
8515 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8516 {
8517 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
8518 int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8519 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
8520 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
8521 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8522 }
8523 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8524 {
8525 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8526 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8527 }
8528
8529 /*
8530 * No longjmps to ring-3 from this point on!!!
8531 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8532 * This also disables flushing of the R0-logger instance (if any).
8533 */
8534 VMMRZCallRing3Disable(pVCpu);
8535
8536 /*
8537 * Export the guest state bits.
8538 *
8539 * We cannot perform longjmps while loading the guest state because we do not preserve the
8540 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8541 * CPU migration.
8542 *
8543 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8544 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8545 * Hence, loading of the guest state needs to be done -after- injection of events.
8546 */
8547 rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu);
8548 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8549 { /* likely */ }
8550 else
8551 {
8552 VMMRZCallRing3Enable(pVCpu);
8553 return rcStrict;
8554 }
8555
8556 /*
8557 * We disable interrupts so that we don't miss any interrupts that would flag preemption
8558 * (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
8559 * preemption disabled for a while. Since this is purly to aid the
8560 * RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
8561 * disable interrupt on NT.
8562 *
8563 * We need to check for force-flags that could've possible been altered since we last
8564 * checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
8565 * see @bugref{6398}).
8566 *
8567 * We also check a couple of other force-flags as a last opportunity to get the EMT back
8568 * to ring-3 before executing guest code.
8569 */
8570 pVmxTransient->fEFlags = ASMIntDisableFlags();
8571
8572 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8573 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8574 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8575 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8576 {
8577 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8578 {
8579 pVCpu->hm.s.Event.fPending = false;
8580
8581 /*
8582 * We've injected any pending events. This is really the point of no return (to ring-3).
8583 *
8584 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
8585 * returns from this function, so don't enable them here.
8586 */
8587 return VINF_SUCCESS;
8588 }
8589
8590 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
8591 rcStrict = VINF_EM_RAW_INTERRUPT;
8592 }
8593 else
8594 {
8595 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8596 rcStrict = VINF_EM_RAW_TO_R3;
8597 }
8598
8599 ASMSetFlags(pVmxTransient->fEFlags);
8600 VMMRZCallRing3Enable(pVCpu);
8601
8602 return rcStrict;
8603}
8604
8605
8606/**
8607 * Prepares to run guest code in VT-x and we've committed to doing so. This
8608 * means there is no backing out to ring-3 or anywhere else at this
8609 * point.
8610 *
8611 * @param pVCpu The cross context virtual CPU structure.
8612 * @param pVmxTransient Pointer to the VMX transient structure.
8613 *
8614 * @remarks Called with preemption disabled.
8615 * @remarks No-long-jump zone!!!
8616 */
8617static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
8618{
8619 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8620 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8621 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8622
8623 /*
8624 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8625 */
8626 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8627 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8628
8629 PVM pVM = pVCpu->CTX_SUFF(pVM);
8630 if (!CPUMIsGuestFPUStateActive(pVCpu))
8631 {
8632 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8633 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8634 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT;
8635 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8636 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
8637 }
8638
8639 /*
8640 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8641 */
8642 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8643 && pVCpu->hm.s.vmx.cMsrs > 0)
8644 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8645
8646 /*
8647 * Re-save the host state bits as we may've been preempted (only happens when
8648 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8649 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8650 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8651 * See @bugref{8432}.
8652 */
8653 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8654 {
8655 int rc = hmR0VmxExportHostState(pVCpu);
8656 AssertRC(rc);
8657 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
8658 }
8659 Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
8660
8661 /*
8662 * Export the state shared between host and guest (FPU, debug, lazy MSRs).
8663 */
8664 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
8665 hmR0VmxExportSharedState(pVCpu);
8666 AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8667
8668 /* Store status of the shared guest-host state at the time of VM-entry. */
8669#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8670 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
8671 {
8672 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8673 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8674 }
8675 else
8676#endif
8677 {
8678 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8679 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8680 }
8681
8682 /*
8683 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8684 */
8685 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
8686 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8687
8688 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8689 RTCPUID idCurrentCpu = pCpu->idCpu;
8690 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8691 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8692 {
8693 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8694 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8695 }
8696
8697 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8698 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8699 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8700 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8701
8702 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8703
8704 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8705 to start executing. */
8706
8707 /*
8708 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8709 */
8710 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
8711 {
8712 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
8713 {
8714 bool fMsrUpdated;
8715 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
8716 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8717 &fMsrUpdated);
8718 AssertRC(rc2);
8719 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8720 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8721 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8722 }
8723 else
8724 {
8725 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8726 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8727 }
8728 }
8729
8730 if (pVM->cpum.ro.GuestFeatures.fIbrs)
8731 {
8732 bool fMsrUpdated;
8733 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_OTHER_MSRS);
8734 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
8735 &fMsrUpdated);
8736 AssertRC(rc2);
8737 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8738 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8739 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8740 }
8741
8742#ifdef VBOX_STRICT
8743 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8744 hmR0VmxCheckHostEferMsr(pVCpu);
8745 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8746#endif
8747#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8748 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
8749 {
8750 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
8751 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8752 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8753 }
8754#endif
8755}
8756
8757
8758/**
8759 * Performs some essential restoration of state after running guest code in
8760 * VT-x.
8761 *
8762 * @param pVCpu The cross context virtual CPU structure.
8763 * @param pVmxTransient Pointer to the VMX transient structure.
8764 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8765 *
8766 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8767 *
8768 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8769 * unconditionally when it is safe to do so.
8770 */
8771static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8772{
8773 uint64_t const uHostTsc = ASMReadTSC();
8774 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8775
8776 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8777 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8778 pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8779 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8780 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8781 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8782
8783 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
8784 TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TscOffset);
8785
8786 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
8787 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8788 Assert(!ASMIntAreEnabled());
8789 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8790
8791#if HC_ARCH_BITS == 64
8792 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8793#endif
8794#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8795 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8796 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8797 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8798#else
8799 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8800#endif
8801#ifdef VBOX_STRICT
8802 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8803#endif
8804 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8805
8806 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8807 uint32_t uExitReason;
8808 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8809 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8810 AssertRC(rc);
8811 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8812 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INT_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8813
8814 if (rcVMRun == VINF_SUCCESS)
8815 {
8816 /*
8817 * Update the VM-exit history array here even if the VM-entry failed due to:
8818 * - Invalid guest state.
8819 * - MSR loading.
8820 * - Machine-check event.
8821 *
8822 * In any of the above cases we will still have a "valid" VM-exit reason
8823 * despite @a fVMEntryFailed being false.
8824 *
8825 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8826 *
8827 * Note! We don't have CS or RIP at this point. Will probably address that later
8828 * by amending the history entry added here.
8829 */
8830 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
8831 UINT64_MAX, uHostTsc);
8832
8833 if (!pVmxTransient->fVMEntryFailed)
8834 {
8835 VMMRZCallRing3Enable(pVCpu);
8836
8837 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8838 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8839
8840#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8841 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
8842 AssertRC(rc);
8843#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8844 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_RFLAGS);
8845 AssertRC(rc);
8846#else
8847 /*
8848 * Import the guest-interruptibility state always as we need it while evaluating
8849 * injecting events on re-entry.
8850 *
8851 * We don't import CR0 (when Unrestricted guest execution is unavailable) despite
8852 * checking for real-mode while exporting the state because all bits that cause
8853 * mode changes wrt CR0 are intercepted.
8854 */
8855 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
8856 AssertRC(rc);
8857#endif
8858
8859 /*
8860 * Sync the TPR shadow with our APIC state.
8861 */
8862 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
8863 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8864 {
8865 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8866 AssertRC(rc);
8867 ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8868 }
8869
8870 return;
8871 }
8872 }
8873 else
8874 Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
8875
8876 VMMRZCallRing3Enable(pVCpu);
8877}
8878
8879
8880/**
8881 * Runs the guest code using VT-x the normal way.
8882 *
8883 * @returns VBox status code.
8884 * @param pVCpu The cross context virtual CPU structure.
8885 *
8886 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8887 */
8888static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu)
8889{
8890 VMXTRANSIENT VmxTransient;
8891 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8892 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8893 uint32_t cLoops = 0;
8894
8895 for (;; cLoops++)
8896 {
8897 Assert(!HMR0SuspendPending());
8898 HMVMX_ASSERT_CPU_SAFE(pVCpu);
8899
8900 /* Preparatory work for running guest code, this may force us to return
8901 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8902 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8903 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
8904 if (rcStrict != VINF_SUCCESS)
8905 break;
8906
8907 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
8908 int rcRun = hmR0VmxRunGuest(pVCpu);
8909
8910 /* Restore any residual host-state and save any bits shared between host
8911 and guest into the guest-CPU state. Re-enables interrupts! */
8912 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
8913
8914 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8915 if (RT_SUCCESS(rcRun))
8916 { /* very likely */ }
8917 else
8918 {
8919 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
8920 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
8921 return rcRun;
8922 }
8923
8924 /* Profile the VM-exit. */
8925 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8926 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8927 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8928 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
8929 HMVMX_START_EXIT_DISPATCH_PROF();
8930
8931 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
8932
8933 /* Handle the VM-exit. */
8934#ifdef HMVMX_USE_FUNCTION_TABLE
8935 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
8936#else
8937 rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient, VmxTransient.uExitReason);
8938#endif
8939 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
8940 if (rcStrict == VINF_SUCCESS)
8941 {
8942 if (cLoops <= pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
8943 continue; /* likely */
8944 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8945 rcStrict = VINF_EM_RAW_INTERRUPT;
8946 }
8947 break;
8948 }
8949
8950 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8951 return rcStrict;
8952}
8953
8954
8955
8956/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8957 * probes.
8958 *
8959 * The following few functions and associated structure contains the bloat
8960 * necessary for providing detailed debug events and dtrace probes as well as
8961 * reliable host side single stepping. This works on the principle of
8962 * "subclassing" the normal execution loop and workers. We replace the loop
8963 * method completely and override selected helpers to add necessary adjustments
8964 * to their core operation.
8965 *
8966 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8967 * any performance for debug and analysis features.
8968 *
8969 * @{
8970 */
8971
8972/**
8973 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
8974 * the debug run loop.
8975 */
8976typedef struct VMXRUNDBGSTATE
8977{
8978 /** The RIP we started executing at. This is for detecting that we stepped. */
8979 uint64_t uRipStart;
8980 /** The CS we started executing with. */
8981 uint16_t uCsStart;
8982
8983 /** Whether we've actually modified the 1st execution control field. */
8984 bool fModifiedProcCtls : 1;
8985 /** Whether we've actually modified the 2nd execution control field. */
8986 bool fModifiedProcCtls2 : 1;
8987 /** Whether we've actually modified the exception bitmap. */
8988 bool fModifiedXcptBitmap : 1;
8989
8990 /** We desire the modified the CR0 mask to be cleared. */
8991 bool fClearCr0Mask : 1;
8992 /** We desire the modified the CR4 mask to be cleared. */
8993 bool fClearCr4Mask : 1;
8994 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
8995 uint32_t fCpe1Extra;
8996 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
8997 uint32_t fCpe1Unwanted;
8998 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
8999 uint32_t fCpe2Extra;
9000 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9001 uint32_t bmXcptExtra;
9002 /** The sequence number of the Dtrace provider settings the state was
9003 * configured against. */
9004 uint32_t uDtraceSettingsSeqNo;
9005 /** VM-exits to check (one bit per VM-exit). */
9006 uint32_t bmExitsToCheck[3];
9007
9008 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9009 uint32_t fProcCtlsInitial;
9010 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9011 uint32_t fProcCtls2Initial;
9012 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9013 uint32_t bmXcptInitial;
9014} VMXRUNDBGSTATE;
9015AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9016typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9017
9018
9019/**
9020 * Initializes the VMXRUNDBGSTATE structure.
9021 *
9022 * @param pVCpu The cross context virtual CPU structure of the
9023 * calling EMT.
9024 * @param pDbgState The structure to initialize.
9025 */
9026static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9027{
9028 pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
9029 pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
9030
9031 pDbgState->fModifiedProcCtls = false;
9032 pDbgState->fModifiedProcCtls2 = false;
9033 pDbgState->fModifiedXcptBitmap = false;
9034 pDbgState->fClearCr0Mask = false;
9035 pDbgState->fClearCr4Mask = false;
9036 pDbgState->fCpe1Extra = 0;
9037 pDbgState->fCpe1Unwanted = 0;
9038 pDbgState->fCpe2Extra = 0;
9039 pDbgState->bmXcptExtra = 0;
9040 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9041 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9042 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9043}
9044
9045
9046/**
9047 * Updates the VMSC fields with changes requested by @a pDbgState.
9048 *
9049 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9050 * immediately before executing guest code, i.e. when interrupts are disabled.
9051 * We don't check status codes here as we cannot easily assert or return in the
9052 * latter case.
9053 *
9054 * @param pVCpu The cross context virtual CPU structure.
9055 * @param pDbgState The debug state.
9056 */
9057static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9058{
9059 /*
9060 * Ensure desired flags in VMCS control fields are set.
9061 * (Ignoring write failure here, as we're committed and it's just debug extras.)
9062 *
9063 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9064 * there should be no stale data in pCtx at this point.
9065 */
9066 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9067 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9068 {
9069 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
9070 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9071 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9072 Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9073 pDbgState->fModifiedProcCtls = true;
9074 }
9075
9076 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9077 {
9078 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
9079 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9080 Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9081 pDbgState->fModifiedProcCtls2 = true;
9082 }
9083
9084 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9085 {
9086 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
9087 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9088 Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9089 pDbgState->fModifiedXcptBitmap = true;
9090 }
9091
9092 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32Cr0Mask != 0)
9093 {
9094 pVCpu->hm.s.vmx.u32Cr0Mask = 0;
9095 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9096 Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9097 }
9098
9099 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32Cr4Mask != 0)
9100 {
9101 pVCpu->hm.s.vmx.u32Cr4Mask = 0;
9102 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9103 Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9104 }
9105}
9106
9107
9108/**
9109 * Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
9110 * re-entry next time around.
9111 *
9112 * @returns Strict VBox status code (i.e. informational status codes too).
9113 * @param pVCpu The cross context virtual CPU structure.
9114 * @param pDbgState The debug state.
9115 * @param rcStrict The return code from executing the guest using single
9116 * stepping.
9117 */
9118static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9119{
9120 /*
9121 * Restore VM-exit control settings as we may not reenter this function the
9122 * next time around.
9123 */
9124 /* We reload the initial value, trigger what we can of recalculations the
9125 next time around. From the looks of things, that's all that's required atm. */
9126 if (pDbgState->fModifiedProcCtls)
9127 {
9128 if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9129 pDbgState->fProcCtlsInitial |= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9130 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9131 AssertRCReturn(rc2, rc2);
9132 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9133 }
9134
9135 /* We're currently the only ones messing with this one, so just restore the
9136 cached value and reload the field. */
9137 if ( pDbgState->fModifiedProcCtls2
9138 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9139 {
9140 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9141 AssertRCReturn(rc2, rc2);
9142 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9143 }
9144
9145 /* If we've modified the exception bitmap, we restore it and trigger
9146 reloading and partial recalculation the next time around. */
9147 if (pDbgState->fModifiedXcptBitmap)
9148 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9149
9150 return rcStrict;
9151}
9152
9153
9154/**
9155 * Configures VM-exit controls for current DBGF and DTrace settings.
9156 *
9157 * This updates @a pDbgState and the VMCS execution control fields to reflect
9158 * the necessary VM-exits demanded by DBGF and DTrace.
9159 *
9160 * @param pVCpu The cross context virtual CPU structure.
9161 * @param pDbgState The debug state.
9162 * @param pVmxTransient Pointer to the VMX transient structure. May update
9163 * fUpdateTscOffsettingAndPreemptTimer.
9164 */
9165static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9166{
9167 /*
9168 * Take down the dtrace serial number so we can spot changes.
9169 */
9170 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9171 ASMCompilerBarrier();
9172
9173 /*
9174 * We'll rebuild most of the middle block of data members (holding the
9175 * current settings) as we go along here, so start by clearing it all.
9176 */
9177 pDbgState->bmXcptExtra = 0;
9178 pDbgState->fCpe1Extra = 0;
9179 pDbgState->fCpe1Unwanted = 0;
9180 pDbgState->fCpe2Extra = 0;
9181 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9182 pDbgState->bmExitsToCheck[i] = 0;
9183
9184 /*
9185 * Software interrupts (INT XXh) - no idea how to trigger these...
9186 */
9187 PVM pVM = pVCpu->CTX_SUFF(pVM);
9188 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9189 || VBOXVMM_INT_SOFTWARE_ENABLED())
9190 {
9191 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9192 }
9193
9194 /*
9195 * INT3 breakpoints - triggered by #BP exceptions.
9196 */
9197 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9198 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9199
9200 /*
9201 * Exception bitmap and XCPT events+probes.
9202 */
9203 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9204 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9205 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9206
9207 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9208 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9209 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9210 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9211 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9212 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9213 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9214 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9215 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9216 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9217 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9218 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9219 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9220 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9221 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9222 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9223 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9224 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9225
9226 if (pDbgState->bmXcptExtra)
9227 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9228
9229 /*
9230 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9231 *
9232 * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
9233 * So, when adding/changing/removing please don't forget to update it.
9234 *
9235 * Some of the macros are picking up local variables to save horizontal space,
9236 * (being able to see it in a table is the lesser evil here).
9237 */
9238#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9239 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9240 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9241#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9242 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9243 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9244 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9245 } else do { } while (0)
9246#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9247 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9248 { \
9249 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9250 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9251 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9252 } else do { } while (0)
9253#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9254 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9255 { \
9256 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9257 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9258 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9259 } else do { } while (0)
9260#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9261 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9262 { \
9263 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9264 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9265 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9266 } else do { } while (0)
9267
9268 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9269 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9270 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9271 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9272 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9273
9274 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9275 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9276 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9277 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9278 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
9279 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9280 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9281 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9282 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
9283 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9284 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
9285 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9286 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
9287 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9288 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9289 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9290 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9291 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9292 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9293 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9294 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9295 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9296 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9297 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9298 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9299 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9300 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9301 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9302 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9303 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9304 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9305 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9306 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9307 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9308 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9309 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9310
9311 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9312 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9313 {
9314 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_APIC_TPR);
9315 AssertRC(rc);
9316
9317#if 0 /** @todo fix me */
9318 pDbgState->fClearCr0Mask = true;
9319 pDbgState->fClearCr4Mask = true;
9320#endif
9321 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9322 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_STORE_EXIT | VMX_PROC_CTLS_CR8_STORE_EXIT;
9323 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9324 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_LOAD_EXIT | VMX_PROC_CTLS_CR8_LOAD_EXIT;
9325 pDbgState->fCpe1Unwanted |= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
9326 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9327 require clearing here and in the loop if we start using it. */
9328 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9329 }
9330 else
9331 {
9332 if (pDbgState->fClearCr0Mask)
9333 {
9334 pDbgState->fClearCr0Mask = false;
9335 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
9336 }
9337 if (pDbgState->fClearCr4Mask)
9338 {
9339 pDbgState->fClearCr4Mask = false;
9340 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
9341 }
9342 }
9343 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9344 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9345
9346 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9347 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9348 {
9349 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9350 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9351 }
9352 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9353 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9354
9355 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
9356 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9357 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
9358 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9359 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
9360 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9361 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
9362 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9363#if 0 /** @todo too slow, fix handler. */
9364 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
9365#endif
9366 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9367
9368 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9369 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9370 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9371 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9372 {
9373 pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
9374 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9375 }
9376 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9377 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9378 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9379 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9380
9381 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9382 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9383 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9384 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9385 {
9386 pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
9387 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9388 }
9389 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9390 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9391 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9392 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9393
9394 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9395 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9396 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
9397 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9398 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9399 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9400 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
9401 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9402 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9403 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9404 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
9405 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9406 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
9407 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9408 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9409 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9410 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
9411 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9412 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9413 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9414 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9415 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9416
9417#undef IS_EITHER_ENABLED
9418#undef SET_ONLY_XBM_IF_EITHER_EN
9419#undef SET_CPE1_XBM_IF_EITHER_EN
9420#undef SET_CPEU_XBM_IF_EITHER_EN
9421#undef SET_CPE2_XBM_IF_EITHER_EN
9422
9423 /*
9424 * Sanitize the control stuff.
9425 */
9426 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
9427 if (pDbgState->fCpe2Extra)
9428 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
9429 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
9430 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0;
9431 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
9432 {
9433 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9434 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9435 }
9436
9437 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9438 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9439 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9440 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9441}
9442
9443
9444/**
9445 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9446 * appropriate.
9447 *
9448 * The caller has checked the VM-exit against the
9449 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9450 * already, so we don't have to do that either.
9451 *
9452 * @returns Strict VBox status code (i.e. informational status codes too).
9453 * @param pVCpu The cross context virtual CPU structure.
9454 * @param pVmxTransient Pointer to the VMX-transient structure.
9455 * @param uExitReason The VM-exit reason.
9456 *
9457 * @remarks The name of this function is displayed by dtrace, so keep it short
9458 * and to the point. No longer than 33 chars long, please.
9459 */
9460static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9461{
9462 /*
9463 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9464 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9465 *
9466 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9467 * does. Must add/change/remove both places. Same ordering, please.
9468 *
9469 * Added/removed events must also be reflected in the next section
9470 * where we dispatch dtrace events.
9471 */
9472 bool fDtrace1 = false;
9473 bool fDtrace2 = false;
9474 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9475 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9476 uint32_t uEventArg = 0;
9477#define SET_EXIT(a_EventSubName) \
9478 do { \
9479 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9480 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9481 } while (0)
9482#define SET_BOTH(a_EventSubName) \
9483 do { \
9484 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9485 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9486 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9487 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9488 } while (0)
9489 switch (uExitReason)
9490 {
9491 case VMX_EXIT_MTF:
9492 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9493
9494 case VMX_EXIT_XCPT_OR_NMI:
9495 {
9496 uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9497 switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
9498 {
9499 case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
9500 case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
9501 case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
9502 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9503 {
9504 if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
9505 {
9506 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9507 uEventArg = pVmxTransient->uExitIntErrorCode;
9508 }
9509 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9510 switch (enmEvent1)
9511 {
9512 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9513 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9514 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9515 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9516 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9517 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9518 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9519 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9520 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9521 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9522 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9523 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9524 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9525 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9526 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9527 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9528 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9529 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9530 default: break;
9531 }
9532 }
9533 else
9534 AssertFailed();
9535 break;
9536
9537 case VMX_EXIT_INT_INFO_TYPE_SW_INT:
9538 uEventArg = idxVector;
9539 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9540 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9541 break;
9542 }
9543 break;
9544 }
9545
9546 case VMX_EXIT_TRIPLE_FAULT:
9547 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9548 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9549 break;
9550 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9551 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9552 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9553 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9554 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9555
9556 /* Instruction specific VM-exits: */
9557 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9558 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9559 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9560 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9561 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9562 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9563 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9564 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9565 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9566 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9567 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9568 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9569 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9570 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9571 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9572 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9573 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9574 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9575 case VMX_EXIT_MOV_CRX:
9576 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9577 if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
9578 SET_BOTH(CRX_READ);
9579 else
9580 SET_BOTH(CRX_WRITE);
9581 uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
9582 break;
9583 case VMX_EXIT_MOV_DRX:
9584 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9585 if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
9586 == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
9587 SET_BOTH(DRX_READ);
9588 else
9589 SET_BOTH(DRX_WRITE);
9590 uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
9591 break;
9592 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9593 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9594 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9595 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9596 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9597 case VMX_EXIT_XDTR_ACCESS:
9598 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9599 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
9600 {
9601 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9602 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9603 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9604 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9605 }
9606 break;
9607
9608 case VMX_EXIT_TR_ACCESS:
9609 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9610 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
9611 {
9612 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9613 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9614 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9615 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9616 }
9617 break;
9618
9619 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9620 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9621 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9622 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9623 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9624 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9625 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9626 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9627 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9628 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9629 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9630
9631 /* Events that aren't relevant at this point. */
9632 case VMX_EXIT_EXT_INT:
9633 case VMX_EXIT_INT_WINDOW:
9634 case VMX_EXIT_NMI_WINDOW:
9635 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9636 case VMX_EXIT_PREEMPT_TIMER:
9637 case VMX_EXIT_IO_INSTR:
9638 break;
9639
9640 /* Errors and unexpected events. */
9641 case VMX_EXIT_INIT_SIGNAL:
9642 case VMX_EXIT_SIPI:
9643 case VMX_EXIT_IO_SMI:
9644 case VMX_EXIT_SMI:
9645 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9646 case VMX_EXIT_ERR_MSR_LOAD:
9647 case VMX_EXIT_ERR_MACHINE_CHECK:
9648 break;
9649
9650 default:
9651 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9652 break;
9653 }
9654#undef SET_BOTH
9655#undef SET_EXIT
9656
9657 /*
9658 * Dtrace tracepoints go first. We do them here at once so we don't
9659 * have to copy the guest state saving and stuff a few dozen times.
9660 * Down side is that we've got to repeat the switch, though this time
9661 * we use enmEvent since the probes are a subset of what DBGF does.
9662 */
9663 if (fDtrace1 || fDtrace2)
9664 {
9665 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9666 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9667 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9668 switch (enmEvent1)
9669 {
9670 /** @todo consider which extra parameters would be helpful for each probe. */
9671 case DBGFEVENT_END: break;
9672 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
9673 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
9674 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
9675 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
9676 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
9677 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
9678 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
9679 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
9680 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
9681 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
9682 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
9683 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
9684 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
9685 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
9686 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
9687 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
9688 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
9689 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
9690 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9691 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9692 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
9693 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
9694 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
9695 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
9696 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
9697 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
9698 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
9699 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9700 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9701 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9702 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9703 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9704 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
9705 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9706 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
9707 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
9708 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
9709 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
9710 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
9711 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
9712 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
9713 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
9714 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
9715 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
9716 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
9717 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
9718 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
9719 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
9720 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
9721 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
9722 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
9723 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
9724 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
9725 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
9726 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
9727 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
9728 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
9729 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
9730 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
9731 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
9732 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
9733 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
9734 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
9735 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
9736 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
9737 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
9738 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9739 }
9740 switch (enmEvent2)
9741 {
9742 /** @todo consider which extra parameters would be helpful for each probe. */
9743 case DBGFEVENT_END: break;
9744 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
9745 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9746 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
9747 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
9748 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
9749 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
9750 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
9751 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
9752 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
9753 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9754 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9755 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9756 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9757 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9758 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
9759 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9760 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
9761 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
9762 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
9763 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
9764 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
9765 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
9766 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
9767 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
9768 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
9769 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
9770 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
9771 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
9772 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
9773 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
9774 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
9775 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
9776 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
9777 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
9778 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
9779 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
9780 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
9781 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
9782 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
9783 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
9784 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
9785 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
9786 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
9787 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
9788 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
9789 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
9790 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
9791 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
9792 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
9793 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
9794 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
9795 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
9796 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9797 }
9798 }
9799
9800 /*
9801 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9802 * the DBGF call will do a full check).
9803 *
9804 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9805 * Note! If we have to events, we prioritize the first, i.e. the instruction
9806 * one, in order to avoid event nesting.
9807 */
9808 PVM pVM = pVCpu->CTX_SUFF(pVM);
9809 if ( enmEvent1 != DBGFEVENT_END
9810 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9811 {
9812 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
9813 if (rcStrict != VINF_SUCCESS)
9814 return rcStrict;
9815 }
9816 else if ( enmEvent2 != DBGFEVENT_END
9817 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9818 {
9819 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
9820 if (rcStrict != VINF_SUCCESS)
9821 return rcStrict;
9822 }
9823
9824 return VINF_SUCCESS;
9825}
9826
9827
9828/**
9829 * Single-stepping VM-exit filtering.
9830 *
9831 * This is preprocessing the VM-exits and deciding whether we've gotten far
9832 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9833 * handling is performed.
9834 *
9835 * @returns Strict VBox status code (i.e. informational status codes too).
9836 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9837 * @param pVmxTransient Pointer to the VMX-transient structure.
9838 * @param pDbgState The debug state.
9839 */
9840DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
9841{
9842 /*
9843 * Expensive (saves context) generic dtrace VM-exit probe.
9844 */
9845 uint32_t const uExitReason = pVmxTransient->uExitReason;
9846 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9847 { /* more likely */ }
9848 else
9849 {
9850 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9851 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9852 AssertRC(rc);
9853 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
9854 }
9855
9856 /*
9857 * Check for host NMI, just to get that out of the way.
9858 */
9859 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9860 { /* normally likely */ }
9861 else
9862 {
9863 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9864 AssertRCReturn(rc2, rc2);
9865 uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
9866 if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
9867 return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
9868 }
9869
9870 /*
9871 * Check for single stepping event if we're stepping.
9872 */
9873 if (pVCpu->hm.s.fSingleInstruction)
9874 {
9875 switch (uExitReason)
9876 {
9877 case VMX_EXIT_MTF:
9878 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9879
9880 /* Various events: */
9881 case VMX_EXIT_XCPT_OR_NMI:
9882 case VMX_EXIT_EXT_INT:
9883 case VMX_EXIT_TRIPLE_FAULT:
9884 case VMX_EXIT_INT_WINDOW:
9885 case VMX_EXIT_NMI_WINDOW:
9886 case VMX_EXIT_TASK_SWITCH:
9887 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9888 case VMX_EXIT_APIC_ACCESS:
9889 case VMX_EXIT_EPT_VIOLATION:
9890 case VMX_EXIT_EPT_MISCONFIG:
9891 case VMX_EXIT_PREEMPT_TIMER:
9892
9893 /* Instruction specific VM-exits: */
9894 case VMX_EXIT_CPUID:
9895 case VMX_EXIT_GETSEC:
9896 case VMX_EXIT_HLT:
9897 case VMX_EXIT_INVD:
9898 case VMX_EXIT_INVLPG:
9899 case VMX_EXIT_RDPMC:
9900 case VMX_EXIT_RDTSC:
9901 case VMX_EXIT_RSM:
9902 case VMX_EXIT_VMCALL:
9903 case VMX_EXIT_VMCLEAR:
9904 case VMX_EXIT_VMLAUNCH:
9905 case VMX_EXIT_VMPTRLD:
9906 case VMX_EXIT_VMPTRST:
9907 case VMX_EXIT_VMREAD:
9908 case VMX_EXIT_VMRESUME:
9909 case VMX_EXIT_VMWRITE:
9910 case VMX_EXIT_VMXOFF:
9911 case VMX_EXIT_VMXON:
9912 case VMX_EXIT_MOV_CRX:
9913 case VMX_EXIT_MOV_DRX:
9914 case VMX_EXIT_IO_INSTR:
9915 case VMX_EXIT_RDMSR:
9916 case VMX_EXIT_WRMSR:
9917 case VMX_EXIT_MWAIT:
9918 case VMX_EXIT_MONITOR:
9919 case VMX_EXIT_PAUSE:
9920 case VMX_EXIT_XDTR_ACCESS:
9921 case VMX_EXIT_TR_ACCESS:
9922 case VMX_EXIT_INVEPT:
9923 case VMX_EXIT_RDTSCP:
9924 case VMX_EXIT_INVVPID:
9925 case VMX_EXIT_WBINVD:
9926 case VMX_EXIT_XSETBV:
9927 case VMX_EXIT_RDRAND:
9928 case VMX_EXIT_INVPCID:
9929 case VMX_EXIT_VMFUNC:
9930 case VMX_EXIT_RDSEED:
9931 case VMX_EXIT_XSAVES:
9932 case VMX_EXIT_XRSTORS:
9933 {
9934 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
9935 AssertRCReturn(rc, rc);
9936 if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
9937 || pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
9938 return VINF_EM_DBG_STEPPED;
9939 break;
9940 }
9941
9942 /* Errors and unexpected events: */
9943 case VMX_EXIT_INIT_SIGNAL:
9944 case VMX_EXIT_SIPI:
9945 case VMX_EXIT_IO_SMI:
9946 case VMX_EXIT_SMI:
9947 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9948 case VMX_EXIT_ERR_MSR_LOAD:
9949 case VMX_EXIT_ERR_MACHINE_CHECK:
9950 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9951 break;
9952
9953 default:
9954 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9955 break;
9956 }
9957 }
9958
9959 /*
9960 * Check for debugger event breakpoints and dtrace probes.
9961 */
9962 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9963 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9964 {
9965 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
9966 if (rcStrict != VINF_SUCCESS)
9967 return rcStrict;
9968 }
9969
9970 /*
9971 * Normal processing.
9972 */
9973#ifdef HMVMX_USE_FUNCTION_TABLE
9974 return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
9975#else
9976 return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
9977#endif
9978}
9979
9980
9981/**
9982 * Single steps guest code using VT-x.
9983 *
9984 * @returns Strict VBox status code (i.e. informational status codes too).
9985 * @param pVCpu The cross context virtual CPU structure.
9986 *
9987 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
9988 */
9989static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu)
9990{
9991 VMXTRANSIENT VmxTransient;
9992 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9993
9994 /* Set HMCPU indicators. */
9995 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
9996 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
9997 pVCpu->hm.s.fDebugWantRdTscExit = false;
9998 pVCpu->hm.s.fUsingDebugLoop = true;
9999
10000 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10001 VMXRUNDBGSTATE DbgState;
10002 hmR0VmxRunDebugStateInit(pVCpu, &DbgState);
10003 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
10004
10005 /*
10006 * The loop.
10007 */
10008 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10009 for (uint32_t cLoops = 0; ; cLoops++)
10010 {
10011 Assert(!HMR0SuspendPending());
10012 HMVMX_ASSERT_CPU_SAFE(pVCpu);
10013 bool fStepping = pVCpu->hm.s.fSingleInstruction;
10014
10015 /*
10016 * Preparatory work for running guest code, this may force us to return
10017 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10018 */
10019 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10020 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10021 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
10022 if (rcStrict != VINF_SUCCESS)
10023 break;
10024
10025 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10026 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10027
10028 /*
10029 * Now we can run the guest code.
10030 */
10031 int rcRun = hmR0VmxRunGuest(pVCpu);
10032
10033 /*
10034 * Restore any residual host-state and save any bits shared between host
10035 * and guest into the guest-CPU state. Re-enables interrupts!
10036 */
10037 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10038
10039 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10040 if (RT_SUCCESS(rcRun))
10041 { /* very likely */ }
10042 else
10043 {
10044 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10045 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10046 return rcRun;
10047 }
10048
10049 /* Profile the VM-exit. */
10050 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10051 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10052 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10053 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10054 HMVMX_START_EXIT_DISPATCH_PROF();
10055
10056 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10057
10058 /*
10059 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10060 */
10061 rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
10062 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10063 if (rcStrict != VINF_SUCCESS)
10064 break;
10065 if (cLoops > pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
10066 {
10067 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10068 rcStrict = VINF_EM_RAW_INTERRUPT;
10069 break;
10070 }
10071
10072 /*
10073 * Stepping: Did the RIP change, if so, consider it a single step.
10074 * Otherwise, make sure one of the TFs gets set.
10075 */
10076 if (fStepping)
10077 {
10078 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
10079 AssertRC(rc);
10080 if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
10081 || pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
10082 {
10083 rcStrict = VINF_EM_DBG_STEPPED;
10084 break;
10085 }
10086 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
10087 }
10088
10089 /*
10090 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
10091 */
10092 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10093 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
10094 }
10095
10096 /*
10097 * Clear the X86_EFL_TF if necessary.
10098 */
10099 if (pVCpu->hm.s.fClearTrapFlag)
10100 {
10101 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
10102 AssertRC(rc);
10103 pVCpu->hm.s.fClearTrapFlag = false;
10104 pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
10105 }
10106 /** @todo there seems to be issues with the resume flag when the monitor trap
10107 * flag is pending without being used. Seen early in bios init when
10108 * accessing APIC page in protected mode. */
10109
10110 /*
10111 * Restore VM-exit control settings as we may not reenter this function the
10112 * next time around.
10113 */
10114 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10115
10116 /* Restore HMCPU indicators. */
10117 pVCpu->hm.s.fUsingDebugLoop = false;
10118 pVCpu->hm.s.fDebugWantRdTscExit = false;
10119 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10120
10121 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10122 return rcStrict;
10123}
10124
10125
10126/** @} */
10127
10128
10129/**
10130 * Checks if any expensive dtrace probes are enabled and we should go to the
10131 * debug loop.
10132 *
10133 * @returns true if we should use debug loop, false if not.
10134 */
10135static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10136{
10137 /* It's probably faster to OR the raw 32-bit counter variables together.
10138 Since the variables are in an array and the probes are next to one
10139 another (more or less), we have good locality. So, better read
10140 eight-nine cache lines ever time and only have one conditional, than
10141 128+ conditionals, right? */
10142 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10143 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10144 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10145 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10146 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10147 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10148 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10149 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10150 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10151 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10152 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10153 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10154 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10155 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10156 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10157 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10158 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10159 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10160 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10161 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10162 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10163 ) != 0
10164 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10165 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10166 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10167 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10168 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10169 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10170 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10171 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10172 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10173 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10174 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10175 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10176 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10177 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10178 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10179 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10180 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10181 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10182 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10183 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10184 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10185 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10186 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10187 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10188 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10189 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10190 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10191 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10192 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10193 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10194 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10195 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10196 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10197 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10198 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10199 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10200 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10201 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10202 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10203 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10204 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10205 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10206 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10207 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10208 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10209 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10210 ) != 0
10211 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10212 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10213 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10214 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10215 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10216 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10217 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10218 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10219 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10220 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10221 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10222 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10223 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10224 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10225 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10226 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10227 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10228 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10229 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10230 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10231 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10232 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10233 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10234 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10235 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10236 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10237 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10238 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10239 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10240 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10241 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10242 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10243 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10244 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10245 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10246 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10247 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10248 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10249 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10250 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10251 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10252 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10253 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10254 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10255 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10256 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10257 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10258 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10259 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10260 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10261 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10262 ) != 0;
10263}
10264
10265
10266/**
10267 * Runs the guest code using VT-x.
10268 *
10269 * @returns Strict VBox status code (i.e. informational status codes too).
10270 * @param pVCpu The cross context virtual CPU structure.
10271 */
10272VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
10273{
10274 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10275 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10276 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10277 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
10278
10279 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10280
10281 VBOXSTRICTRC rcStrict;
10282 if ( !pVCpu->hm.s.fUseDebugLoop
10283 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10284 && !DBGFIsStepping(pVCpu)
10285 && !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
10286 rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu);
10287 else
10288 rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu);
10289
10290 if (rcStrict == VERR_EM_INTERPRETER)
10291 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10292 else if (rcStrict == VINF_EM_RESET)
10293 rcStrict = VINF_EM_TRIPLE_FAULT;
10294
10295 int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
10296 if (RT_FAILURE(rc2))
10297 {
10298 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10299 rcStrict = rc2;
10300 }
10301 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10302 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10303 return rcStrict;
10304}
10305
10306
10307#ifndef HMVMX_USE_FUNCTION_TABLE
10308DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10309{
10310#ifdef DEBUG_ramshankar
10311#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
10312 do { \
10313 if (a_fSave != 0) \
10314 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
10315 VBOXSTRICTRC rcStrict = a_CallExpr; \
10316 if (a_fSave != 0) \
10317 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
10318 return rcStrict; \
10319 } while (0)
10320#else
10321# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
10322#endif
10323 switch (rcReason)
10324 {
10325 case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
10326 case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
10327 case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
10328 case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
10329 case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
10330 case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
10331 case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
10332 case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
10333 case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
10334 case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
10335 case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
10336 case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
10337 case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
10338 case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
10339 case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
10340 case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
10341 case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
10342 case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
10343 case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
10344 case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
10345 case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
10346 case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
10347 case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
10348 case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pVmxTransient));
10349 case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
10350 case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
10351 case VMX_EXIT_XDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10352 case VMX_EXIT_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10353 case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
10354 case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
10355 case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
10356 case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
10357 case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
10358 case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
10359
10360 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
10361 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
10362 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pVmxTransient);
10363 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pVmxTransient);
10364 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pVmxTransient);
10365 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pVmxTransient);
10366 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pVmxTransient);
10367 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
10368 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pVmxTransient);
10369
10370 case VMX_EXIT_VMCLEAR:
10371 case VMX_EXIT_VMLAUNCH:
10372 case VMX_EXIT_VMPTRLD:
10373 case VMX_EXIT_VMPTRST:
10374 case VMX_EXIT_VMREAD:
10375 case VMX_EXIT_VMRESUME:
10376 case VMX_EXIT_VMWRITE:
10377 case VMX_EXIT_VMXOFF:
10378 case VMX_EXIT_VMXON:
10379 case VMX_EXIT_INVEPT:
10380 case VMX_EXIT_INVVPID:
10381 case VMX_EXIT_VMFUNC:
10382 case VMX_EXIT_XSAVES:
10383 case VMX_EXIT_XRSTORS:
10384 return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
10385
10386 case VMX_EXIT_ENCLS:
10387 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10388 case VMX_EXIT_PML_FULL:
10389 default:
10390 return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
10391 }
10392#undef VMEXIT_CALL_RET
10393}
10394#endif /* !HMVMX_USE_FUNCTION_TABLE */
10395
10396
10397#ifdef VBOX_STRICT
10398/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10399# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10400 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10401
10402# define HMVMX_ASSERT_PREEMPT_CPUID() \
10403 do { \
10404 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10405 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10406 } while (0)
10407
10408# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10409 do { \
10410 AssertPtr((a_pVCpu)); \
10411 AssertPtr((a_pVmxTransient)); \
10412 Assert((a_pVmxTransient)->fVMEntryFailed == false); \
10413 Assert(ASMIntAreEnabled()); \
10414 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10415 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10416 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
10417 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10418 if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
10419 HMVMX_ASSERT_PREEMPT_CPUID(); \
10420 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10421 } while (0)
10422
10423# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10424 do { \
10425 Log4Func(("\n")); \
10426 } while (0)
10427#else
10428# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10429 do { \
10430 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10431 NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
10432 } while (0)
10433# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
10434#endif
10435
10436
10437/**
10438 * Advances the guest RIP by the specified number of bytes.
10439 *
10440 * @param pVCpu The cross context virtual CPU structure.
10441 * @param cbInstr Number of bytes to advance the RIP by.
10442 *
10443 * @remarks No-long-jump zone!!!
10444 */
10445DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
10446{
10447 /* Advance the RIP. */
10448 pVCpu->cpum.GstCtx.rip += cbInstr;
10449 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
10450
10451 /* Update interrupt inhibition. */
10452 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10453 && pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
10454 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10455}
10456
10457
10458/**
10459 * Advances the guest RIP after reading it from the VMCS.
10460 *
10461 * @returns VBox status code, no informational status codes.
10462 * @param pVCpu The cross context virtual CPU structure.
10463 * @param pVmxTransient Pointer to the VMX transient structure.
10464 *
10465 * @remarks No-long-jump zone!!!
10466 */
10467static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10468{
10469 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10470 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS);
10471 AssertRCReturn(rc, rc);
10472
10473 hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
10474
10475 /*
10476 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10477 * pending debug exception field as it takes care of priority of events.
10478 *
10479 * See Intel spec. 32.2.1 "Debug Exceptions".
10480 */
10481 if ( !pVCpu->hm.s.fSingleInstruction
10482 && pVCpu->cpum.GstCtx.eflags.Bits.u1TF)
10483 {
10484 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10485 AssertRCReturn(rc, rc);
10486 }
10487
10488 return VINF_SUCCESS;
10489}
10490
10491
10492/**
10493 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10494 * and update error record fields accordingly.
10495 *
10496 * @return VMX_IGS_* return codes.
10497 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10498 * wrong with the guest state.
10499 *
10500 * @param pVCpu The cross context virtual CPU structure.
10501 *
10502 * @remarks This function assumes our cache of the VMCS controls
10503 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10504 */
10505static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu)
10506{
10507#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10508#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10509 uError = (err); \
10510 break; \
10511 } else do { } while (0)
10512
10513 int rc;
10514 PVM pVM = pVCpu->CTX_SUFF(pVM);
10515 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10516 uint32_t uError = VMX_IGS_ERROR;
10517 uint32_t u32Val;
10518 bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10519
10520 do
10521 {
10522 /*
10523 * CR0.
10524 */
10525 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10526 uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10527 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10528 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10529 if (fUnrestrictedGuest)
10530 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
10531
10532 uint32_t u32GuestCr0;
10533 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10534 AssertRCBreak(rc);
10535 HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10536 HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10537 if ( !fUnrestrictedGuest
10538 && (u32GuestCr0 & X86_CR0_PG)
10539 && !(u32GuestCr0 & X86_CR0_PE))
10540 {
10541 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10542 }
10543
10544 /*
10545 * CR4.
10546 */
10547 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10548 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10549
10550 uint32_t u32GuestCr4;
10551 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10552 AssertRCBreak(rc);
10553 HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10554 HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10555
10556 /*
10557 * IA32_DEBUGCTL MSR.
10558 */
10559 uint64_t u64Val;
10560 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10561 AssertRCBreak(rc);
10562 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10563 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10564 {
10565 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10566 }
10567 uint64_t u64DebugCtlMsr = u64Val;
10568
10569#ifdef VBOX_STRICT
10570 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10571 AssertRCBreak(rc);
10572 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10573#endif
10574 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
10575
10576 /*
10577 * RIP and RFLAGS.
10578 */
10579 uint32_t u32Eflags;
10580#if HC_ARCH_BITS == 64
10581 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10582 AssertRCBreak(rc);
10583 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10584 if ( !fLongModeGuest
10585 || !pCtx->cs.Attr.n.u1Long)
10586 {
10587 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10588 }
10589 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10590 * must be identical if the "IA-32e mode guest" VM-entry
10591 * control is 1 and CS.L is 1. No check applies if the
10592 * CPU supports 64 linear-address bits. */
10593
10594 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10595 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10596 AssertRCBreak(rc);
10597 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10598 VMX_IGS_RFLAGS_RESERVED);
10599 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10600 u32Eflags = u64Val;
10601#else
10602 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10603 AssertRCBreak(rc);
10604 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10605 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10606#endif
10607
10608 if ( fLongModeGuest
10609 || ( fUnrestrictedGuest
10610 && !(u32GuestCr0 & X86_CR0_PE)))
10611 {
10612 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10613 }
10614
10615 uint32_t u32EntryInfo;
10616 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10617 AssertRCBreak(rc);
10618 if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
10619 && VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
10620 {
10621 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10622 }
10623
10624 /*
10625 * 64-bit checks.
10626 */
10627#if HC_ARCH_BITS == 64
10628 if (fLongModeGuest)
10629 {
10630 HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10631 HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10632 }
10633
10634 if ( !fLongModeGuest
10635 && (u32GuestCr4 & X86_CR4_PCIDE))
10636 {
10637 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10638 }
10639
10640 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10641 * 51:32 beyond the processor's physical-address width are 0. */
10642
10643 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10644 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10645 {
10646 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10647 }
10648
10649 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10650 AssertRCBreak(rc);
10651 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10652
10653 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10654 AssertRCBreak(rc);
10655 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10656#endif
10657
10658 /*
10659 * PERF_GLOBAL MSR.
10660 */
10661 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
10662 {
10663 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10664 AssertRCBreak(rc);
10665 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10666 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10667 }
10668
10669 /*
10670 * PAT MSR.
10671 */
10672 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
10673 {
10674 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10675 AssertRCBreak(rc);
10676 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10677 for (unsigned i = 0; i < 8; i++)
10678 {
10679 uint8_t u8Val = (u64Val & 0xff);
10680 if ( u8Val != 0 /* UC */
10681 && u8Val != 1 /* WC */
10682 && u8Val != 4 /* WT */
10683 && u8Val != 5 /* WP */
10684 && u8Val != 6 /* WB */
10685 && u8Val != 7 /* UC- */)
10686 {
10687 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10688 }
10689 u64Val >>= 8;
10690 }
10691 }
10692
10693 /*
10694 * EFER MSR.
10695 */
10696 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
10697 {
10698 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10699 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10700 AssertRCBreak(rc);
10701 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10702 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10703 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10704 & VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
10705 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10706 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10707 || !(u32GuestCr0 & X86_CR0_PG)
10708 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10709 VMX_IGS_EFER_LMA_LME_MISMATCH);
10710 }
10711
10712 /*
10713 * Segment registers.
10714 */
10715 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10716 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10717 if (!(u32Eflags & X86_EFL_VM))
10718 {
10719 /* CS */
10720 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10721 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10722 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10723 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10724 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10725 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10726 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10727 /* CS cannot be loaded with NULL in protected mode. */
10728 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10729 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10730 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10731 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10732 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10733 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10734 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10735 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10736 else
10737 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10738
10739 /* SS */
10740 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10741 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10742 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10743 if ( !(pCtx->cr0 & X86_CR0_PE)
10744 || pCtx->cs.Attr.n.u4Type == 3)
10745 {
10746 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10747 }
10748 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10749 {
10750 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10751 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10752 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10753 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10754 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10755 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10756 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10757 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10758 }
10759
10760 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmenReg(). */
10761 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10762 {
10763 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10764 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10765 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10766 || pCtx->ds.Attr.n.u4Type > 11
10767 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10768 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10769 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10770 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10771 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10772 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10773 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10774 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10775 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10776 }
10777 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10778 {
10779 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10780 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10781 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10782 || pCtx->es.Attr.n.u4Type > 11
10783 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10784 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10785 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10786 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10787 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10788 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10789 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10790 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10791 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10792 }
10793 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10794 {
10795 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10796 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10797 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10798 || pCtx->fs.Attr.n.u4Type > 11
10799 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10800 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10801 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10802 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10803 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10804 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10805 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10806 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10807 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10808 }
10809 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10810 {
10811 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10812 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10813 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10814 || pCtx->gs.Attr.n.u4Type > 11
10815 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10816 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10817 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10818 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10819 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10820 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10821 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10822 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10823 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10824 }
10825 /* 64-bit capable CPUs. */
10826#if HC_ARCH_BITS == 64
10827 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10828 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10829 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10830 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10831 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10832 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10833 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10834 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10835 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10836 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10837 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10838#endif
10839 }
10840 else
10841 {
10842 /* V86 mode checks. */
10843 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10844 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10845 {
10846 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10847 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10848 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10849 }
10850 else
10851 {
10852 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10853 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10854 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10855 }
10856
10857 /* CS */
10858 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10859 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10860 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10861 /* SS */
10862 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10863 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10864 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10865 /* DS */
10866 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10867 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10868 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10869 /* ES */
10870 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10871 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10872 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10873 /* FS */
10874 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10875 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10876 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10877 /* GS */
10878 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10879 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10880 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10881 /* 64-bit capable CPUs. */
10882#if HC_ARCH_BITS == 64
10883 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10884 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10885 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10886 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10887 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10888 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10889 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10890 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10891 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10892 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10893 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10894#endif
10895 }
10896
10897 /*
10898 * TR.
10899 */
10900 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10901 /* 64-bit capable CPUs. */
10902#if HC_ARCH_BITS == 64
10903 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10904#endif
10905 if (fLongModeGuest)
10906 {
10907 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10908 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10909 }
10910 else
10911 {
10912 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10913 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10914 VMX_IGS_TR_ATTR_TYPE_INVALID);
10915 }
10916 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10917 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10918 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10919 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10920 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10921 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10922 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10923 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10924
10925 /*
10926 * GDTR and IDTR.
10927 */
10928#if HC_ARCH_BITS == 64
10929 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10930 AssertRCBreak(rc);
10931 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10932
10933 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10934 AssertRCBreak(rc);
10935 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10936#endif
10937
10938 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10939 AssertRCBreak(rc);
10940 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10941
10942 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10943 AssertRCBreak(rc);
10944 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10945
10946 /*
10947 * Guest Non-Register State.
10948 */
10949 /* Activity State. */
10950 uint32_t u32ActivityState;
10951 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10952 AssertRCBreak(rc);
10953 HMVMX_CHECK_BREAK( !u32ActivityState
10954 || (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
10955 VMX_IGS_ACTIVITY_STATE_INVALID);
10956 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10957 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10958 uint32_t u32IntrState;
10959 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
10960 AssertRCBreak(rc);
10961 if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
10962 || u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10963 {
10964 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10965 }
10966
10967 /** @todo Activity state and injecting interrupts. Left as a todo since we
10968 * currently don't use activity states but ACTIVE. */
10969
10970 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10971 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10972
10973 /* Guest interruptibility-state. */
10974 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10975 HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
10976 != (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10977 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10978 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10979 || !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
10980 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10981 if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
10982 {
10983 if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
10984 {
10985 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10986 && !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10987 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10988 }
10989 else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
10990 {
10991 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10992 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10993 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
10994 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10995 }
10996 }
10997 /** @todo Assumes the processor is not in SMM. */
10998 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
10999 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11000 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
11001 || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
11002 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11003 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
11004 && VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
11005 && VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
11006 {
11007 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
11008 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11009 }
11010
11011 /* Pending debug exceptions. */
11012#if HC_ARCH_BITS == 64
11013 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
11014 AssertRCBreak(rc);
11015 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11016 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11017 u32Val = u64Val; /* For pending debug exceptions checks below. */
11018#else
11019 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
11020 AssertRCBreak(rc);
11021 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11022 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11023#endif
11024
11025 if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
11026 || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
11027 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11028 {
11029 if ( (u32Eflags & X86_EFL_TF)
11030 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11031 {
11032 /* Bit 14 is PendingDebug.BS. */
11033 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11034 }
11035 if ( !(u32Eflags & X86_EFL_TF)
11036 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11037 {
11038 /* Bit 14 is PendingDebug.BS. */
11039 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11040 }
11041 }
11042
11043 /* VMCS link pointer. */
11044 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11045 AssertRCBreak(rc);
11046 if (u64Val != UINT64_C(0xffffffffffffffff))
11047 {
11048 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11049 /** @todo Bits beyond the processor's physical-address width MBZ. */
11050 /** @todo 32-bit located in memory referenced by value of this field (as a
11051 * physical address) must contain the processor's VMCS revision ID. */
11052 /** @todo SMM checks. */
11053 }
11054
11055 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11056 * not using Nested Paging? */
11057 if ( pVM->hm.s.fNestedPaging
11058 && !fLongModeGuest
11059 && CPUMIsGuestInPAEModeEx(pCtx))
11060 {
11061 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11062 AssertRCBreak(rc);
11063 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11064
11065 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11066 AssertRCBreak(rc);
11067 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11068
11069 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11070 AssertRCBreak(rc);
11071 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11072
11073 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11074 AssertRCBreak(rc);
11075 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11076 }
11077
11078 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11079 if (uError == VMX_IGS_ERROR)
11080 uError = VMX_IGS_REASON_NOT_FOUND;
11081 } while (0);
11082
11083 pVCpu->hm.s.u32HMError = uError;
11084 return uError;
11085
11086#undef HMVMX_ERROR_BREAK
11087#undef HMVMX_CHECK_BREAK
11088}
11089
11090
11091/** @name VM-exit handlers.
11092 * @{
11093 */
11094/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11095/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11096/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11097
11098/**
11099 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11100 */
11101HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11102{
11103 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11104 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11105 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11106 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11107 return VINF_SUCCESS;
11108 return VINF_EM_RAW_INTERRUPT;
11109}
11110
11111
11112/**
11113 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11114 */
11115HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11116{
11117 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11118 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11119
11120 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11121 AssertRCReturn(rc, rc);
11122
11123 uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
11124 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
11125 && uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
11126 Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11127
11128 if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
11129 {
11130 /*
11131 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
11132 * injected it ourselves and anything we inject is not going to cause a VM-exit directly
11133 * for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
11134 *
11135 * [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11136 * [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
11137 */
11138 VMXDispatchHostNmi();
11139 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11140 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11141 return VINF_SUCCESS;
11142 }
11143
11144 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11145 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
11146 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11147 { /* likely */ }
11148 else
11149 {
11150 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11151 rcStrictRc1 = VINF_SUCCESS;
11152 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11153 return rcStrictRc1;
11154 }
11155
11156 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11157 uint32_t uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
11158 switch (uIntType)
11159 {
11160 case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11161 Assert(uVector == X86_XCPT_DB);
11162 RT_FALL_THRU();
11163 case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11164 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
11165 RT_FALL_THRU();
11166 case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11167 {
11168 /*
11169 * If there's any exception caused as a result of event injection, the resulting
11170 * secondary/final execption will be pending, we shall continue guest execution
11171 * after injecting the event. The page-fault case is complicated and we manually
11172 * handle any currently pending event in hmR0VmxExitXcptPF.
11173 */
11174 if (!pVCpu->hm.s.Event.fPending)
11175 { /* likely */ }
11176 else if (uVector != X86_XCPT_PF)
11177 {
11178 rc = VINF_SUCCESS;
11179 break;
11180 }
11181
11182 switch (uVector)
11183 {
11184 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
11185 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
11186 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
11187 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
11188 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
11189 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
11190
11191 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11192 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11193 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11194 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11195 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11196 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11197 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11198 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11199 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11200 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11201 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11202 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11203 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11204 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11205 default:
11206 {
11207 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11208 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11209 {
11210 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11211 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11212 Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
11213
11214 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
11215 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11216 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11217 AssertRCReturn(rc, rc);
11218 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11219 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11220 0 /* GCPtrFaultAddress */);
11221 }
11222 else
11223 {
11224 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11225 pVCpu->hm.s.u32HMError = uVector;
11226 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11227 }
11228 break;
11229 }
11230 }
11231 break;
11232 }
11233
11234 default:
11235 {
11236 pVCpu->hm.s.u32HMError = uExitIntInfo;
11237 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11238 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
11239 break;
11240 }
11241 }
11242 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11243 return rc;
11244}
11245
11246
11247/**
11248 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11249 */
11250HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11251{
11252 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11253
11254 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11255 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11256
11257 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11258 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11259 return VINF_SUCCESS;
11260}
11261
11262
11263/**
11264 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11265 */
11266HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11267{
11268 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11269 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)))
11270 {
11271 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11272 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11273 }
11274
11275 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11276
11277 /*
11278 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11279 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11280 */
11281 uint32_t fIntrState = 0;
11282 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
11283 AssertRCReturn(rc, rc);
11284
11285 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
11286 if ( fBlockSti
11287 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11288 {
11289 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11290 }
11291
11292 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11293 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11294
11295 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11296 return VINF_SUCCESS;
11297}
11298
11299
11300/**
11301 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11302 */
11303HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11304{
11305 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11306 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11307}
11308
11309
11310/**
11311 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11312 */
11313HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11314{
11315 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11316 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11317}
11318
11319
11320/**
11321 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11322 */
11323HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11324{
11325 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11326
11327 /*
11328 * Get the state we need and update the exit history entry.
11329 */
11330 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11331 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX);
11332 AssertRCReturn(rc, rc);
11333
11334 VBOXSTRICTRC rcStrict;
11335 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11336 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
11337 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11338 if (!pExitRec)
11339 {
11340 /*
11341 * Regular CPUID instruction execution.
11342 */
11343 rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
11344 if (rcStrict == VINF_SUCCESS)
11345 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_RAX
11346 | HM_CHANGED_GUEST_RCX | HM_CHANGED_GUEST_RDX | HM_CHANGED_GUEST_RBX);
11347 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11348 {
11349 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11350 rcStrict = VINF_SUCCESS;
11351 }
11352 }
11353 else
11354 {
11355 /*
11356 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
11357 */
11358 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11359 AssertRCReturn(rc2, rc2);
11360
11361 Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11362 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11363
11364 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11365 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
11366
11367 Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11368 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11369 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11370 }
11371 return rcStrict;
11372}
11373
11374
11375/**
11376 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11377 */
11378HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11379{
11380 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11381 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4);
11382 AssertRCReturn(rc, rc);
11383
11384 if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
11385 return VINF_EM_RAW_EMULATE_INSTR;
11386
11387 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11388 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11389}
11390
11391
11392/**
11393 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11394 */
11395HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11396{
11397 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11398 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11399 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11400 AssertRCReturn(rc, rc);
11401
11402 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11403 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11404 {
11405 /* If we get a spurious VM-exit when offsetting is enabled,
11406 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11407 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
11408 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11409 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11410 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX);
11411 }
11412 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11413 {
11414 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11415 rcStrict = VINF_SUCCESS;
11416 }
11417 return rcStrict;
11418}
11419
11420
11421/**
11422 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11423 */
11424HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11425{
11426 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11427 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_TSC_AUX);
11428 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11429 AssertRCReturn(rc, rc);
11430
11431 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11432 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11433 {
11434 /* If we get a spurious VM-exit when offsetting is enabled,
11435 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11436 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
11437 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11438 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11439 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX | HM_CHANGED_GUEST_RCX);
11440 }
11441 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11442 {
11443 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11444 rcStrict = VINF_SUCCESS;
11445 }
11446 return rcStrict;
11447}
11448
11449
11450/**
11451 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11452 */
11453HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11454{
11455 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11456 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11457 AssertRCReturn(rc, rc);
11458
11459 PVM pVM = pVCpu->CTX_SUFF(pVM);
11460 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11461 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11462 if (RT_LIKELY(rc == VINF_SUCCESS))
11463 {
11464 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11465 Assert(pVmxTransient->cbInstr == 2);
11466 }
11467 else
11468 {
11469 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11470 rc = VERR_EM_INTERPRETER;
11471 }
11472 return rc;
11473}
11474
11475
11476/**
11477 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11478 */
11479HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11480{
11481 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11482
11483 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11484 if (EMAreHypercallInstructionsEnabled(pVCpu))
11485 {
11486 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_SS
11487 | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_EFER);
11488 AssertRCReturn(rc, rc);
11489
11490 /* Perform the hypercall. */
11491 rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
11492 if (rcStrict == VINF_SUCCESS)
11493 {
11494 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11495 AssertRCReturn(rc, rc);
11496 }
11497 else
11498 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11499 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11500 || RT_FAILURE(rcStrict));
11501
11502 /* If the hypercall changes anything other than guest's general-purpose registers,
11503 we would need to reload the guest changed bits here before VM-entry. */
11504 }
11505 else
11506 Log4Func(("Hypercalls not enabled\n"));
11507
11508 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11509 if (RT_FAILURE(rcStrict))
11510 {
11511 hmR0VmxSetPendingXcptUD(pVCpu);
11512 rcStrict = VINF_SUCCESS;
11513 }
11514
11515 return rcStrict;
11516}
11517
11518
11519/**
11520 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11521 */
11522HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11523{
11524 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11525 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11526
11527 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11528 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11529 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_DS);
11530 AssertRCReturn(rc, rc);
11531
11532 VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
11533
11534 if (rcStrict == VINF_SUCCESS || rcStrict == VINF_PGM_SYNC_CR3)
11535 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11536 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11537 {
11538 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11539 rcStrict = VINF_SUCCESS;
11540 }
11541 else
11542 AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) sttus: %Rrc\n", pVmxTransient->uExitQual,
11543 VBOXSTRICTRC_VAL(rcStrict)));
11544 return rcStrict;
11545}
11546
11547
11548/**
11549 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11550 */
11551HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11552{
11553 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11554 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11555 AssertRCReturn(rc, rc);
11556
11557 PVM pVM = pVCpu->CTX_SUFF(pVM);
11558 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11559 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11560 if (RT_LIKELY(rc == VINF_SUCCESS))
11561 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11562 else
11563 {
11564 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11565 rc = VERR_EM_INTERPRETER;
11566 }
11567 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11568 return rc;
11569}
11570
11571
11572/**
11573 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11574 */
11575HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11576{
11577 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11578 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11579 AssertRCReturn(rc, rc);
11580
11581 PVM pVM = pVCpu->CTX_SUFF(pVM);
11582 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11583 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11584 rc = VBOXSTRICTRC_VAL(rc2);
11585 if (RT_LIKELY( rc == VINF_SUCCESS
11586 || rc == VINF_EM_HALT))
11587 {
11588 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11589 AssertRCReturn(rc3, rc3);
11590
11591 if ( rc == VINF_EM_HALT
11592 && EMMonitorWaitShouldContinue(pVCpu, pCtx))
11593 rc = VINF_SUCCESS;
11594 }
11595 else
11596 {
11597 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11598 rc = VERR_EM_INTERPRETER;
11599 }
11600 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11601 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11602 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11603 return rc;
11604}
11605
11606
11607/**
11608 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11609 */
11610HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11611{
11612 /*
11613 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
11614 * mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
11615 * treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
11616 * VMX root operation. If we get here, something funny is going on.
11617 *
11618 * See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
11619 */
11620 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11621 AssertMsgFailed(("Unexpected RSM VM-exit\n"));
11622 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11623}
11624
11625
11626/**
11627 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11628 */
11629HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11630{
11631 /*
11632 * This can only happen if we support dual-monitor treatment of SMI, which can be activated
11633 * by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
11634 * this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
11635 * an SMI. If we get here, something funny is going on.
11636 *
11637 * See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11638 * See Intel spec. 25.3 "Other Causes of VM-Exits"
11639 */
11640 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11641 AssertMsgFailed(("Unexpected SMI VM-exit\n"));
11642 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11643}
11644
11645
11646/**
11647 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11648 */
11649HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11650{
11651 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11652 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11653 AssertMsgFailed(("Unexpected IO SMI VM-exit\n"));
11654 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11655}
11656
11657
11658/**
11659 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11660 */
11661HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11662{
11663 /*
11664 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
11665 * We don't make use of it as our guests don't have direct access to the host LAPIC.
11666 * See Intel spec. 25.3 "Other Causes of VM-exits".
11667 */
11668 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11669 AssertMsgFailed(("Unexpected SIPI VM-exit\n"));
11670 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11671}
11672
11673
11674/**
11675 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11676 * VM-exit.
11677 */
11678HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11679{
11680 /*
11681 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11682 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11683 *
11684 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11685 * See Intel spec. "23.8 Restrictions on VMX operation".
11686 */
11687 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11688 return VINF_SUCCESS;
11689}
11690
11691
11692/**
11693 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11694 * VM-exit.
11695 */
11696HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11697{
11698 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11699 return VINF_EM_RESET;
11700}
11701
11702
11703/**
11704 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11705 */
11706HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11707{
11708 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11709 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_HLT_EXIT);
11710
11711 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11712 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
11713 AssertRCReturn(rc, rc);
11714
11715 if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
11716 rc = VINF_SUCCESS;
11717 else
11718 rc = VINF_EM_HALT;
11719
11720 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11721 if (rc != VINF_SUCCESS)
11722 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11723 return rc;
11724}
11725
11726
11727/**
11728 * VM-exit handler for instructions that result in a \#UD exception delivered to
11729 * the guest.
11730 */
11731HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11732{
11733 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11734 hmR0VmxSetPendingXcptUD(pVCpu);
11735 return VINF_SUCCESS;
11736}
11737
11738
11739/**
11740 * VM-exit handler for expiry of the VMX preemption timer.
11741 */
11742HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11743{
11744 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11745
11746 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11747 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11748
11749 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11750 PVM pVM = pVCpu->CTX_SUFF(pVM);
11751 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11752 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11753 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11754}
11755
11756
11757/**
11758 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11759 */
11760HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11761{
11762 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11763
11764 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11765 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_CR4);
11766 AssertRCReturn(rc, rc);
11767
11768 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11769 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11770 : HM_CHANGED_RAISED_XCPT_MASK);
11771
11772 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11773 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
11774
11775 return rcStrict;
11776}
11777
11778
11779/**
11780 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11781 */
11782HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11783{
11784 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11785 /** @todo Use VM-exit instruction information. */
11786 return VERR_EM_INTERPRETER;
11787}
11788
11789
11790/**
11791 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11792 * Error VM-exit.
11793 */
11794HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11795{
11796 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11797 AssertRCReturn(rc, rc);
11798 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11799 if (RT_FAILURE(rc))
11800 return rc;
11801
11802 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
11803 NOREF(uInvalidReason);
11804
11805#ifdef VBOX_STRICT
11806 uint32_t fIntrState;
11807 RTHCUINTREG uHCReg;
11808 uint64_t u64Val;
11809 uint32_t u32Val;
11810
11811 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11812 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11813 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11814 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
11815 AssertRCReturn(rc, rc);
11816
11817 Log4(("uInvalidReason %u\n", uInvalidReason));
11818 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11819 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11820 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11821 Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
11822
11823 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11824 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11825 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11826 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11827 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11828 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11829 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11830 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11831 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11832 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11833 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11834 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11835
11836 hmR0DumpRegs(pVCpu);
11837#else
11838 NOREF(pVmxTransient);
11839#endif
11840
11841 return VERR_VMX_INVALID_GUEST_STATE;
11842}
11843
11844
11845/**
11846 * VM-exit handler for VM-entry failure due to an MSR-load
11847 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11848 */
11849HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11850{
11851 AssertMsgFailed(("Unexpected MSR-load exit\n"));
11852 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11853}
11854
11855
11856/**
11857 * VM-exit handler for VM-entry failure due to a machine-check event
11858 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11859 */
11860HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11861{
11862 AssertMsgFailed(("Unexpected machine-check event exit\n"));
11863 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11864}
11865
11866
11867/**
11868 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11869 * theory.
11870 */
11871HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11872{
11873 RT_NOREF2(pVCpu, pVmxTransient);
11874 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d\n", pVmxTransient->uExitReason));
11875 return VERR_VMX_UNDEFINED_EXIT_CODE;
11876}
11877
11878
11879/**
11880 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11881 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11882 * Conditional VM-exit.
11883 */
11884HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11885{
11886 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11887
11888 /* By default, we don't enable VMX_PROC_CTLS2_DESCRIPTOR_TABLE_EXIT. */
11889 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11890 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_DESC_TABLE_EXIT)
11891 return VERR_EM_INTERPRETER;
11892 AssertMsgFailed(("Unexpected XDTR access\n"));
11893 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11894}
11895
11896
11897/**
11898 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11899 */
11900HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11901{
11902 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11903
11904 /* By default, we don't enable VMX_PROC_CTLS2_RDRAND_EXIT. */
11905 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_RDRAND_EXIT)
11906 return VERR_EM_INTERPRETER;
11907 AssertMsgFailed(("Unexpected RDRAND exit\n"));
11908 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11909}
11910
11911
11912/**
11913 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11914 */
11915HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11916{
11917 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11918
11919 /** @todo Optimize this: We currently drag in in the whole MSR state
11920 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11921 * MSRs required. That would require changes to IEM and possibly CPUM too.
11922 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11923 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; NOREF(idMsr); /* Save it. */
11924 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11925 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
11926 AssertRCReturn(rc, rc);
11927
11928 Log4Func(("ecx=%#RX32\n", idMsr));
11929
11930#ifdef VBOX_STRICT
11931 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
11932 {
11933 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr)
11934 && idMsr != MSR_K6_EFER)
11935 {
11936 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
11937 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11938 }
11939 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11940 {
11941 VMXMSREXITREAD enmRead;
11942 VMXMSREXITWRITE enmWrite;
11943 int rc2 = hmR0VmxGetMsrPermission(pVCpu, idMsr, &enmRead, &enmWrite);
11944 AssertRCReturn(rc2, rc2);
11945 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11946 {
11947 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
11948 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11949 }
11950 }
11951 }
11952#endif
11953
11954 VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
11955 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11956 if (rcStrict == VINF_SUCCESS)
11957 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11958 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX);
11959 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11960 {
11961 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11962 rcStrict = VINF_SUCCESS;
11963 }
11964 else
11965 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11966
11967 return rcStrict;
11968}
11969
11970
11971/**
11972 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11973 */
11974HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11975{
11976 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11977
11978 /** @todo Optimize this: We currently drag in in the whole MSR state
11979 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11980 * MSRs required. That would require changes to IEM and possibly CPUM too.
11981 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11982 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; /* Save it. */
11983 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11984 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
11985 AssertRCReturn(rc, rc);
11986
11987 Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
11988
11989 VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
11990 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
11991
11992 if (rcStrict == VINF_SUCCESS)
11993 {
11994 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11995
11996 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
11997 if ( idMsr == MSR_IA32_APICBASE
11998 || ( idMsr >= MSR_IA32_X2APIC_START
11999 && idMsr <= MSR_IA32_X2APIC_END))
12000 {
12001 /*
12002 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12003 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before IEM changes it.
12004 */
12005 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
12006 }
12007 else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12008 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12009 else if (idMsr == MSR_K6_EFER)
12010 {
12011 /*
12012 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12013 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12014 * the other bits as well, SCE and NXE. See @bugref{7368}.
12015 */
12016 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS
12017 | HM_CHANGED_VMX_EXIT_CTLS);
12018 }
12019
12020 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12021 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
12022 {
12023 switch (idMsr)
12024 {
12025 case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
12026 case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
12027 case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
12028 case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
12029 case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
12030 case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
12031 default:
12032 {
12033 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
12034 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12035 else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
12036 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
12037 break;
12038 }
12039 }
12040 }
12041#ifdef VBOX_STRICT
12042 else
12043 {
12044 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12045 switch (idMsr)
12046 {
12047 case MSR_IA32_SYSENTER_CS:
12048 case MSR_IA32_SYSENTER_EIP:
12049 case MSR_IA32_SYSENTER_ESP:
12050 case MSR_K8_FS_BASE:
12051 case MSR_K8_GS_BASE:
12052 {
12053 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
12054 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12055 }
12056
12057 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12058 default:
12059 {
12060 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
12061 {
12062 /* EFER writes are always intercepted, see hmR0VmxExportGuestMsrs(). */
12063 if (idMsr != MSR_K6_EFER)
12064 {
12065 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12066 idMsr));
12067 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12068 }
12069 }
12070
12071 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
12072 {
12073 VMXMSREXITREAD enmRead;
12074 VMXMSREXITWRITE enmWrite;
12075 int rc2 = hmR0VmxGetMsrPermission(pVCpu, idMsr, &enmRead, &enmWrite);
12076 AssertRCReturn(rc2, rc2);
12077 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12078 {
12079 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
12080 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12081 }
12082 }
12083 break;
12084 }
12085 }
12086 }
12087#endif /* VBOX_STRICT */
12088 }
12089 else if (rcStrict == VINF_IEM_RAISED_XCPT)
12090 {
12091 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
12092 rcStrict = VINF_SUCCESS;
12093 }
12094 else
12095 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12096
12097 return rcStrict;
12098}
12099
12100
12101/**
12102 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12103 */
12104HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12105{
12106 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12107 /** @todo The guest has likely hit a contended spinlock. We might want to
12108 * poke a schedule different guest VCPU. */
12109 return VINF_EM_RAW_INTERRUPT;
12110}
12111
12112
12113/**
12114 * VM-exit handler for when the TPR value is lowered below the specified
12115 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12116 */
12117HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12118{
12119 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12120 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
12121
12122 /*
12123 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12124 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12125 */
12126 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12127 return VINF_SUCCESS;
12128}
12129
12130
12131/**
12132 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12133 * VM-exit.
12134 *
12135 * @retval VINF_SUCCESS when guest execution can continue.
12136 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12137 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12138 * interpreter.
12139 */
12140HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12141{
12142 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12143 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12144
12145 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12146 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12147 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12148 AssertRCReturn(rc, rc);
12149
12150 VBOXSTRICTRC rcStrict;
12151 PVM pVM = pVCpu->CTX_SUFF(pVM);
12152 RTGCUINTPTR const uExitQual = pVmxTransient->uExitQual;
12153 uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
12154 switch (uAccessType)
12155 {
12156 case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
12157 {
12158 uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
12159 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
12160 VMX_EXIT_QUAL_CRX_GENREG(uExitQual));
12161 AssertMsg( rcStrict == VINF_SUCCESS
12162 || rcStrict == VINF_IEM_RAISED_XCPT
12163 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12164
12165 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
12166 {
12167 case 0:
12168 {
12169 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12170 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12171 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
12172 Log4Func(("CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
12173
12174 /*
12175 * This is a kludge for handling switches back to real mode when we try to use
12176 * V86 mode to run real mode code directly. Problem is that V86 mode cannot
12177 * deal with special selector values, so we have to return to ring-3 and run
12178 * there till the selector values are V86 mode compatible.
12179 *
12180 * Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
12181 * latter is an alias for VINF_IEM_RAISED_XCPT which is converted to VINF_SUCCESs
12182 * at the end of this function.
12183 */
12184 if ( rc == VINF_SUCCESS
12185 && !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
12186 && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
12187 && (uOldCr0 & X86_CR0_PE)
12188 && !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
12189 {
12190 /** @todo check selectors rather than returning all the time. */
12191 Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
12192 rcStrict = VINF_EM_RESCHEDULE_REM;
12193 }
12194 break;
12195 }
12196
12197 case 2:
12198 {
12199 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
12200 /* Nothing to do here, CR2 it's not part of the VMCS. */
12201 break;
12202 }
12203
12204 case 3:
12205 {
12206 Assert( !pVM->hm.s.fNestedPaging
12207 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
12208 || pVCpu->hm.s.fUsingDebugLoop);
12209 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
12210 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12211 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR3);
12212 Log4Func(("CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
12213 break;
12214 }
12215
12216 case 4:
12217 {
12218 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
12219 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12220 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR4);
12221 Log4Func(("CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
12222 pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12223 break;
12224 }
12225
12226 case 8:
12227 {
12228 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
12229 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
12230 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12231 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_APIC_TPR);
12232 break;
12233 }
12234 default:
12235 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual)));
12236 break;
12237 }
12238 break;
12239 }
12240
12241 case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
12242 {
12243 Assert( !pVM->hm.s.fNestedPaging
12244 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
12245 || pVCpu->hm.s.fUsingDebugLoop
12246 || VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 3);
12247 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12248 Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 8
12249 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
12250
12251 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_GENREG(uExitQual),
12252 VMX_EXIT_QUAL_CRX_REGISTER(uExitQual));
12253 AssertMsg( rcStrict == VINF_SUCCESS
12254 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12255#ifdef VBOX_WITH_STATISTICS
12256 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
12257 {
12258 case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
12259 case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
12260 case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
12261 case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
12262 case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
12263 }
12264#endif
12265 Log4Func(("CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
12266 VBOXSTRICTRC_VAL(rcStrict)));
12267 if (VMX_EXIT_QUAL_CRX_GENREG(uExitQual) == X86_GREG_xSP)
12268 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_RSP);
12269 else
12270 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12271 break;
12272 }
12273
12274 case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12275 {
12276 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12277 AssertMsg( rcStrict == VINF_SUCCESS
12278 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12279
12280 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12281 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12282 Log4Func(("CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12283 break;
12284 }
12285
12286 case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12287 {
12288 /* Note! LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here. */
12289 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual));
12290 AssertMsg( rcStrict == VINF_SUCCESS
12291 || rcStrict == VINF_IEM_RAISED_XCPT
12292 , ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12293
12294 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12295 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12296 Log4Func(("LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12297 break;
12298 }
12299
12300 default:
12301 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12302 VERR_VMX_UNEXPECTED_EXCEPTION);
12303 }
12304
12305 Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS))
12306 == (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS));
12307 if (rcStrict == VINF_IEM_RAISED_XCPT)
12308 {
12309 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
12310 rcStrict = VINF_SUCCESS;
12311 }
12312
12313 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12314 NOREF(pVM);
12315 return rcStrict;
12316}
12317
12318
12319/**
12320 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12321 * VM-exit.
12322 */
12323HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12324{
12325 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12326 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12327
12328 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12329 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12330 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12331 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_EFER);
12332 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12333 AssertRCReturn(rc, rc);
12334
12335 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12336 uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
12337 uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQual);
12338 bool fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12339 bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
12340 bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
12341 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12342 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12343
12344 /*
12345 * Update exit history to see if this exit can be optimized.
12346 */
12347 VBOXSTRICTRC rcStrict;
12348 PCEMEXITREC pExitRec = NULL;
12349 if ( !fGstStepping
12350 && !fDbgStepping)
12351 pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12352 !fIOString
12353 ? !fIOWrite
12354 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12355 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12356 : !fIOWrite
12357 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12358 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12359 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12360 if (!pExitRec)
12361 {
12362 /* I/O operation lookup arrays. */
12363 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12364 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12365 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12366 uint32_t const cbInstr = pVmxTransient->cbInstr;
12367 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12368 PVM pVM = pVCpu->CTX_SUFF(pVM);
12369 if (fIOString)
12370 {
12371 /*
12372 * INS/OUTS - I/O String instruction.
12373 *
12374 * Use instruction-information if available, otherwise fall back on
12375 * interpreting the instruction.
12376 */
12377 Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12378 AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
12379 bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
12380 if (fInsOutsInfo)
12381 {
12382 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12383 AssertRCReturn(rc2, rc2);
12384 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12385 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12386 IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12387 bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
12388 if (fIOWrite)
12389 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12390 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12391 else
12392 {
12393 /*
12394 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12395 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12396 * See Intel Instruction spec. for "INS".
12397 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12398 */
12399 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12400 }
12401 }
12402 else
12403 rcStrict = IEMExecOne(pVCpu);
12404
12405 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12406 fUpdateRipAlready = true;
12407 }
12408 else
12409 {
12410 /*
12411 * IN/OUT - I/O instruction.
12412 */
12413 Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12414 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12415 Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
12416 if (fIOWrite)
12417 {
12418 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
12419 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12420 if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
12421 && !pCtx->eflags.Bits.u1TF)
12422 rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
12423 }
12424 else
12425 {
12426 uint32_t u32Result = 0;
12427 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12428 if (IOM_SUCCESS(rcStrict))
12429 {
12430 /* Save result of I/O IN instr. in AL/AX/EAX. */
12431 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12432 }
12433 if ( rcStrict == VINF_IOM_R3_IOPORT_READ
12434 && !pCtx->eflags.Bits.u1TF)
12435 rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
12436 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12437 }
12438 }
12439
12440 if (IOM_SUCCESS(rcStrict))
12441 {
12442 if (!fUpdateRipAlready)
12443 {
12444 hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
12445 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12446 }
12447
12448 /*
12449 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12450 * while booting Fedora 17 64-bit guest.
12451 *
12452 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12453 */
12454 if (fIOString)
12455 {
12456 /** @todo Single-step for INS/OUTS with REP prefix? */
12457 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12458 }
12459 else if ( !fDbgStepping
12460 && fGstStepping)
12461 {
12462 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12463 AssertRCReturn(rc, rc);
12464 }
12465
12466 /*
12467 * If any I/O breakpoints are armed, we need to check if one triggered
12468 * and take appropriate action.
12469 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12470 */
12471 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12472 AssertRCReturn(rc, rc);
12473
12474 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12475 * execution engines about whether hyper BPs and such are pending. */
12476 uint32_t const uDr7 = pCtx->dr[7];
12477 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12478 && X86_DR7_ANY_RW_IO(uDr7)
12479 && (pCtx->cr4 & X86_CR4_DE))
12480 || DBGFBpIsHwIoArmed(pVM)))
12481 {
12482 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12483
12484 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12485 VMMRZCallRing3Disable(pVCpu);
12486 HM_DISABLE_PREEMPT(pVCpu);
12487
12488 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12489
12490 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
12491 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12492 {
12493 /* Raise #DB. */
12494 if (fIsGuestDbgActive)
12495 ASMSetDR6(pCtx->dr[6]);
12496 if (pCtx->dr[7] != uDr7)
12497 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_DR7;
12498
12499 hmR0VmxSetPendingXcptDB(pVCpu);
12500 }
12501 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12502 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12503 else if ( rcStrict2 != VINF_SUCCESS
12504 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12505 rcStrict = rcStrict2;
12506 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12507
12508 HM_RESTORE_PREEMPT();
12509 VMMRZCallRing3Enable(pVCpu);
12510 }
12511 }
12512
12513#ifdef VBOX_STRICT
12514 if ( rcStrict == VINF_IOM_R3_IOPORT_READ
12515 || rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
12516 Assert(!fIOWrite);
12517 else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
12518 || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
12519 || rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
12520 Assert(fIOWrite);
12521 else
12522 {
12523# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12524 * statuses, that the VMM device and some others may return. See
12525 * IOM_SUCCESS() for guidance. */
12526 AssertMsg( RT_FAILURE(rcStrict)
12527 || rcStrict == VINF_SUCCESS
12528 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12529 || rcStrict == VINF_EM_DBG_BREAKPOINT
12530 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12531 || rcStrict == VINF_EM_RAW_TO_R3
12532 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12533# endif
12534 }
12535#endif
12536 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12537 }
12538 else
12539 {
12540 /*
12541 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12542 */
12543 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12544 AssertRCReturn(rc2, rc2);
12545 STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
12546 : fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
12547 Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12548 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12549 VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
12550 fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
12551
12552 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12553 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12554
12555 Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12556 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12557 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12558 }
12559 return rcStrict;
12560}
12561
12562
12563/**
12564 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12565 * VM-exit.
12566 */
12567HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12568{
12569 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12570
12571 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12572 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12573 AssertRCReturn(rc, rc);
12574 if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12575 {
12576 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12577 AssertRCReturn(rc, rc);
12578 if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
12579 {
12580 uint32_t uErrCode;
12581 RTGCUINTPTR GCPtrFaultAddress;
12582 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12583 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12584 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
12585 if (fErrorCodeValid)
12586 {
12587 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12588 AssertRCReturn(rc, rc);
12589 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12590 }
12591 else
12592 uErrCode = 0;
12593
12594 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12595 && uVector == X86_XCPT_PF)
12596 GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
12597 else
12598 GCPtrFaultAddress = 0;
12599
12600 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12601 0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
12602
12603 Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
12604 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12605 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12606 }
12607 }
12608
12609 /* Fall back to the interpreter to emulate the task-switch. */
12610 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12611 return VERR_EM_INTERPRETER;
12612}
12613
12614
12615/**
12616 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12617 */
12618HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12619{
12620 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12621 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_MONITOR_TRAP_FLAG);
12622 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
12623 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12624 AssertRCReturn(rc, rc);
12625 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12626 return VINF_EM_DBG_STEPPED;
12627}
12628
12629
12630/**
12631 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12632 */
12633HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12634{
12635 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12636
12637 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12638
12639 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12640 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12641 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12642 {
12643 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12644 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12645 {
12646 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12647 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12648 }
12649 }
12650 else
12651 {
12652 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12653 rcStrict1 = VINF_SUCCESS;
12654 return rcStrict1;
12655 }
12656
12657 /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12658 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12659 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12660 AssertRCReturn(rc, rc);
12661
12662 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12663 uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
12664 VBOXSTRICTRC rcStrict2;
12665 switch (uAccessType)
12666 {
12667 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12668 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12669 {
12670 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
12671 || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
12672 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12673
12674 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12675 GCPhys &= PAGE_BASE_GC_MASK;
12676 GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
12677 PVM pVM = pVCpu->CTX_SUFF(pVM);
12678 Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12679 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
12680
12681 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12682 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12683 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12684 CPUMCTX2CORE(pCtx), GCPhys);
12685 Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12686 if ( rcStrict2 == VINF_SUCCESS
12687 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12688 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12689 {
12690 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12691 | HM_CHANGED_GUEST_APIC_TPR);
12692 rcStrict2 = VINF_SUCCESS;
12693 }
12694 break;
12695 }
12696
12697 default:
12698 Log4Func(("uAccessType=%#x\n", uAccessType));
12699 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12700 break;
12701 }
12702
12703 if (rcStrict2 != VINF_SUCCESS)
12704 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12705 return rcStrict2;
12706}
12707
12708
12709/**
12710 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12711 * VM-exit.
12712 */
12713HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12714{
12715 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12716
12717 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12718 if (pVmxTransient->fWasGuestDebugStateActive)
12719 {
12720 AssertMsgFailed(("Unexpected MOV DRx exit\n"));
12721 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12722 }
12723
12724 if ( !pVCpu->hm.s.fSingleInstruction
12725 && !pVmxTransient->fWasHyperDebugStateActive)
12726 {
12727 Assert(!DBGFIsStepping(pVCpu));
12728 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12729
12730 /* Don't intercept MOV DRx any more. */
12731 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
12732 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12733 AssertRCReturn(rc, rc);
12734
12735 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12736 VMMRZCallRing3Disable(pVCpu);
12737 HM_DISABLE_PREEMPT(pVCpu);
12738
12739 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12740 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12741 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12742
12743 HM_RESTORE_PREEMPT();
12744 VMMRZCallRing3Enable(pVCpu);
12745
12746#ifdef VBOX_WITH_STATISTICS
12747 rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12748 AssertRCReturn(rc, rc);
12749 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12750 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12751 else
12752 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12753#endif
12754 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12755 return VINF_SUCCESS;
12756 }
12757
12758 /*
12759 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12760 * Update the segment registers and DR7 from the CPU.
12761 */
12762 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12763 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12764 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_DR7);
12765 AssertRCReturn(rc, rc);
12766 Log4Func(("CS:RIP=%04x:%08RX64\n", pCtx->cs.Sel, pCtx->rip));
12767
12768 PVM pVM = pVCpu->CTX_SUFF(pVM);
12769 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12770 {
12771 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12772 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
12773 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
12774 if (RT_SUCCESS(rc))
12775 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12776 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12777 }
12778 else
12779 {
12780 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12781 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
12782 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
12783 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12784 }
12785
12786 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12787 if (RT_SUCCESS(rc))
12788 {
12789 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12790 AssertRCReturn(rc2, rc2);
12791 return VINF_SUCCESS;
12792 }
12793 return rc;
12794}
12795
12796
12797/**
12798 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12799 * Conditional VM-exit.
12800 */
12801HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12802{
12803 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12804 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12805
12806 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12807 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12808 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12809 {
12810 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12811 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12812 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12813 {
12814 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12815 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12816 }
12817 }
12818 else
12819 {
12820 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12821 rcStrict1 = VINF_SUCCESS;
12822 return rcStrict1;
12823 }
12824
12825 /*
12826 * Get sufficent state and update the exit history entry.
12827 */
12828 RTGCPHYS GCPhys;
12829 int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
12830 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12831 AssertRCReturn(rc, rc);
12832
12833 VBOXSTRICTRC rcStrict;
12834 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12835 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
12836 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12837 if (!pExitRec)
12838 {
12839 /*
12840 * If we succeed, resume guest execution.
12841 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12842 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12843 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12844 * weird case. See @bugref{6043}.
12845 */
12846 PVM pVM = pVCpu->CTX_SUFF(pVM);
12847 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12848 rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
12849 Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12850 if ( rcStrict == VINF_SUCCESS
12851 || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12852 || rcStrict == VERR_PAGE_NOT_PRESENT)
12853 {
12854 /* Successfully handled MMIO operation. */
12855 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12856 | HM_CHANGED_GUEST_APIC_TPR);
12857 rcStrict = VINF_SUCCESS;
12858 }
12859 }
12860 else
12861 {
12862 /*
12863 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12864 */
12865 int rc2 = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12866 AssertRCReturn(rc2, rc2);
12867
12868 Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12869 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12870
12871 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12872 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12873
12874 Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12875 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12876 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12877 }
12878 return VBOXSTRICTRC_TODO(rcStrict);
12879}
12880
12881
12882/**
12883 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12884 * VM-exit.
12885 */
12886HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12887{
12888 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12889 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12890
12891 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12892 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12893 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12894 {
12895 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12896 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12897 Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12898 }
12899 else
12900 {
12901 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12902 rcStrict1 = VINF_SUCCESS;
12903 return rcStrict1;
12904 }
12905
12906 RTGCPHYS GCPhys;
12907 int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
12908 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12909 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12910 AssertRCReturn(rc, rc);
12911
12912 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12913 AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
12914
12915 RTGCUINT uErrorCode = 0;
12916 if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
12917 uErrorCode |= X86_TRAP_PF_ID;
12918 if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
12919 uErrorCode |= X86_TRAP_PF_RW;
12920 if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
12921 uErrorCode |= X86_TRAP_PF_P;
12922
12923 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12924
12925
12926 /* Handle the pagefault trap for the nested shadow table. */
12927 PVM pVM = pVCpu->CTX_SUFF(pVM);
12928 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12929
12930 Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
12931 pCtx->cs.Sel, pCtx->rip));
12932
12933 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
12934 TRPMResetTrap(pVCpu);
12935
12936 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12937 if ( rcStrict2 == VINF_SUCCESS
12938 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12939 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12940 {
12941 /* Successfully synced our nested page tables. */
12942 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12943 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
12944 return VINF_SUCCESS;
12945 }
12946
12947 Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12948 return rcStrict2;
12949}
12950
12951/** @} */
12952
12953/** @name VM-exit exception handlers.
12954 * @{
12955 */
12956/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12957/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12958/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12959
12960/**
12961 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12962 */
12963static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12964{
12965 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12966 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12967
12968 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
12969 AssertRCReturn(rc, rc);
12970
12971 if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
12972 {
12973 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12974 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12975
12976 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12977 * provides VM-exit instruction length. If this causes problem later,
12978 * disassemble the instruction like it's done on AMD-V. */
12979 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12980 AssertRCReturn(rc2, rc2);
12981 return rc;
12982 }
12983
12984 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
12985 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12986 return rc;
12987}
12988
12989
12990/**
12991 * VM-exit exception handler for \#BP (Breakpoint exception).
12992 */
12993static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12994{
12995 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12996 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12997
12998 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12999 AssertRCReturn(rc, rc);
13000
13001 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13002 rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13003 if (rc == VINF_EM_RAW_GUEST_TRAP)
13004 {
13005 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13006 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13007 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13008 AssertRCReturn(rc, rc);
13009
13010 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13011 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13012 }
13013
13014 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
13015 return rc;
13016}
13017
13018
13019/**
13020 * VM-exit exception handler for \#AC (alignment check exception).
13021 */
13022static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13023{
13024 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13025
13026 /*
13027 * Re-inject it. We'll detect any nesting before getting here.
13028 */
13029 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13030 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13031 AssertRCReturn(rc, rc);
13032 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13033
13034 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13035 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13036 return VINF_SUCCESS;
13037}
13038
13039
13040/**
13041 * VM-exit exception handler for \#DB (Debug exception).
13042 */
13043static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13044{
13045 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13046 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13047
13048 /*
13049 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13050 * for processing.
13051 */
13052 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13053
13054 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13055 uint64_t uDR6 = X86_DR6_INIT_VAL;
13056 uDR6 |= (pVmxTransient->uExitQual & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
13057
13058 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13059 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13060 Log6Func(("rc=%Rrc\n", rc));
13061 if (rc == VINF_EM_RAW_GUEST_TRAP)
13062 {
13063 /*
13064 * The exception was for the guest. Update DR6, DR7.GD and
13065 * IA32_DEBUGCTL.LBR before forwarding it.
13066 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13067 */
13068 VMMRZCallRing3Disable(pVCpu);
13069 HM_DISABLE_PREEMPT(pVCpu);
13070
13071 pCtx->dr[6] &= ~X86_DR6_B_MASK;
13072 pCtx->dr[6] |= uDR6;
13073 if (CPUMIsGuestDebugStateActive(pVCpu))
13074 ASMSetDR6(pCtx->dr[6]);
13075
13076 HM_RESTORE_PREEMPT();
13077 VMMRZCallRing3Enable(pVCpu);
13078
13079 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
13080 AssertRCReturn(rc, rc);
13081
13082 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13083 pCtx->dr[7] &= ~X86_DR7_GD;
13084
13085 /* Paranoia. */
13086 pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13087 pCtx->dr[7] |= X86_DR7_RA1_MASK;
13088
13089 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
13090 AssertRCReturn(rc, rc);
13091
13092 /*
13093 * Raise #DB in the guest.
13094 *
13095 * It is important to reflect exactly what the VM-exit gave us (preserving the
13096 * interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
13097 * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
13098 * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
13099 *
13100 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
13101 * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
13102 */
13103 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13104 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13105 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13106 AssertRCReturn(rc, rc);
13107 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13108 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13109 return VINF_SUCCESS;
13110 }
13111
13112 /*
13113 * Not a guest trap, must be a hypervisor related debug event then.
13114 * Update DR6 in case someone is interested in it.
13115 */
13116 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13117 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13118 CPUMSetHyperDR6(pVCpu, uDR6);
13119
13120 return rc;
13121}
13122
13123/**
13124 * VM-exit exception handler for \#GP (General-protection exception).
13125 *
13126 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13127 */
13128static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13129{
13130 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13131 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13132
13133 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13134 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13135 { /* likely */ }
13136 else
13137 {
13138#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13139 Assert(pVCpu->hm.s.fUsingDebugLoop);
13140#endif
13141 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13142 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13143 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13144 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13145 rc |= hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13146 AssertRCReturn(rc, rc);
13147 Log4Func(("Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pCtx->cs.Sel, pCtx->rip,
13148 pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
13149 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13150 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13151 return rc;
13152 }
13153
13154 Assert(CPUMIsGuestInRealModeEx(pCtx));
13155 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13156
13157 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13158 AssertRCReturn(rc, rc);
13159
13160 VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
13161 if (rcStrict == VINF_SUCCESS)
13162 {
13163 if (!CPUMIsGuestInRealModeEx(pCtx))
13164 {
13165 /*
13166 * The guest is no longer in real-mode, check if we can continue executing the
13167 * guest using hardware-assisted VMX. Otherwise, fall back to emulation.
13168 */
13169 if (HMVmxCanExecuteGuest(pVCpu, pCtx))
13170 {
13171 Log4Func(("Mode changed but guest still suitable for executing using VT-x\n"));
13172 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
13173 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13174 }
13175 else
13176 {
13177 Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
13178 rcStrict = VINF_EM_RESCHEDULE;
13179 }
13180 }
13181 else
13182 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13183 }
13184 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13185 {
13186 rcStrict = VINF_SUCCESS;
13187 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13188 }
13189 return VBOXSTRICTRC_VAL(rcStrict);
13190}
13191
13192
13193/**
13194 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13195 * the exception reported in the VMX transient structure back into the VM.
13196 *
13197 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13198 * up-to-date.
13199 */
13200static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13201{
13202 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13203#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13204 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13205 ("uVector=%#x u32XcptBitmap=%#X32\n",
13206 VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13207#endif
13208
13209 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13210 hmR0VmxCheckExitDueToEventDelivery(). */
13211 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13212 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13213 AssertRCReturn(rc, rc);
13214 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13215
13216#ifdef DEBUG_ramshankar
13217 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
13218 uint8_t uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13219 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13220#endif
13221
13222 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13223 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13224 return VINF_SUCCESS;
13225}
13226
13227
13228/**
13229 * VM-exit exception handler for \#PF (Page-fault exception).
13230 */
13231static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13232{
13233 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13234 PVM pVM = pVCpu->CTX_SUFF(pVM);
13235 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13236 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13237 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13238 AssertRCReturn(rc, rc);
13239
13240 if (!pVM->hm.s.fNestedPaging)
13241 { /* likely */ }
13242 else
13243 {
13244#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13245 Assert(pVCpu->hm.s.fUsingDebugLoop);
13246#endif
13247 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13248 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13249 {
13250 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13251 pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
13252 }
13253 else
13254 {
13255 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13256 hmR0VmxSetPendingXcptDF(pVCpu);
13257 Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13258 }
13259 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13260 return rc;
13261 }
13262
13263 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13264 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13265 if (pVmxTransient->fVectoringPF)
13266 {
13267 Assert(pVCpu->hm.s.Event.fPending);
13268 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13269 }
13270
13271 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13272 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13273 AssertRCReturn(rc, rc);
13274
13275 Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
13276 pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
13277
13278 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13279 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
13280
13281 Log4Func(("#PF: rc=%Rrc\n", rc));
13282 if (rc == VINF_SUCCESS)
13283 {
13284 /*
13285 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13286 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13287 */
13288 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13289 TRPMResetTrap(pVCpu);
13290 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13291 return rc;
13292 }
13293
13294 if (rc == VINF_EM_RAW_GUEST_TRAP)
13295 {
13296 if (!pVmxTransient->fVectoringDoublePF)
13297 {
13298 /* It's a guest page fault and needs to be reflected to the guest. */
13299 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13300 TRPMResetTrap(pVCpu);
13301 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13302 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13303 uGstErrorCode, pVmxTransient->uExitQual);
13304 }
13305 else
13306 {
13307 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13308 TRPMResetTrap(pVCpu);
13309 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13310 hmR0VmxSetPendingXcptDF(pVCpu);
13311 Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13312 }
13313
13314 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13315 return VINF_SUCCESS;
13316 }
13317
13318 TRPMResetTrap(pVCpu);
13319 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13320 return rc;
13321}
13322
13323/** @} */
13324
13325#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13326
13327/** @name Nested-guest VM-exit handlers.
13328 * @{
13329 */
13330/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13331/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= Nested-guest VM-exit handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13332/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13333
13334/**
13335 * VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
13336 */
13337HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13338{
13339 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13340
13341 /** @todo NSTVMX: Vmclear. */
13342 hmR0VmxSetPendingXcptUD(pVCpu);
13343 return VINF_SUCCESS;
13344}
13345
13346
13347/**
13348 * VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
13349 */
13350HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13351{
13352 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13353
13354 /** @todo NSTVMX: Vmlaunch. */
13355 hmR0VmxSetPendingXcptUD(pVCpu);
13356 return VINF_SUCCESS;
13357}
13358
13359
13360/**
13361 * VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
13362 */
13363HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13364{
13365 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13366
13367 /** @todo NSTVMX: Vmptrld. */
13368 hmR0VmxSetPendingXcptUD(pVCpu);
13369 return VINF_SUCCESS;
13370}
13371
13372
13373/**
13374 * VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
13375 */
13376HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13377{
13378 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13379
13380 /** @todo NSTVMX: Vmptrst. */
13381 hmR0VmxSetPendingXcptUD(pVCpu);
13382 return VINF_SUCCESS;
13383}
13384
13385
13386/**
13387 * VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Unconditional VM-exit.
13388 */
13389HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13390{
13391 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13392
13393 /** @todo NSTVMX: Vmread. */
13394 hmR0VmxSetPendingXcptUD(pVCpu);
13395 return VINF_SUCCESS;
13396}
13397
13398
13399/**
13400 * VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
13401 */
13402HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13403{
13404 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13405
13406 /** @todo NSTVMX: Vmresume. */
13407 hmR0VmxSetPendingXcptUD(pVCpu);
13408 return VINF_SUCCESS;
13409}
13410
13411
13412/**
13413 * VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Unconditional VM-exit.
13414 */
13415HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13416{
13417 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13418
13419 /** @todo NSTVMX: Vmwrite. */
13420 hmR0VmxSetPendingXcptUD(pVCpu);
13421 return VINF_SUCCESS;
13422}
13423
13424
13425/**
13426 * VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
13427 */
13428HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13429{
13430 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13431
13432 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13433 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13434 AssertRCReturn(rc, rc);
13435
13436 VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
13437 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13438 {
13439 /* VMXOFF on success changes the internal hwvirt state but not anything that's visible to the guest. */
13440 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
13441 }
13442 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13443 {
13444 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13445 rcStrict = VINF_SUCCESS;
13446 }
13447 return rcStrict;
13448}
13449
13450
13451/**
13452 * VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
13453 */
13454HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13455{
13456 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13457
13458 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13459 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13460 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13461 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13462 AssertRCReturn(rc, rc);
13463
13464 VBOXSTRICTRC rcStrict = hmR0VmxCheckExitDueToVmxInstr(pVCpu, pVmxTransient);
13465 if (rcStrict == VINF_SUCCESS)
13466 { /* likely */ }
13467 else if (rcStrict == VINF_HM_PENDING_XCPT)
13468 {
13469 Log4Func(("Privilege checks failed, raising xcpt %#x!\n", VMX_ENTRY_INT_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo)));
13470 return VINF_SUCCESS;
13471 }
13472 else
13473 {
13474 Log4Func(("hmR0VmxCheckExitDueToVmxInstr failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13475 return rcStrict;
13476 }
13477
13478 RTGCPTR GCPtrVmxon;
13479 PCVMXEXITINSTRINFO pExitInstrInfo = &pVmxTransient->ExitInstrInfo;
13480 RTGCPTR const GCPtrDisp = pVmxTransient->uExitQual;
13481 rcStrict = hmR0VmxDecodeMemOperand(pVCpu, pExitInstrInfo, GCPtrDisp, false /*fIsWrite*/, &GCPtrVmxon);
13482 if (rcStrict == VINF_SUCCESS)
13483 { /* likely */ }
13484 else if (rcStrict == VINF_HM_PENDING_XCPT)
13485 {
13486 Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", VMX_ENTRY_INT_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo)));
13487 return VINF_SUCCESS;
13488 }
13489 else
13490 {
13491 Log4Func(("hmR0VmxCheckExitDueToVmxInstr failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13492 return rcStrict;
13493 }
13494
13495 rcStrict = IEMExecDecodedVmxon(pVCpu, pVmxTransient->cbInstr, GCPtrVmxon, pExitInstrInfo->u, GCPtrDisp);
13496 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13497 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13498 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13499 {
13500 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13501 rcStrict = VINF_SUCCESS;
13502 }
13503 return rcStrict;
13504}
13505
13506/** @} */
13507#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13508
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette