VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 72849

Last change on this file since 72849 was 72849, checked in by vboxsync, 6 years ago

VMM/HMVMXR0: bugref:9193 Add assertions where possible to verify we don't overwrite stale values from CPUMCTX into the VMCS.

  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 576.1 KB
Line 
1/* $Id: HMVMXR0.cpp 72849 2018-07-04 05:27:56Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/x86.h>
25#include <iprt/asm-amd64-x86.h>
26#include <iprt/thread.h>
27
28#include <VBox/vmm/pdmapi.h>
29#include <VBox/vmm/dbgf.h>
30#include <VBox/vmm/iem.h>
31#include <VBox/vmm/iom.h>
32#include <VBox/vmm/selm.h>
33#include <VBox/vmm/tm.h>
34#include <VBox/vmm/gim.h>
35#include <VBox/vmm/apic.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#include "HMInternal.h"
40#include <VBox/vmm/vm.h>
41#include "HMVMXR0.h"
42#include "dtrace/VBoxVMM.h"
43
44#ifdef DEBUG_ramshankar
45# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
47# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_CHECK_GUEST_STATE
49# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
50# define HMVMX_ALWAYS_TRAP_PF
51# define HMVMX_ALWAYS_FLUSH_TLB
52# define HMVMX_ALWAYS_SWAP_EFER
53#endif
54
55
56/*********************************************************************************************************************************
57* Defined Constants And Macros *
58*********************************************************************************************************************************/
59/** Use the function table. */
60#define HMVMX_USE_FUNCTION_TABLE
61
62/** Determine which tagged-TLB flush handler to use. */
63#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68/** @name HMVMX_READ_XXX
69 * Flags to skip redundant reads of some common VMCS fields that are not part of
70 * the guest-CPU or VCPU state but are needed while handling VM-exits.
71 */
72#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
73#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
74#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
75#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
76#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
77#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
78#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
79/** @} */
80
81/**
82 * States of the VMCS.
83 *
84 * This does not reflect all possible VMCS states but currently only those
85 * needed for maintaining the VMCS consistently even when thread-context hooks
86 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
87 */
88#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
89#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
90#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
91
92/**
93 * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
94 * guest using hardware-assisted VMX.
95 *
96 * This excludes state like GPRs (other than RSP) which are always are
97 * swapped and restored across the world-switch and also registers like EFER,
98 * MSR which cannot be modified by the guest without causing a VM-exit.
99 */
100#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
101 | CPUMCTX_EXTRN_RFLAGS \
102 | CPUMCTX_EXTRN_RSP \
103 | CPUMCTX_EXTRN_SREG_MASK \
104 | CPUMCTX_EXTRN_TABLE_MASK \
105 | CPUMCTX_EXTRN_KERNEL_GS_BASE \
106 | CPUMCTX_EXTRN_SYSCALL_MSRS \
107 | CPUMCTX_EXTRN_SYSENTER_MSRS \
108 | CPUMCTX_EXTRN_TSC_AUX \
109 | CPUMCTX_EXTRN_OTHER_MSRS \
110 | CPUMCTX_EXTRN_CR0 \
111 | CPUMCTX_EXTRN_CR3 \
112 | CPUMCTX_EXTRN_CR4 \
113 | CPUMCTX_EXTRN_DR7 \
114 | CPUMCTX_EXTRN_HM_VMX_MASK)
115
116/**
117 * Exception bitmap mask for real-mode guests (real-on-v86).
118 *
119 * We need to intercept all exceptions manually except:
120 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
121 * due to bugs in Intel CPUs.
122 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
123 * support.
124 */
125#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
126 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
127 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
128 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
129 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
130 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
131 | RT_BIT(X86_XCPT_XF))
132
133/**
134 * Exception bitmap mask for all contributory exceptions.
135 *
136 * Page fault is deliberately excluded here as it's conditional as to whether
137 * it's contributory or benign. Page faults are handled separately.
138 */
139#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
140 | RT_BIT(X86_XCPT_DE))
141
142/** Maximum VM-instruction error number. */
143#define HMVMX_INSTR_ERROR_MAX 28
144
145/** Profiling macro. */
146#ifdef HM_PROFILE_EXIT_DISPATCH
147# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
148# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
149#else
150# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
151# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
152#endif
153
154/** Assert that preemption is disabled or covered by thread-context hooks. */
155#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
156 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
157
158/** Assert that we haven't migrated CPUs when thread-context hooks are not
159 * used. */
160#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
161 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
162 ("Illegal migration! Entered on CPU %u Current %u\n", \
163 pVCpu->hm.s.idEnteredCpu, RTMpCpuId()))
164
165/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
166 * context. */
167#define HMVMX_CPUMCTX_ASSERT(pVCpu, fExtrnMbz) AssertMsg(!((pVCpu)->cpum.GstCtx.fExtrn & (fExtrnMbz)), \
168 ("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", (pVCpu)->cpum.GstCtx.fExtrn, \
169 (fExtrnMbz)))
170
171/** Helper macro for VM-exit handlers called unexpectedly. */
172#define HMVMX_RETURN_UNEXPECTED_EXIT() \
173 do { \
174 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
175 return VERR_VMX_UNEXPECTED_EXIT; \
176 } while (0)
177
178/** Macro for importing segment registers to the VMCS from the guest-CPU context. */
179#ifdef VMX_USE_CACHED_VMCS_ACCESSES
180# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
181 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
182 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
183#else
184# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
185 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
186 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
187#endif
188
189/** Macro for exporting segment registers to the VMCS from the guest-CPU context. */
190# define HMVMX_EXPORT_SREG(Sel, a_pCtxSelReg) \
191 hmR0VmxExportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
192 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
193
194
195/*********************************************************************************************************************************
196* Structures and Typedefs *
197*********************************************************************************************************************************/
198/**
199 * VMX transient state.
200 *
201 * A state structure for holding miscellaneous information across
202 * VMX non-root operation and restored after the transition.
203 */
204typedef struct VMXTRANSIENT
205{
206 /** The host's rflags/eflags. */
207 RTCCUINTREG fEFlags;
208#if HC_ARCH_BITS == 32
209 uint32_t u32Alignment0;
210#endif
211 /** The guest's TPR value used for TPR shadowing. */
212 uint8_t u8GuestTpr;
213 /** Alignment. */
214 uint8_t abAlignment0[7];
215
216 /** The basic VM-exit reason. */
217 uint16_t uExitReason;
218 /** Alignment. */
219 uint16_t u16Alignment0;
220 /** The VM-exit interruption error code. */
221 uint32_t uExitIntErrorCode;
222 /** The VM-exit exit code qualification. */
223 uint64_t uExitQualification;
224
225 /** The VM-exit interruption-information field. */
226 uint32_t uExitIntInfo;
227 /** The VM-exit instruction-length field. */
228 uint32_t cbInstr;
229 /** The VM-exit instruction-information field. */
230 union
231 {
232 /** Plain unsigned int representation. */
233 uint32_t u;
234 /** INS and OUTS information. */
235 struct
236 {
237 uint32_t u7Reserved0 : 7;
238 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
239 uint32_t u3AddrSize : 3;
240 uint32_t u5Reserved1 : 5;
241 /** The segment register (X86_SREG_XXX). */
242 uint32_t iSegReg : 3;
243 uint32_t uReserved2 : 14;
244 } StrIo;
245 /** INVEPT, INVVPID, INVPCID information. */
246 struct
247 {
248 /** Scaling; 0=no scaling, 1=scale-by-2, 2=scale-by-4, 3=scale-by-8. */
249 uint32_t u2Scaling : 2;
250 uint32_t u5Reserved0 : 5;
251 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
252 uint32_t u3AddrSize : 3;
253 uint32_t u1Reserved0 : 1;
254 uint32_t u4Reserved0 : 4;
255 /** The segment register (X86_SREG_XXX). */
256 uint32_t iSegReg : 3;
257 /** The index register (X86_GREG_XXX). */
258 uint32_t iIdxReg : 4;
259 /** Set if index register is invalid. */
260 uint32_t fIdxRegValid : 1;
261 /** The base register (X86_GREG_XXX). */
262 uint32_t iBaseReg : 4;
263 /** Set if base register is invalid. */
264 uint32_t fBaseRegValid : 1;
265 /** Register 2 (X86_GREG_XXX). */
266 uint32_t iReg2 : 4;
267 } Inv;
268 } ExitInstrInfo;
269 /** Whether the VM-entry failed or not. */
270 bool fVMEntryFailed;
271 /** Alignment. */
272 uint8_t abAlignment1[3];
273
274 /** The VM-entry interruption-information field. */
275 uint32_t uEntryIntInfo;
276 /** The VM-entry exception error code field. */
277 uint32_t uEntryXcptErrorCode;
278 /** The VM-entry instruction length field. */
279 uint32_t cbEntryInstr;
280
281 /** IDT-vectoring information field. */
282 uint32_t uIdtVectoringInfo;
283 /** IDT-vectoring error code. */
284 uint32_t uIdtVectoringErrorCode;
285
286 /** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
287 uint32_t fVmcsFieldsRead;
288
289 /** Whether the guest debug state was active at the time of VM-exit. */
290 bool fWasGuestDebugStateActive;
291 /** Whether the hyper debug state was active at the time of VM-exit. */
292 bool fWasHyperDebugStateActive;
293 /** Whether TSC-offsetting should be setup before VM-entry. */
294 bool fUpdateTscOffsettingAndPreemptTimer;
295 /** Whether the VM-exit was caused by a page-fault during delivery of a
296 * contributory exception or a page-fault. */
297 bool fVectoringDoublePF;
298 /** Whether the VM-exit was caused by a page-fault during delivery of an
299 * external interrupt or NMI. */
300 bool fVectoringPF;
301} VMXTRANSIENT;
302AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
303AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
304AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
305AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
306AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
307/** Pointer to VMX transient state. */
308typedef VMXTRANSIENT *PVMXTRANSIENT;
309
310
311/**
312 * MSR-bitmap read permissions.
313 */
314typedef enum VMXMSREXITREAD
315{
316 /** Reading this MSR causes a VM-exit. */
317 VMXMSREXIT_INTERCEPT_READ = 0xb,
318 /** Reading this MSR does not cause a VM-exit. */
319 VMXMSREXIT_PASSTHRU_READ
320} VMXMSREXITREAD;
321/** Pointer to MSR-bitmap read permissions. */
322typedef VMXMSREXITREAD* PVMXMSREXITREAD;
323
324/**
325 * MSR-bitmap write permissions.
326 */
327typedef enum VMXMSREXITWRITE
328{
329 /** Writing to this MSR causes a VM-exit. */
330 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
331 /** Writing to this MSR does not cause a VM-exit. */
332 VMXMSREXIT_PASSTHRU_WRITE
333} VMXMSREXITWRITE;
334/** Pointer to MSR-bitmap write permissions. */
335typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
336
337
338/**
339 * VMX VM-exit handler.
340 *
341 * @returns Strict VBox status code (i.e. informational status codes too).
342 * @param pVCpu The cross context virtual CPU structure.
343 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
344 * out-of-sync. Make sure to update the required
345 * fields before using them.
346 * @param pVmxTransient Pointer to the VMX-transient structure.
347 */
348#ifndef HMVMX_USE_FUNCTION_TABLE
349typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
350#else
351typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
352/** Pointer to VM-exit handler. */
353typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
354#endif
355
356/**
357 * VMX VM-exit handler, non-strict status code.
358 *
359 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
360 *
361 * @returns VBox status code, no informational status code returned.
362 * @param pVCpu The cross context virtual CPU structure.
363 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
364 * out-of-sync. Make sure to update the required
365 * fields before using them.
366 * @param pVmxTransient Pointer to the VMX-transient structure.
367 *
368 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
369 * use of that status code will be replaced with VINF_EM_SOMETHING
370 * later when switching over to IEM.
371 */
372#ifndef HMVMX_USE_FUNCTION_TABLE
373typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
374#else
375typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
376#endif
377
378
379/*********************************************************************************************************************************
380* Internal Functions *
381*********************************************************************************************************************************/
382static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
383static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
384static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
385static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat);
386static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
387 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState);
388#if HC_ARCH_BITS == 32
389static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
390#endif
391#ifndef HMVMX_USE_FUNCTION_TABLE
392DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
393# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
394# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
395#else
396# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
397# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
398#endif
399
400
401/** @name VM-exit handlers.
402 * @{
403 */
404static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
405static FNVMXEXITHANDLER hmR0VmxExitExtInt;
406static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
407static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
408static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
409static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
410static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
411static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
412static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
413static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
414static FNVMXEXITHANDLER hmR0VmxExitCpuid;
415static FNVMXEXITHANDLER hmR0VmxExitGetsec;
416static FNVMXEXITHANDLER hmR0VmxExitHlt;
417static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
418static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
419static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
420static FNVMXEXITHANDLER hmR0VmxExitVmcall;
421static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
422static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
423static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
424static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
425static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
426static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
427static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
428static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
429static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
430static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
431static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
432static FNVMXEXITHANDLER hmR0VmxExitMwait;
433static FNVMXEXITHANDLER hmR0VmxExitMtf;
434static FNVMXEXITHANDLER hmR0VmxExitMonitor;
435static FNVMXEXITHANDLER hmR0VmxExitPause;
436static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
437static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
438static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
439static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
440static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
441static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
442static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
443static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
444static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
445static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
446static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
447static FNVMXEXITHANDLER hmR0VmxExitRdrand;
448static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
449/** @} */
450
451static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
452static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
453static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
454static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
455static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
456static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
457static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
458static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCPUMCTX pCtx);
459
460
461/*********************************************************************************************************************************
462* Global Variables *
463*********************************************************************************************************************************/
464#ifdef HMVMX_USE_FUNCTION_TABLE
465
466/**
467 * VMX_EXIT dispatch table.
468 */
469static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
470{
471 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
472 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
473 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
474 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
475 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
476 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
477 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
478 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
479 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
480 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
481 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
482 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
483 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
484 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
485 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
486 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
487 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
488 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
489 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
490 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
491 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
492 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
493 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
494 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
495 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
496 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
497 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
498 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
499 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
500 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
501 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
502 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
503 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
504 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
505 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
506 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
507 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
508 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
509 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
510 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
511 /* 40 UNDEFINED */ hmR0VmxExitPause,
512 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
513 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
514 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
515 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
516 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
517 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
518 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
519 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
520 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
521 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
522 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
523 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
524 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
525 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
526 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
527 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
528 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
529 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
530 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
531 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
532 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
533 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
534 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
535 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
536};
537#endif /* HMVMX_USE_FUNCTION_TABLE */
538
539#ifdef VBOX_STRICT
540static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
541{
542 /* 0 */ "(Not Used)",
543 /* 1 */ "VMCALL executed in VMX root operation.",
544 /* 2 */ "VMCLEAR with invalid physical address.",
545 /* 3 */ "VMCLEAR with VMXON pointer.",
546 /* 4 */ "VMLAUNCH with non-clear VMCS.",
547 /* 5 */ "VMRESUME with non-launched VMCS.",
548 /* 6 */ "VMRESUME after VMXOFF",
549 /* 7 */ "VM-entry with invalid control fields.",
550 /* 8 */ "VM-entry with invalid host state fields.",
551 /* 9 */ "VMPTRLD with invalid physical address.",
552 /* 10 */ "VMPTRLD with VMXON pointer.",
553 /* 11 */ "VMPTRLD with incorrect revision identifier.",
554 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
555 /* 13 */ "VMWRITE to read-only VMCS component.",
556 /* 14 */ "(Not Used)",
557 /* 15 */ "VMXON executed in VMX root operation.",
558 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
559 /* 17 */ "VM-entry with non-launched executing VMCS.",
560 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
561 /* 19 */ "VMCALL with non-clear VMCS.",
562 /* 20 */ "VMCALL with invalid VM-exit control fields.",
563 /* 21 */ "(Not Used)",
564 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
565 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
566 /* 24 */ "VMCALL with invalid SMM-monitor features.",
567 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
568 /* 26 */ "VM-entry with events blocked by MOV SS.",
569 /* 27 */ "(Not Used)",
570 /* 28 */ "Invalid operand to INVEPT/INVVPID."
571};
572#endif /* VBOX_STRICT */
573
574
575
576/**
577 * Updates the VM's last error record.
578 *
579 * If there was a VMX instruction error, reads the error data from the VMCS and
580 * updates VCPU's last error record as well.
581 *
582 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
583 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
584 * VERR_VMX_INVALID_VMCS_FIELD.
585 * @param rc The error code.
586 */
587static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
588{
589 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
590 || rc == VERR_VMX_UNABLE_TO_START_VM)
591 {
592 AssertPtrReturnVoid(pVCpu);
593 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
594 }
595 pVCpu->CTX_SUFF(pVM)->hm.s.lLastError = rc;
596}
597
598
599/**
600 * Reads the VM-entry interruption-information field from the VMCS into the VMX
601 * transient structure.
602 *
603 * @returns VBox status code.
604 * @param pVmxTransient Pointer to the VMX transient structure.
605 *
606 * @remarks No-long-jump zone!!!
607 */
608DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
609{
610 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
611 AssertRCReturn(rc, rc);
612 return VINF_SUCCESS;
613}
614
615#ifdef VBOX_STRICT
616/**
617 * Reads the VM-entry exception error code field from the VMCS into
618 * the VMX transient structure.
619 *
620 * @returns VBox status code.
621 * @param pVmxTransient Pointer to the VMX transient structure.
622 *
623 * @remarks No-long-jump zone!!!
624 */
625DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
626{
627 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
628 AssertRCReturn(rc, rc);
629 return VINF_SUCCESS;
630}
631
632
633/**
634 * Reads the VM-entry exception error code field from the VMCS into
635 * the VMX transient structure.
636 *
637 * @returns VBox status code.
638 * @param pVmxTransient Pointer to the VMX transient structure.
639 *
640 * @remarks No-long-jump zone!!!
641 */
642DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
643{
644 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
645 AssertRCReturn(rc, rc);
646 return VINF_SUCCESS;
647}
648#endif /* VBOX_STRICT */
649
650
651/**
652 * Reads the VM-exit interruption-information field from the VMCS into the VMX
653 * transient structure.
654 *
655 * @returns VBox status code.
656 * @param pVmxTransient Pointer to the VMX transient structure.
657 */
658DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
659{
660 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
661 {
662 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
663 AssertRCReturn(rc,rc);
664 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_INFO;
665 }
666 return VINF_SUCCESS;
667}
668
669
670/**
671 * Reads the VM-exit interruption error code from the VMCS into the VMX
672 * transient structure.
673 *
674 * @returns VBox status code.
675 * @param pVmxTransient Pointer to the VMX transient structure.
676 */
677DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
678{
679 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
680 {
681 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
682 AssertRCReturn(rc, rc);
683 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
684 }
685 return VINF_SUCCESS;
686}
687
688
689/**
690 * Reads the VM-exit instruction length field from the VMCS into the VMX
691 * transient structure.
692 *
693 * @returns VBox status code.
694 * @param pVmxTransient Pointer to the VMX transient structure.
695 */
696DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
697{
698 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
699 {
700 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
701 AssertRCReturn(rc, rc);
702 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_LEN;
703 }
704 return VINF_SUCCESS;
705}
706
707
708/**
709 * Reads the VM-exit instruction-information field from the VMCS into
710 * the VMX transient structure.
711 *
712 * @returns VBox status code.
713 * @param pVmxTransient Pointer to the VMX transient structure.
714 */
715DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
716{
717 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
718 {
719 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
720 AssertRCReturn(rc, rc);
721 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_INFO;
722 }
723 return VINF_SUCCESS;
724}
725
726
727/**
728 * Reads the exit code qualification from the VMCS into the VMX transient
729 * structure.
730 *
731 * @returns VBox status code.
732 * @param pVCpu The cross context virtual CPU structure of the
733 * calling EMT. (Required for the VMCS cache case.)
734 * @param pVmxTransient Pointer to the VMX transient structure.
735 */
736DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
737{
738 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
739 {
740 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
741 AssertRCReturn(rc, rc);
742 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION;
743 }
744 return VINF_SUCCESS;
745}
746
747
748/**
749 * Reads the IDT-vectoring information field from the VMCS into the VMX
750 * transient structure.
751 *
752 * @returns VBox status code.
753 * @param pVmxTransient Pointer to the VMX transient structure.
754 *
755 * @remarks No-long-jump zone!!!
756 */
757DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
758{
759 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
760 {
761 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
762 AssertRCReturn(rc, rc);
763 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_INFO;
764 }
765 return VINF_SUCCESS;
766}
767
768
769/**
770 * Reads the IDT-vectoring error code from the VMCS into the VMX
771 * transient structure.
772 *
773 * @returns VBox status code.
774 * @param pVmxTransient Pointer to the VMX transient structure.
775 */
776DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
777{
778 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
779 {
780 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
781 AssertRCReturn(rc, rc);
782 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
783 }
784 return VINF_SUCCESS;
785}
786
787
788/**
789 * Enters VMX root mode operation on the current CPU.
790 *
791 * @returns VBox status code.
792 * @param pVM The cross context VM structure. Can be
793 * NULL, after a resume.
794 * @param HCPhysCpuPage Physical address of the VMXON region.
795 * @param pvCpuPage Pointer to the VMXON region.
796 */
797static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
798{
799 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
800 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
801 Assert(pvCpuPage);
802 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
803
804 if (pVM)
805 {
806 /* Write the VMCS revision dword to the VMXON region. */
807 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
808 }
809
810 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
811 RTCCUINTREG fEFlags = ASMIntDisableFlags();
812
813 /* Enable the VMX bit in CR4 if necessary. */
814 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
815
816 /* Enter VMX root mode. */
817 int rc = VMXEnable(HCPhysCpuPage);
818 if (RT_FAILURE(rc))
819 {
820 if (!(uOldCr4 & X86_CR4_VMXE))
821 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
822
823 if (pVM)
824 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
825 }
826
827 /* Restore interrupts. */
828 ASMSetFlags(fEFlags);
829 return rc;
830}
831
832
833/**
834 * Exits VMX root mode operation on the current CPU.
835 *
836 * @returns VBox status code.
837 */
838static int hmR0VmxLeaveRootMode(void)
839{
840 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
841
842 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
843 RTCCUINTREG fEFlags = ASMIntDisableFlags();
844
845 /* If we're for some reason not in VMX root mode, then don't leave it. */
846 RTCCUINTREG uHostCR4 = ASMGetCR4();
847
848 int rc;
849 if (uHostCR4 & X86_CR4_VMXE)
850 {
851 /* Exit VMX root mode and clear the VMX bit in CR4. */
852 VMXDisable();
853 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
854 rc = VINF_SUCCESS;
855 }
856 else
857 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
858
859 /* Restore interrupts. */
860 ASMSetFlags(fEFlags);
861 return rc;
862}
863
864
865/**
866 * Allocates and maps one physically contiguous page. The allocated page is
867 * zero'd out. (Used by various VT-x structures).
868 *
869 * @returns IPRT status code.
870 * @param pMemObj Pointer to the ring-0 memory object.
871 * @param ppVirt Where to store the virtual address of the
872 * allocation.
873 * @param pHCPhys Where to store the physical address of the
874 * allocation.
875 */
876static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
877{
878 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
879 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
880 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
881
882 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
883 if (RT_FAILURE(rc))
884 return rc;
885 *ppVirt = RTR0MemObjAddress(*pMemObj);
886 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
887 ASMMemZero32(*ppVirt, PAGE_SIZE);
888 return VINF_SUCCESS;
889}
890
891
892/**
893 * Frees and unmaps an allocated physical page.
894 *
895 * @param pMemObj Pointer to the ring-0 memory object.
896 * @param ppVirt Where to re-initialize the virtual address of
897 * allocation as 0.
898 * @param pHCPhys Where to re-initialize the physical address of the
899 * allocation as 0.
900 */
901static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
902{
903 AssertPtr(pMemObj);
904 AssertPtr(ppVirt);
905 AssertPtr(pHCPhys);
906 if (*pMemObj != NIL_RTR0MEMOBJ)
907 {
908 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
909 AssertRC(rc);
910 *pMemObj = NIL_RTR0MEMOBJ;
911 *ppVirt = 0;
912 *pHCPhys = 0;
913 }
914}
915
916
917/**
918 * Worker function to free VT-x related structures.
919 *
920 * @returns IPRT status code.
921 * @param pVM The cross context VM structure.
922 */
923static void hmR0VmxStructsFree(PVM pVM)
924{
925 for (VMCPUID i = 0; i < pVM->cCpus; i++)
926 {
927 PVMCPU pVCpu = &pVM->aCpus[i];
928 AssertPtr(pVCpu);
929
930 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
931 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
932
933 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
934 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
935
936 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
937 }
938
939 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
940#ifdef VBOX_WITH_CRASHDUMP_MAGIC
941 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
942#endif
943}
944
945
946/**
947 * Worker function to allocate VT-x related VM structures.
948 *
949 * @returns IPRT status code.
950 * @param pVM The cross context VM structure.
951 */
952static int hmR0VmxStructsAlloc(PVM pVM)
953{
954 /*
955 * Initialize members up-front so we can cleanup properly on allocation failure.
956 */
957#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
958 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
959 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
960 pVM->hm.s.vmx.HCPhys##a_Name = 0;
961
962#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
963 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
964 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
965 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
966
967#ifdef VBOX_WITH_CRASHDUMP_MAGIC
968 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
969#endif
970 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
971
972 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
973 for (VMCPUID i = 0; i < pVM->cCpus; i++)
974 {
975 PVMCPU pVCpu = &pVM->aCpus[i];
976 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
977 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
978 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
979 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
980 }
981#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
982#undef VMXLOCAL_INIT_VM_MEMOBJ
983
984 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
985 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
986 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
987 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
988
989 /*
990 * Allocate all the VT-x structures.
991 */
992 int rc = VINF_SUCCESS;
993#ifdef VBOX_WITH_CRASHDUMP_MAGIC
994 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
995 if (RT_FAILURE(rc))
996 goto cleanup;
997 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
998 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
999#endif
1000
1001 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1002 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1003 {
1004 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1005 &pVM->hm.s.vmx.HCPhysApicAccess);
1006 if (RT_FAILURE(rc))
1007 goto cleanup;
1008 }
1009
1010 /*
1011 * Initialize per-VCPU VT-x structures.
1012 */
1013 for (VMCPUID i = 0; i < pVM->cCpus; i++)
1014 {
1015 PVMCPU pVCpu = &pVM->aCpus[i];
1016 AssertPtr(pVCpu);
1017
1018 /* Allocate the VM control structure (VMCS). */
1019 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1020 if (RT_FAILURE(rc))
1021 goto cleanup;
1022
1023 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1024 if ( PDMHasApic(pVM)
1025 && (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW))
1026 {
1027 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1028 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1029 if (RT_FAILURE(rc))
1030 goto cleanup;
1031 }
1032
1033 /*
1034 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1035 * transparent accesses of specific MSRs.
1036 *
1037 * If the condition for enabling MSR bitmaps changes here, don't forget to
1038 * update HMAreMsrBitmapsAvailable().
1039 */
1040 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1041 {
1042 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1043 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1044 if (RT_FAILURE(rc))
1045 goto cleanup;
1046 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1047 }
1048
1049 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1050 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1051 if (RT_FAILURE(rc))
1052 goto cleanup;
1053
1054 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1055 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1056 if (RT_FAILURE(rc))
1057 goto cleanup;
1058 }
1059
1060 return VINF_SUCCESS;
1061
1062cleanup:
1063 hmR0VmxStructsFree(pVM);
1064 return rc;
1065}
1066
1067
1068/**
1069 * Does global VT-x initialization (called during module initialization).
1070 *
1071 * @returns VBox status code.
1072 */
1073VMMR0DECL(int) VMXR0GlobalInit(void)
1074{
1075#ifdef HMVMX_USE_FUNCTION_TABLE
1076 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1077# ifdef VBOX_STRICT
1078 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1079 Assert(g_apfnVMExitHandlers[i]);
1080# endif
1081#endif
1082 return VINF_SUCCESS;
1083}
1084
1085
1086/**
1087 * Does global VT-x termination (called during module termination).
1088 */
1089VMMR0DECL(void) VMXR0GlobalTerm()
1090{
1091 /* Nothing to do currently. */
1092}
1093
1094
1095/**
1096 * Sets up and activates VT-x on the current CPU.
1097 *
1098 * @returns VBox status code.
1099 * @param pHostCpu Pointer to the global CPU info struct.
1100 * @param pVM The cross context VM structure. Can be
1101 * NULL after a host resume operation.
1102 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1103 * fEnabledByHost is @c true).
1104 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1105 * @a fEnabledByHost is @c true).
1106 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1107 * enable VT-x on the host.
1108 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1109 */
1110VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1111 void *pvMsrs)
1112{
1113 Assert(pHostCpu);
1114 Assert(pvMsrs);
1115 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1116
1117 /* Enable VT-x if it's not already enabled by the host. */
1118 if (!fEnabledByHost)
1119 {
1120 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1121 if (RT_FAILURE(rc))
1122 return rc;
1123 }
1124
1125 /*
1126 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
1127 * using EPTPs) so we don't retain any stale guest-physical mappings which won't get
1128 * invalidated when flushing by VPID.
1129 */
1130 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1131 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1132 {
1133 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1134 pHostCpu->fFlushAsidBeforeUse = false;
1135 }
1136 else
1137 pHostCpu->fFlushAsidBeforeUse = true;
1138
1139 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1140 ++pHostCpu->cTlbFlushes;
1141
1142 return VINF_SUCCESS;
1143}
1144
1145
1146/**
1147 * Deactivates VT-x on the current CPU.
1148 *
1149 * @returns VBox status code.
1150 * @param pHostCpu Pointer to the global CPU info struct.
1151 * @param pvCpuPage Pointer to the VMXON region.
1152 * @param HCPhysCpuPage Physical address of the VMXON region.
1153 *
1154 * @remarks This function should never be called when SUPR0EnableVTx() or
1155 * similar was used to enable VT-x on the host.
1156 */
1157VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1158{
1159 RT_NOREF3(pHostCpu, pvCpuPage, HCPhysCpuPage);
1160
1161 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1162 return hmR0VmxLeaveRootMode();
1163}
1164
1165
1166/**
1167 * Sets the permission bits for the specified MSR in the MSR bitmap.
1168 *
1169 * @param pVCpu The cross context virtual CPU structure.
1170 * @param uMsr The MSR value.
1171 * @param enmRead Whether reading this MSR causes a VM-exit.
1172 * @param enmWrite Whether writing this MSR causes a VM-exit.
1173 */
1174static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1175{
1176 int32_t iBit;
1177 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1178
1179 /*
1180 * Layout:
1181 * 0x000 - 0x3ff - Low MSR read bits
1182 * 0x400 - 0x7ff - High MSR read bits
1183 * 0x800 - 0xbff - Low MSR write bits
1184 * 0xc00 - 0xfff - High MSR write bits
1185 */
1186 if (uMsr <= 0x00001FFF)
1187 iBit = uMsr;
1188 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1189 {
1190 iBit = uMsr - UINT32_C(0xC0000000);
1191 pbMsrBitmap += 0x400;
1192 }
1193 else
1194 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1195
1196 Assert(iBit <= 0x1fff);
1197 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1198 ASMBitSet(pbMsrBitmap, iBit);
1199 else
1200 ASMBitClear(pbMsrBitmap, iBit);
1201
1202 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1203 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1204 else
1205 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1206}
1207
1208
1209#ifdef VBOX_STRICT
1210/**
1211 * Gets the permission bits for the specified MSR in the MSR bitmap.
1212 *
1213 * @returns VBox status code.
1214 * @retval VINF_SUCCESS if the specified MSR is found.
1215 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1216 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1217 *
1218 * @param pVCpu The cross context virtual CPU structure.
1219 * @param uMsr The MSR.
1220 * @param penmRead Where to store the read permissions.
1221 * @param penmWrite Where to store the write permissions.
1222 */
1223static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1224{
1225 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1226 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1227 int32_t iBit;
1228 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1229
1230 /* See hmR0VmxSetMsrPermission() for the layout. */
1231 if (uMsr <= 0x00001FFF)
1232 iBit = uMsr;
1233 else if ( uMsr >= 0xC0000000
1234 && uMsr <= 0xC0001FFF)
1235 {
1236 iBit = (uMsr - 0xC0000000);
1237 pbMsrBitmap += 0x400;
1238 }
1239 else
1240 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1241
1242 Assert(iBit <= 0x1fff);
1243 if (ASMBitTest(pbMsrBitmap, iBit))
1244 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1245 else
1246 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1247
1248 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1249 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1250 else
1251 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1252 return VINF_SUCCESS;
1253}
1254#endif /* VBOX_STRICT */
1255
1256
1257/**
1258 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1259 * area.
1260 *
1261 * @returns VBox status code.
1262 * @param pVCpu The cross context virtual CPU structure.
1263 * @param cMsrs The number of MSRs.
1264 */
1265static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1266{
1267 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1268 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1269 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1270 {
1271 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1272 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1273 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1274 }
1275
1276 /* Update number of guest MSRs to load/store across the world-switch. */
1277 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1278 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1279
1280 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1281 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1282 AssertRCReturn(rc, rc);
1283
1284 /* Update the VCPU's copy of the MSR count. */
1285 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1286
1287 return VINF_SUCCESS;
1288}
1289
1290
1291/**
1292 * Adds a new (or updates the value of an existing) guest/host MSR
1293 * pair to be swapped during the world-switch as part of the
1294 * auto-load/store MSR area in the VMCS.
1295 *
1296 * @returns VBox status code.
1297 * @param pVCpu The cross context virtual CPU structure.
1298 * @param uMsr The MSR.
1299 * @param uGuestMsrValue Value of the guest MSR.
1300 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1301 * necessary.
1302 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1303 * its value was updated. Optional, can be NULL.
1304 */
1305static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1306 bool *pfAddedAndUpdated)
1307{
1308 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1309 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1310 uint32_t i;
1311 for (i = 0; i < cMsrs; i++)
1312 {
1313 if (pGuestMsr->u32Msr == uMsr)
1314 break;
1315 pGuestMsr++;
1316 }
1317
1318 bool fAdded = false;
1319 if (i == cMsrs)
1320 {
1321 ++cMsrs;
1322 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1323 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1324
1325 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1326 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1327 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1328
1329 fAdded = true;
1330 }
1331
1332 /* Update the MSR values in the auto-load/store MSR area. */
1333 pGuestMsr->u32Msr = uMsr;
1334 pGuestMsr->u64Value = uGuestMsrValue;
1335
1336 /* Create/update the MSR slot in the host MSR area. */
1337 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1338 pHostMsr += i;
1339 pHostMsr->u32Msr = uMsr;
1340
1341 /*
1342 * Update the host MSR only when requested by the caller AND when we're
1343 * adding it to the auto-load/store area. Otherwise, it would have been
1344 * updated by hmR0VmxExportHostMsrs(). We do this for performance reasons.
1345 */
1346 bool fUpdatedMsrValue = false;
1347 if ( fAdded
1348 && fUpdateHostMsr)
1349 {
1350 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1351 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1352 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1353 fUpdatedMsrValue = true;
1354 }
1355
1356 if (pfAddedAndUpdated)
1357 *pfAddedAndUpdated = fUpdatedMsrValue;
1358 return VINF_SUCCESS;
1359}
1360
1361
1362/**
1363 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1364 * auto-load/store MSR area in the VMCS.
1365 *
1366 * @returns VBox status code.
1367 * @param pVCpu The cross context virtual CPU structure.
1368 * @param uMsr The MSR.
1369 */
1370static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1371{
1372 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1373 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1374 for (uint32_t i = 0; i < cMsrs; i++)
1375 {
1376 /* Find the MSR. */
1377 if (pGuestMsr->u32Msr == uMsr)
1378 {
1379 /* If it's the last MSR, simply reduce the count. */
1380 if (i == cMsrs - 1)
1381 {
1382 --cMsrs;
1383 break;
1384 }
1385
1386 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1387 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1388 pLastGuestMsr += cMsrs - 1;
1389 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1390 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1391
1392 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1393 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1394 pLastHostMsr += cMsrs - 1;
1395 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1396 pHostMsr->u64Value = pLastHostMsr->u64Value;
1397 --cMsrs;
1398 break;
1399 }
1400 pGuestMsr++;
1401 }
1402
1403 /* Update the VMCS if the count changed (meaning the MSR was found). */
1404 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1405 {
1406 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1407 AssertRCReturn(rc, rc);
1408
1409 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1410 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1411 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1412
1413 Log4Func(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1414 return VINF_SUCCESS;
1415 }
1416
1417 return VERR_NOT_FOUND;
1418}
1419
1420
1421/**
1422 * Checks if the specified guest MSR is part of the auto-load/store area in
1423 * the VMCS.
1424 *
1425 * @returns true if found, false otherwise.
1426 * @param pVCpu The cross context virtual CPU structure.
1427 * @param uMsr The MSR to find.
1428 */
1429static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1430{
1431 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1432 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1433
1434 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1435 {
1436 if (pGuestMsr->u32Msr == uMsr)
1437 return true;
1438 }
1439 return false;
1440}
1441
1442
1443/**
1444 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1445 *
1446 * @param pVCpu The cross context virtual CPU structure.
1447 *
1448 * @remarks No-long-jump zone!!!
1449 */
1450static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1451{
1452 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1453 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1454 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1455 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1456
1457 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1458 {
1459 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1460
1461 /*
1462 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1463 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1464 */
1465 if (pHostMsr->u32Msr == MSR_K6_EFER)
1466 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1467 else
1468 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1469 }
1470
1471 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1472}
1473
1474
1475/**
1476 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1477 * perform lazy restoration of the host MSRs while leaving VT-x.
1478 *
1479 * @param pVCpu The cross context virtual CPU structure.
1480 *
1481 * @remarks No-long-jump zone!!!
1482 */
1483static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1484{
1485 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1486
1487 /*
1488 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1489 */
1490 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1491 {
1492 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1493#if HC_ARCH_BITS == 64
1494 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1495 {
1496 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1497 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1498 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1499 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1500 }
1501#endif
1502 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1503 }
1504}
1505
1506
1507/**
1508 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1509 * lazily while leaving VT-x.
1510 *
1511 * @returns true if it does, false otherwise.
1512 * @param pVCpu The cross context virtual CPU structure.
1513 * @param uMsr The MSR to check.
1514 */
1515static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1516{
1517 NOREF(pVCpu);
1518#if HC_ARCH_BITS == 64
1519 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1520 {
1521 switch (uMsr)
1522 {
1523 case MSR_K8_LSTAR:
1524 case MSR_K6_STAR:
1525 case MSR_K8_SF_MASK:
1526 case MSR_K8_KERNEL_GS_BASE:
1527 return true;
1528 }
1529 }
1530#else
1531 RT_NOREF(pVCpu, uMsr);
1532#endif
1533 return false;
1534}
1535
1536
1537/**
1538 * Loads a set of guests MSRs to allow read/passthru to the guest.
1539 *
1540 * The name of this function is slightly confusing. This function does NOT
1541 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1542 * common prefix for functions dealing with "lazy restoration" of the shared
1543 * MSRs.
1544 *
1545 * @param pVCpu The cross context virtual CPU structure.
1546 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1547 * out-of-sync. Make sure to update the required fields
1548 * before using them.
1549 *
1550 * @remarks No-long-jump zone!!!
1551 */
1552static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1553{
1554 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1555 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1556
1557 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1558#if HC_ARCH_BITS == 64
1559 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1560 {
1561 /*
1562 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1563 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1564 * we can skip a few MSR writes.
1565 *
1566 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1567 * guest MSR values in the guest-CPU context might be different to what's currently
1568 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1569 * CPU, see @bugref{8728}.
1570 */
1571 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1572 && pMixedCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1573 && pMixedCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1574 && pMixedCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1575 && pMixedCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1576 {
1577#ifdef VBOX_STRICT
1578 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pMixedCtx->msrKERNELGSBASE);
1579 Assert(ASMRdMsr(MSR_K8_LSTAR) == pMixedCtx->msrLSTAR);
1580 Assert(ASMRdMsr(MSR_K6_STAR) == pMixedCtx->msrSTAR);
1581 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pMixedCtx->msrSFMASK);
1582#endif
1583 }
1584 else
1585 {
1586 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1587 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1588 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1589 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1590 }
1591 }
1592#else
1593 RT_NOREF(pMixedCtx);
1594#endif
1595 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1596}
1597
1598
1599/**
1600 * Performs lazy restoration of the set of host MSRs if they were previously
1601 * loaded with guest MSR values.
1602 *
1603 * @param pVCpu The cross context virtual CPU structure.
1604 *
1605 * @remarks No-long-jump zone!!!
1606 * @remarks The guest MSRs should have been saved back into the guest-CPU
1607 * context by hmR0VmxImportGuestState()!!!
1608 */
1609static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1610{
1611 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1612 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1613
1614 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1615 {
1616 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1617#if HC_ARCH_BITS == 64
1618 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1619 {
1620 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1621 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1622 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1623 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1624 }
1625#endif
1626 }
1627 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1628}
1629
1630
1631/**
1632 * Verifies that our cached values of the VMCS controls are all
1633 * consistent with what's actually present in the VMCS.
1634 *
1635 * @returns VBox status code.
1636 * @param pVCpu The cross context virtual CPU structure.
1637 */
1638static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1639{
1640 uint32_t u32Val;
1641 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1642 AssertRCReturn(rc, rc);
1643 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1644 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1645
1646 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1647 AssertRCReturn(rc, rc);
1648 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1649 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1650
1651 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1652 AssertRCReturn(rc, rc);
1653 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1654 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1655
1656 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1657 AssertRCReturn(rc, rc);
1658 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1659 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1660
1661 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1662 {
1663 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1664 AssertRCReturn(rc, rc);
1665 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1666 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1667 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1668 }
1669
1670 return VINF_SUCCESS;
1671}
1672
1673
1674#ifdef VBOX_STRICT
1675/**
1676 * Verifies that our cached host EFER value has not changed
1677 * since we cached it.
1678 *
1679 * @param pVCpu The cross context virtual CPU structure.
1680 */
1681static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1682{
1683 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1684
1685 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1686 {
1687 uint64_t u64Val;
1688 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1689 AssertRC(rc);
1690
1691 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1692 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1693 }
1694}
1695
1696
1697/**
1698 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1699 * VMCS are correct.
1700 *
1701 * @param pVCpu The cross context virtual CPU structure.
1702 */
1703static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1704{
1705 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1706
1707 /* Verify MSR counts in the VMCS are what we think it should be. */
1708 uint32_t cMsrs;
1709 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1710 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1711
1712 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1713 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1714
1715 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1716 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1717
1718 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1719 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1720 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1721 {
1722 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1723 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1724 pGuestMsr->u32Msr, cMsrs));
1725
1726 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1727 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1728 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1729
1730 /* Verify that the permissions are as expected in the MSR bitmap. */
1731 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1732 {
1733 VMXMSREXITREAD enmRead;
1734 VMXMSREXITWRITE enmWrite;
1735 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1736 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1737 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1738 {
1739 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1740 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1741 }
1742 else
1743 {
1744 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1745 pGuestMsr->u32Msr, cMsrs));
1746 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1747 pGuestMsr->u32Msr, cMsrs));
1748 }
1749 }
1750 }
1751}
1752#endif /* VBOX_STRICT */
1753
1754
1755/**
1756 * Flushes the TLB using EPT.
1757 *
1758 * @returns VBox status code.
1759 * @param pVCpu The cross context virtual CPU structure of the calling
1760 * EMT. Can be NULL depending on @a enmFlush.
1761 * @param enmFlush Type of flush.
1762 *
1763 * @remarks Caller is responsible for making sure this function is called only
1764 * when NestedPaging is supported and providing @a enmFlush that is
1765 * supported by the CPU.
1766 * @remarks Can be called with interrupts disabled.
1767 */
1768static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1769{
1770 uint64_t au64Descriptor[2];
1771 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1772 au64Descriptor[0] = 0;
1773 else
1774 {
1775 Assert(pVCpu);
1776 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1777 }
1778 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1779
1780 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1781 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1782 rc));
1783 if ( RT_SUCCESS(rc)
1784 && pVCpu)
1785 {
1786 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1787 }
1788}
1789
1790
1791/**
1792 * Flushes the TLB using VPID.
1793 *
1794 * @returns VBox status code.
1795 * @param pVCpu The cross context virtual CPU structure of the calling
1796 * EMT. Can be NULL depending on @a enmFlush.
1797 * @param enmFlush Type of flush.
1798 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1799 * on @a enmFlush).
1800 *
1801 * @remarks Can be called with interrupts disabled.
1802 */
1803static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1804{
1805 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
1806
1807 uint64_t au64Descriptor[2];
1808 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1809 {
1810 au64Descriptor[0] = 0;
1811 au64Descriptor[1] = 0;
1812 }
1813 else
1814 {
1815 AssertPtr(pVCpu);
1816 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1817 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1818 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1819 au64Descriptor[1] = GCPtr;
1820 }
1821
1822 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]);
1823 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmFlush,
1824 pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1825 if ( RT_SUCCESS(rc)
1826 && pVCpu)
1827 {
1828 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1829 }
1830 NOREF(rc);
1831}
1832
1833
1834/**
1835 * Invalidates a guest page by guest virtual address. Only relevant for
1836 * EPT/VPID, otherwise there is nothing really to invalidate.
1837 *
1838 * @returns VBox status code.
1839 * @param pVCpu The cross context virtual CPU structure.
1840 * @param GCVirt Guest virtual address of the page to invalidate.
1841 */
1842VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
1843{
1844 AssertPtr(pVCpu);
1845 LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
1846
1847 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1848 if (!fFlushPending)
1849 {
1850 /*
1851 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1852 * the EPT case. See @bugref{6043} and @bugref{6177}.
1853 *
1854 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1855 * as this function maybe called in a loop with individual addresses.
1856 */
1857 PVM pVM = pVCpu->CTX_SUFF(pVM);
1858 if (pVM->hm.s.vmx.fVpid)
1859 {
1860 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1861
1862#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1863 /*
1864 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1865 * where executing INVVPID outside 64-bit mode does not flush translations of
1866 * 64-bit linear addresses, see @bugref{6208#c72}.
1867 */
1868 if (RT_HI_U32(GCVirt))
1869 fVpidFlush = false;
1870#endif
1871
1872 if (fVpidFlush)
1873 {
1874 hmR0VmxFlushVpid(pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1875 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1876 }
1877 else
1878 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1879 }
1880 else if (pVM->hm.s.fNestedPaging)
1881 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1882 }
1883
1884 return VINF_SUCCESS;
1885}
1886
1887
1888/**
1889 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1890 * case where neither EPT nor VPID is supported by the CPU.
1891 *
1892 * @param pVCpu The cross context virtual CPU structure.
1893 * @param pCpu Pointer to the global HM struct.
1894 *
1895 * @remarks Called with interrupts disabled.
1896 */
1897static void hmR0VmxFlushTaggedTlbNone(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1898{
1899 AssertPtr(pVCpu);
1900 AssertPtr(pCpu);
1901
1902 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1903
1904 Assert(pCpu->idCpu != NIL_RTCPUID);
1905 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1906 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1907 pVCpu->hm.s.fForceTLBFlush = false;
1908 return;
1909}
1910
1911
1912/**
1913 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1914 *
1915 * @param pVCpu The cross context virtual CPU structure.
1916 * @param pCpu Pointer to the global HM CPU struct.
1917 *
1918 * @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
1919 * nomenclature. The reason is, to avoid confusion in compare statements
1920 * since the host-CPU copies are named "ASID".
1921 *
1922 * @remarks Called with interrupts disabled.
1923 */
1924static void hmR0VmxFlushTaggedTlbBoth(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1925{
1926#ifdef VBOX_WITH_STATISTICS
1927 bool fTlbFlushed = false;
1928# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1929# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1930 if (!fTlbFlushed) \
1931 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1932 } while (0)
1933#else
1934# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1935# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1936#endif
1937
1938 AssertPtr(pCpu);
1939 AssertPtr(pVCpu);
1940 Assert(pCpu->idCpu != NIL_RTCPUID);
1941
1942 PVM pVM = pVCpu->CTX_SUFF(pVM);
1943 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1944 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1945 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1946
1947 /*
1948 * Force a TLB flush for the first world-switch if the current CPU differs from the one we
1949 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1950 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1951 * cannot reuse the current ASID anymore.
1952 */
1953 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1954 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1955 {
1956 ++pCpu->uCurrentAsid;
1957 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1958 {
1959 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1960 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1961 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1962 }
1963
1964 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1965 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1966 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1967
1968 /*
1969 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1970 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1971 */
1972 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1973 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1974 HMVMX_SET_TAGGED_TLB_FLUSHED();
1975 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1976 }
1977
1978 /* Check for explicit TLB flushes. */
1979 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1980 {
1981 /*
1982 * Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
1983 * creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
1984 * tables when EPT is in use. Flushing-by-VPID will only flush linear (only
1985 * VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
1986 * mappings, see @bugref{6568}.
1987 *
1988 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
1989 */
1990 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1991 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1992 HMVMX_SET_TAGGED_TLB_FLUSHED();
1993 }
1994
1995 pVCpu->hm.s.fForceTLBFlush = false;
1996 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1997
1998 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1999 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2000 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2001 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2002 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2003 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2004 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2005 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2006 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2007
2008 /* Update VMCS with the VPID. */
2009 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2010 AssertRC(rc);
2011
2012#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2013}
2014
2015
2016/**
2017 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2018 *
2019 * @returns VBox status code.
2020 * @param pVCpu The cross context virtual CPU structure.
2021 * @param pCpu Pointer to the global HM CPU struct.
2022 *
2023 * @remarks Called with interrupts disabled.
2024 */
2025static void hmR0VmxFlushTaggedTlbEpt(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2026{
2027 AssertPtr(pVCpu);
2028 AssertPtr(pCpu);
2029 Assert(pCpu->idCpu != NIL_RTCPUID);
2030 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2031 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2032
2033 /*
2034 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2035 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2036 */
2037 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2038 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2039 {
2040 pVCpu->hm.s.fForceTLBFlush = true;
2041 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2042 }
2043
2044 /* Check for explicit TLB flushes. */
2045 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2046 {
2047 pVCpu->hm.s.fForceTLBFlush = true;
2048 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2049 }
2050
2051 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2052 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2053
2054 if (pVCpu->hm.s.fForceTLBFlush)
2055 {
2056 hmR0VmxFlushEpt(pVCpu, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmFlushEpt);
2057 pVCpu->hm.s.fForceTLBFlush = false;
2058 }
2059}
2060
2061
2062/**
2063 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2064 *
2065 * @returns VBox status code.
2066 * @param pVCpu The cross context virtual CPU structure.
2067 * @param pCpu Pointer to the global HM CPU struct.
2068 *
2069 * @remarks Called with interrupts disabled.
2070 */
2071static void hmR0VmxFlushTaggedTlbVpid(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2072{
2073 AssertPtr(pVCpu);
2074 AssertPtr(pCpu);
2075 Assert(pCpu->idCpu != NIL_RTCPUID);
2076 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2077 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2078
2079 /*
2080 * Force a TLB flush for the first world switch if the current CPU differs from the one we
2081 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2082 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2083 * cannot reuse the current ASID anymore.
2084 */
2085 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2086 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2087 {
2088 pVCpu->hm.s.fForceTLBFlush = true;
2089 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2090 }
2091
2092 /* Check for explicit TLB flushes. */
2093 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2094 {
2095 /*
2096 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2097 * hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2098 * fExplicitFlush = true here and change the pCpu->fFlushAsidBeforeUse check below to
2099 * include fExplicitFlush's too) - an obscure corner case.
2100 */
2101 pVCpu->hm.s.fForceTLBFlush = true;
2102 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2103 }
2104
2105 PVM pVM = pVCpu->CTX_SUFF(pVM);
2106 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2107 if (pVCpu->hm.s.fForceTLBFlush)
2108 {
2109 ++pCpu->uCurrentAsid;
2110 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2111 {
2112 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2113 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2114 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2115 }
2116
2117 pVCpu->hm.s.fForceTLBFlush = false;
2118 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2119 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2120 if (pCpu->fFlushAsidBeforeUse)
2121 {
2122 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2123 hmR0VmxFlushVpid(pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2124 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2125 {
2126 hmR0VmxFlushVpid(pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2127 pCpu->fFlushAsidBeforeUse = false;
2128 }
2129 else
2130 {
2131 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2132 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2133 }
2134 }
2135 }
2136
2137 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2138 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2139 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2140 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2141 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2142 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2143 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2144
2145 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2146 AssertRC(rc);
2147}
2148
2149
2150/**
2151 * Flushes the guest TLB entry based on CPU capabilities.
2152 *
2153 * @param pVCpu The cross context virtual CPU structure.
2154 * @param pCpu Pointer to the global HM CPU struct.
2155 */
2156DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2157{
2158#ifdef HMVMX_ALWAYS_FLUSH_TLB
2159 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2160#endif
2161 PVM pVM = pVCpu->CTX_SUFF(pVM);
2162 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2163 {
2164 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVCpu, pCpu); break;
2165 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVCpu, pCpu); break;
2166 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVCpu, pCpu); break;
2167 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVCpu, pCpu); break;
2168 default:
2169 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2170 break;
2171 }
2172 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2173}
2174
2175
2176/**
2177 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2178 * TLB entries from the host TLB before VM-entry.
2179 *
2180 * @returns VBox status code.
2181 * @param pVM The cross context VM structure.
2182 */
2183static int hmR0VmxSetupTaggedTlb(PVM pVM)
2184{
2185 /*
2186 * Determine optimal flush type for Nested Paging.
2187 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2188 * guest execution (see hmR3InitFinalizeR0()).
2189 */
2190 if (pVM->hm.s.fNestedPaging)
2191 {
2192 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2193 {
2194 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2195 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2196 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2197 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2198 else
2199 {
2200 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2201 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2202 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2203 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2204 }
2205
2206 /* Make sure the write-back cacheable memory type for EPT is supported. */
2207 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2208 {
2209 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2210 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2211 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2212 }
2213
2214 /* EPT requires a page-walk length of 4. */
2215 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2216 {
2217 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2218 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2219 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2220 }
2221 }
2222 else
2223 {
2224 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2225 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2226 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2227 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2228 }
2229 }
2230
2231 /*
2232 * Determine optimal flush type for VPID.
2233 */
2234 if (pVM->hm.s.vmx.fVpid)
2235 {
2236 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2237 {
2238 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2239 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2240 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2241 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2242 else
2243 {
2244 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2245 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2246 LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2247 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2248 LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2249 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2250 pVM->hm.s.vmx.fVpid = false;
2251 }
2252 }
2253 else
2254 {
2255 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2256 Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2257 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2258 pVM->hm.s.vmx.fVpid = false;
2259 }
2260 }
2261
2262 /*
2263 * Setup the handler for flushing tagged-TLBs.
2264 */
2265 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2266 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2267 else if (pVM->hm.s.fNestedPaging)
2268 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2269 else if (pVM->hm.s.vmx.fVpid)
2270 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2271 else
2272 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2273 return VINF_SUCCESS;
2274}
2275
2276
2277/**
2278 * Sets up pin-based VM-execution controls in the VMCS.
2279 *
2280 * @returns VBox status code.
2281 * @param pVCpu The cross context virtual CPU structure.
2282 *
2283 * @remarks We don't really care about optimizing vmwrites here as it's done only
2284 * once per VM and hence we don't care about VMCS-field cache comparisons.
2285 */
2286static int hmR0VmxSetupPinCtls(PVMCPU pVCpu)
2287{
2288 PVM pVM = pVCpu->CTX_SUFF(pVM);
2289 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2290 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2291
2292 fVal |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2293 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2294
2295 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2296 fVal |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2297
2298 /* Enable the VMX preemption timer. */
2299 if (pVM->hm.s.vmx.fUsePreemptTimer)
2300 {
2301 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2302 fVal |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2303 }
2304
2305#if 0
2306 /* Enable posted-interrupt processing. */
2307 if (pVM->hm.s.fPostedIntrs)
2308 {
2309 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2310 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2311 fVal |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2312 }
2313#endif
2314
2315 if ((fVal & fZap) != fVal)
2316 {
2317 LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2318 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, fVal, fZap));
2319 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2320 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2321 }
2322
2323 /* Commit it to the VMCS and update our cache. */
2324 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2325 AssertRCReturn(rc, rc);
2326 pVCpu->hm.s.vmx.u32PinCtls = fVal;
2327
2328 return VINF_SUCCESS;
2329}
2330
2331
2332/**
2333 * Sets up secondary processor-based VM-execution controls in the VMCS.
2334 *
2335 * @returns VBox status code.
2336 * @param pVCpu The cross context virtual CPU structure.
2337 *
2338 * @remarks We don't really care about optimizing vmwrites here as it's done only
2339 * once per VM and hence we don't care about VMCS-field cache comparisons.
2340 */
2341static int hmR0VmxSetupProcCtls2(PVMCPU pVCpu)
2342{
2343 PVM pVM = pVCpu->CTX_SUFF(pVM);
2344 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2345 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2346
2347 /* WBINVD causes a VM-exit. */
2348 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2349 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
2350
2351 /* Enable EPT (aka nested-paging). */
2352 if (pVM->hm.s.fNestedPaging)
2353 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
2354
2355 /*
2356 * Enable the INVPCID instruction if supported by the hardware and we expose
2357 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2358 */
2359 if ( (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2360 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2361 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2362
2363 /* Enable VPID. */
2364 if (pVM->hm.s.vmx.fVpid)
2365 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
2366
2367 /* Enable Unrestricted guest execution. */
2368 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2369 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST;
2370
2371#if 0
2372 if (pVM->hm.s.fVirtApicRegs)
2373 {
2374 /* Enable APIC-register virtualization. */
2375 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2376 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT;
2377
2378 /* Enable virtual-interrupt delivery. */
2379 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2380 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY;
2381 }
2382#endif
2383
2384 /* Enable Virtual-APIC page accesses if supported by the CPU. This is where the TPR shadow resides. */
2385 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2386 * done dynamically. */
2387 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2388 {
2389 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2390 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2391 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2392 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2393 AssertRCReturn(rc, rc);
2394 }
2395
2396 /* Enable RDTSCP. */
2397 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2398 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP;
2399
2400 /* Enable Pause-Loop exiting. */
2401 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2402 && pVM->hm.s.vmx.cPleGapTicks
2403 && pVM->hm.s.vmx.cPleWindowTicks)
2404 {
2405 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT;
2406
2407 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2408 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2409 AssertRCReturn(rc, rc);
2410 }
2411
2412 if ((fVal & fZap) != fVal)
2413 {
2414 LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2415 pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, fVal, fZap));
2416 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2417 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2418 }
2419
2420 /* Commit it to the VMCS and update our cache. */
2421 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2422 AssertRCReturn(rc, rc);
2423 pVCpu->hm.s.vmx.u32ProcCtls2 = fVal;
2424
2425 return VINF_SUCCESS;
2426}
2427
2428
2429/**
2430 * Sets up processor-based VM-execution controls in the VMCS.
2431 *
2432 * @returns VBox status code.
2433 * @param pVCpu The cross context virtual CPU structure.
2434 *
2435 * @remarks We don't really care about optimizing vmwrites here as it's done only
2436 * once per VM and hence we don't care about VMCS-field cache comparisons.
2437 */
2438static int hmR0VmxSetupProcCtls(PVMCPU pVCpu)
2439{
2440 PVM pVM = pVCpu->CTX_SUFF(pVM);
2441 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2442 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2443
2444 fVal |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2445 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2446 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2447 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2448 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2449 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2450 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2451
2452 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2453 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2454 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2455 {
2456 LogRelFunc(("Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2457 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2458 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2459 }
2460
2461 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2462 if (!pVM->hm.s.fNestedPaging)
2463 {
2464 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2465 fVal |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2466 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2467 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2468 }
2469
2470 /* Use TPR shadowing if supported by the CPU. */
2471 if ( PDMHasApic(pVM)
2472 && pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2473 {
2474 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2475 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2476 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2477 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2478 AssertRCReturn(rc, rc);
2479
2480 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2481 /* CR8 writes cause a VM-exit based on TPR threshold. */
2482 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2483 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2484 }
2485 else
2486 {
2487 /*
2488 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2489 * Set this control only for 64-bit guests.
2490 */
2491 if (pVM->hm.s.fAllow64BitGuests)
2492 {
2493 fVal |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2494 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2495 }
2496 }
2497
2498 /* Use MSR-bitmaps if supported by the CPU. */
2499 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2500 {
2501 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2502
2503 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2504 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2505 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2506 AssertRCReturn(rc, rc);
2507
2508 /*
2509 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2510 * automatically using dedicated fields in the VMCS.
2511 */
2512 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2513 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2514 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2515 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2516 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2517#if HC_ARCH_BITS == 64
2518 /*
2519 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2520 */
2521 if (pVM->hm.s.fAllow64BitGuests)
2522 {
2523 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2524 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2525 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2526 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2527 }
2528#endif
2529 /*
2530 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2531 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2532 */
2533 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2534 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2535
2536 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2537 }
2538
2539 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2540 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2541 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2542
2543 if ((fVal & fZap) != fVal)
2544 {
2545 LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2546 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, fVal, fZap));
2547 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2548 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2549 }
2550
2551 /* Commit it to the VMCS and update our cache. */
2552 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2553 AssertRCReturn(rc, rc);
2554 pVCpu->hm.s.vmx.u32ProcCtls = fVal;
2555
2556 /* Set up secondary processor-based VM-execution controls if the CPU supports it. */
2557 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2558 return hmR0VmxSetupProcCtls2(pVCpu);
2559
2560 /* Sanity check, should not really happen. */
2561 if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2562 {
2563 LogRelFunc(("Unrestricted Guest enabled when secondary processor-based VM-execution controls not available\n"));
2564 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2565 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2566 }
2567
2568 /* Old CPUs without secondary processor-based VM-execution controls would end up here. */
2569 return VINF_SUCCESS;
2570}
2571
2572
2573/**
2574 * Sets up miscellaneous (everything other than Pin & Processor-based
2575 * VM-execution) control fields in the VMCS.
2576 *
2577 * @returns VBox status code.
2578 * @param pVCpu The cross context virtual CPU structure.
2579 */
2580static int hmR0VmxSetupMiscCtls(PVMCPU pVCpu)
2581{
2582 AssertPtr(pVCpu);
2583
2584 int rc = VERR_GENERAL_FAILURE;
2585
2586 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2587#if 0
2588 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxExportGuestCR3AndCR4())*/
2589 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2590 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2591
2592 /*
2593 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2594 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2595 * We thus use the exception bitmap to control it rather than use both.
2596 */
2597 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2598 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2599
2600 /* All IO & IOIO instructions cause VM-exits. */
2601 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2602 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2603
2604 /* Initialize the MSR-bitmap area. */
2605 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2606 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2607 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2608 AssertRCReturn(rc, rc);
2609#endif
2610
2611 /* Setup MSR auto-load/store area. */
2612 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2613 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2614 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2615 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2616 AssertRCReturn(rc, rc);
2617
2618 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2619 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2620 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2621 AssertRCReturn(rc, rc);
2622
2623 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2624 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2625 AssertRCReturn(rc, rc);
2626
2627 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2628#if 0
2629 /* Setup debug controls */
2630 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2631 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2632 AssertRCReturn(rc, rc);
2633#endif
2634
2635 return rc;
2636}
2637
2638
2639/**
2640 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2641 *
2642 * We shall setup those exception intercepts that don't change during the
2643 * lifetime of the VM here. The rest are done dynamically while loading the
2644 * guest state.
2645 *
2646 * @returns VBox status code.
2647 * @param pVCpu The cross context virtual CPU structure.
2648 */
2649static int hmR0VmxInitXcptBitmap(PVMCPU pVCpu)
2650{
2651 AssertPtr(pVCpu);
2652
2653 uint32_t u32XcptBitmap;
2654
2655 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2656 u32XcptBitmap = RT_BIT_32(X86_XCPT_AC);
2657
2658 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2659 and writes, and because recursive #DBs can cause the CPU hang, we must always
2660 intercept #DB. */
2661 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2662
2663 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2664 if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
2665 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2666
2667 /* Commit it to the VMCS. */
2668 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2669 AssertRCReturn(rc, rc);
2670
2671 /* Update our cache of the exception bitmap. */
2672 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2673 return VINF_SUCCESS;
2674}
2675
2676
2677/**
2678 * Does per-VM VT-x initialization.
2679 *
2680 * @returns VBox status code.
2681 * @param pVM The cross context VM structure.
2682 */
2683VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2684{
2685 LogFlowFunc(("pVM=%p\n", pVM));
2686
2687 int rc = hmR0VmxStructsAlloc(pVM);
2688 if (RT_FAILURE(rc))
2689 {
2690 LogRelFunc(("hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2691 return rc;
2692 }
2693
2694 return VINF_SUCCESS;
2695}
2696
2697
2698/**
2699 * Does per-VM VT-x termination.
2700 *
2701 * @returns VBox status code.
2702 * @param pVM The cross context VM structure.
2703 */
2704VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2705{
2706 LogFlowFunc(("pVM=%p\n", pVM));
2707
2708#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2709 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2710 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2711#endif
2712 hmR0VmxStructsFree(pVM);
2713 return VINF_SUCCESS;
2714}
2715
2716
2717/**
2718 * Sets up the VM for execution under VT-x.
2719 * This function is only called once per-VM during initialization.
2720 *
2721 * @returns VBox status code.
2722 * @param pVM The cross context VM structure.
2723 */
2724VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2725{
2726 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2727 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2728
2729 LogFlowFunc(("pVM=%p\n", pVM));
2730
2731 /*
2732 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be
2733 * allocated. We no longer support the highly unlikely case of UnrestrictedGuest without
2734 * pRealModeTSS, see hmR3InitFinalizeR0Intel().
2735 */
2736 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2737 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2738 || !pVM->hm.s.vmx.pRealModeTSS))
2739 {
2740 LogRelFunc(("Invalid real-on-v86 state.\n"));
2741 return VERR_INTERNAL_ERROR;
2742 }
2743
2744 /* Initialize these always, see hmR3InitFinalizeR0().*/
2745 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2746 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2747
2748 /* Setup the tagged-TLB flush handlers. */
2749 int rc = hmR0VmxSetupTaggedTlb(pVM);
2750 if (RT_FAILURE(rc))
2751 {
2752 LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2753 return rc;
2754 }
2755
2756 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2757 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2758#if HC_ARCH_BITS == 64
2759 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2760 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2761 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2762 {
2763 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2764 }
2765#endif
2766
2767 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2768 RTCCUINTREG uHostCR4 = ASMGetCR4();
2769 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2770 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2771
2772 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2773 {
2774 PVMCPU pVCpu = &pVM->aCpus[i];
2775 AssertPtr(pVCpu);
2776 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2777
2778 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2779 Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2780
2781 /* Set revision dword at the beginning of the VMCS structure. */
2782 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2783
2784 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2785 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2786 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc\n", rc),
2787 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2788
2789 /* Load this VMCS as the current VMCS. */
2790 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2791 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc\n", rc),
2792 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2793
2794 rc = hmR0VmxSetupPinCtls(pVCpu);
2795 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc\n", rc),
2796 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2797
2798 rc = hmR0VmxSetupProcCtls(pVCpu);
2799 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc\n", rc),
2800 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2801
2802 rc = hmR0VmxSetupMiscCtls(pVCpu);
2803 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc\n", rc),
2804 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2805
2806 rc = hmR0VmxInitXcptBitmap(pVCpu);
2807 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc\n", rc),
2808 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2809
2810#if HC_ARCH_BITS == 32
2811 rc = hmR0VmxInitVmcsReadCache(pVCpu);
2812 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc\n", rc),
2813 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2814#endif
2815
2816 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2817 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2818 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc\n", rc),
2819 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2820
2821 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2822
2823 hmR0VmxUpdateErrorRecord(pVCpu, rc);
2824 }
2825
2826 return VINF_SUCCESS;
2827}
2828
2829
2830/**
2831 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2832 * the VMCS.
2833 *
2834 * @returns VBox status code.
2835 */
2836static int hmR0VmxExportHostControlRegs(void)
2837{
2838 RTCCUINTREG uReg = ASMGetCR0();
2839 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2840 AssertRCReturn(rc, rc);
2841
2842 uReg = ASMGetCR3();
2843 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2844 AssertRCReturn(rc, rc);
2845
2846 uReg = ASMGetCR4();
2847 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2848 AssertRCReturn(rc, rc);
2849 return rc;
2850}
2851
2852
2853/**
2854 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2855 * the host-state area in the VMCS.
2856 *
2857 * @returns VBox status code.
2858 * @param pVCpu The cross context virtual CPU structure.
2859 */
2860static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
2861{
2862#if HC_ARCH_BITS == 64
2863/**
2864 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2865 * requirements. See hmR0VmxExportHostSegmentRegs().
2866 */
2867# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2868 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2869 { \
2870 bool fValidSelector = true; \
2871 if ((selValue) & X86_SEL_LDT) \
2872 { \
2873 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2874 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2875 } \
2876 if (fValidSelector) \
2877 { \
2878 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2879 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2880 } \
2881 (selValue) = 0; \
2882 }
2883
2884 /*
2885 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2886 * should -not- save the messed up state without restoring the original host-state,
2887 * see @bugref{7240}.
2888 *
2889 * This apparently can happen (most likely the FPU changes), deal with it rather than
2890 * asserting. Was observed booting Solaris 10u10 32-bit guest.
2891 */
2892 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2893 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2894 {
2895 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2896 pVCpu->idCpu));
2897 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2898 }
2899 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2900#else
2901 RT_NOREF(pVCpu);
2902#endif
2903
2904 /*
2905 * Host DS, ES, FS and GS segment registers.
2906 */
2907#if HC_ARCH_BITS == 64
2908 RTSEL uSelDS = ASMGetDS();
2909 RTSEL uSelES = ASMGetES();
2910 RTSEL uSelFS = ASMGetFS();
2911 RTSEL uSelGS = ASMGetGS();
2912#else
2913 RTSEL uSelDS = 0;
2914 RTSEL uSelES = 0;
2915 RTSEL uSelFS = 0;
2916 RTSEL uSelGS = 0;
2917#endif
2918
2919 /*
2920 * Host CS and SS segment registers.
2921 */
2922 RTSEL uSelCS = ASMGetCS();
2923 RTSEL uSelSS = ASMGetSS();
2924
2925 /*
2926 * Host TR segment register.
2927 */
2928 RTSEL uSelTR = ASMGetTR();
2929
2930#if HC_ARCH_BITS == 64
2931 /*
2932 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
2933 * gain VM-entry and restore them before we get preempted.
2934 *
2935 * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2936 */
2937 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2938 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2939 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2940 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2941# undef VMXLOCAL_ADJUST_HOST_SEG
2942#endif
2943
2944 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2945 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2946 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2947 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2948 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2949 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2950 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2951 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2952 Assert(uSelCS);
2953 Assert(uSelTR);
2954
2955 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2956#if 0
2957 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2958 Assert(uSelSS != 0);
2959#endif
2960
2961 /* Write these host selector fields into the host-state area in the VMCS. */
2962 int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2963 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2964#if HC_ARCH_BITS == 64
2965 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2966 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2967 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2968 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2969#else
2970 NOREF(uSelDS);
2971 NOREF(uSelES);
2972 NOREF(uSelFS);
2973 NOREF(uSelGS);
2974#endif
2975 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2976 AssertRCReturn(rc, rc);
2977
2978 /*
2979 * Host GDTR and IDTR.
2980 */
2981 RTGDTR Gdtr;
2982 RTIDTR Idtr;
2983 RT_ZERO(Gdtr);
2984 RT_ZERO(Idtr);
2985 ASMGetGDTR(&Gdtr);
2986 ASMGetIDTR(&Idtr);
2987 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2988 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2989 AssertRCReturn(rc, rc);
2990
2991#if HC_ARCH_BITS == 64
2992 /*
2993 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
2994 * them to the maximum limit (0xffff) on every VM-exit.
2995 */
2996 if (Gdtr.cbGdt != 0xffff)
2997 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2998
2999 /*
3000 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
3001 * Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
3002 * as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
3003 * However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
3004 * uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
3005 * alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
3006 * at 0xffff on hosts where we are sure it won't cause trouble.
3007 */
3008# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3009 if (Idtr.cbIdt < 0x0fff)
3010# else
3011 if (Idtr.cbIdt != 0xffff)
3012# endif
3013 {
3014 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3015 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3016 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3017 }
3018#endif
3019
3020 /*
3021 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3022 * and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3023 * RPL should be too in most cases.
3024 */
3025 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3026 ("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3027
3028 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3029#if HC_ARCH_BITS == 64
3030 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3031
3032 /*
3033 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3034 * all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3035 * restoration if the host has something else. Task switching is not supported in 64-bit
3036 * mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3037 * limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3038 *
3039 * [1] See Intel spec. 3.5 "System Descriptor Types".
3040 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3041 */
3042 PVM pVM = pVCpu->CTX_SUFF(pVM);
3043 Assert(pDesc->System.u4Type == 11);
3044 if ( pDesc->System.u16LimitLow != 0x67
3045 || pDesc->System.u4LimitHigh)
3046 {
3047 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3048 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3049 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3050 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3051 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3052 }
3053
3054 /*
3055 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3056 */
3057 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3058 {
3059 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3060 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3061 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3062 {
3063 /* The GDT is read-only but the writable GDT is available. */
3064 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3065 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3066 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3067 AssertRCReturn(rc, rc);
3068 }
3069 }
3070#else
3071 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3072#endif
3073 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3074 AssertRCReturn(rc, rc);
3075
3076 /*
3077 * Host FS base and GS base.
3078 */
3079#if HC_ARCH_BITS == 64
3080 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3081 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3082 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3083 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3084 AssertRCReturn(rc, rc);
3085
3086 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3087 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3088 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3089 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3090 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3091#endif
3092 return VINF_SUCCESS;
3093}
3094
3095
3096/**
3097 * Exports certain host MSRs in the VM-exit MSR-load area and some in the
3098 * host-state area of the VMCS.
3099 *
3100 * Theses MSRs will be automatically restored on the host after every successful
3101 * VM-exit.
3102 *
3103 * @returns VBox status code.
3104 * @param pVCpu The cross context virtual CPU structure.
3105 *
3106 * @remarks No-long-jump zone!!!
3107 */
3108static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
3109{
3110 AssertPtr(pVCpu);
3111 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3112
3113 /*
3114 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3115 * rather than swapping them on every VM-entry.
3116 */
3117 hmR0VmxLazySaveHostMsrs(pVCpu);
3118
3119 /*
3120 * Host Sysenter MSRs.
3121 */
3122 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3123#if HC_ARCH_BITS == 32
3124 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3125 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3126#else
3127 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3128 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3129#endif
3130 AssertRCReturn(rc, rc);
3131
3132 /*
3133 * Host EFER MSR.
3134 *
3135 * If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3136 * done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3137 */
3138 PVM pVM = pVCpu->CTX_SUFF(pVM);
3139 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3140 {
3141 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3142 AssertRCReturn(rc, rc);
3143 }
3144
3145 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see hmR0VmxExportGuestExitCtls(). */
3146
3147 return VINF_SUCCESS;
3148}
3149
3150
3151/**
3152 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3153 *
3154 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3155 * these two bits are handled by VM-entry, see hmR0VmxExportGuestExitCtls() and
3156 * hmR0VMxExportGuestEntryCtls().
3157 *
3158 * @returns true if we need to load guest EFER, false otherwise.
3159 * @param pVCpu The cross context virtual CPU structure.
3160 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3161 * out-of-sync. Make sure to update the required fields
3162 * before using them.
3163 *
3164 * @remarks Requires EFER, CR4.
3165 * @remarks No-long-jump zone!!!
3166 */
3167static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3168{
3169#ifdef HMVMX_ALWAYS_SWAP_EFER
3170 return true;
3171#endif
3172
3173#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3174 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3175 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3176 return false;
3177#endif
3178
3179 PVM pVM = pVCpu->CTX_SUFF(pVM);
3180 uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3181 uint64_t const u64GuestEfer = pMixedCtx->msrEFER;
3182
3183 /*
3184 * For 64-bit guests, if EFER.SCE bit differs, we need to swap EFER to ensure that the
3185 * guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3186 */
3187 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3188 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3189 {
3190 return true;
3191 }
3192
3193 /*
3194 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3195 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3196 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3197 */
3198 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3199 && (pMixedCtx->cr0 & X86_CR0_PG)
3200 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3201 {
3202 /* Assert that host is PAE capable. */
3203 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3204 return true;
3205 }
3206
3207 return false;
3208}
3209
3210
3211/**
3212 * Exports the guest state with appropriate VM-entry controls in the VMCS.
3213 *
3214 * These controls can affect things done on VM-exit; e.g. "load debug controls",
3215 * see Intel spec. 24.8.1 "VM-entry controls".
3216 *
3217 * @returns VBox status code.
3218 * @param pVCpu The cross context virtual CPU structure.
3219 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3220 * out-of-sync. Make sure to update the required fields
3221 * before using them.
3222 *
3223 * @remarks Requires EFER.
3224 * @remarks No-long-jump zone!!!
3225 */
3226static int hmR0VmxExportGuestEntryCtls(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3227{
3228 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_CTLS)
3229 {
3230 PVM pVM = pVCpu->CTX_SUFF(pVM);
3231 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3232 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3233
3234 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3235 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3236
3237 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3238 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3239 {
3240 fVal |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3241 Log4Func(("VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n"));
3242 }
3243 else
3244 Assert(!(fVal & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3245
3246 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3247 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3248 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3249 {
3250 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3251 Log4Func(("VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n"));
3252 }
3253
3254 /*
3255 * The following should -not- be set (since we're not in SMM mode):
3256 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3257 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3258 */
3259
3260 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3261 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3262
3263 if ((fVal & fZap) != fVal)
3264 {
3265 Log4Func(("Invalid VM-entry controls combo! Cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3266 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, fVal, fZap));
3267 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3268 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3269 }
3270
3271 /* Commit it to the VMCS and update our cache. */
3272 if (pVCpu->hm.s.vmx.u32EntryCtls != fVal)
3273 {
3274 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
3275 AssertRCReturn(rc, rc);
3276 pVCpu->hm.s.vmx.u32EntryCtls = fVal;
3277 }
3278
3279 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_CTLS);
3280 }
3281 return VINF_SUCCESS;
3282}
3283
3284
3285/**
3286 * Exports the guest state with appropriate VM-exit controls in the VMCS.
3287 *
3288 * @returns VBox status code.
3289 * @param pVCpu The cross context virtual CPU structure.
3290 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3291 * out-of-sync. Make sure to update the required fields
3292 * before using them.
3293 *
3294 * @remarks Requires EFER.
3295 */
3296static int hmR0VmxExportGuestExitCtls(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3297{
3298 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_EXIT_CTLS)
3299 {
3300 PVM pVM = pVCpu->CTX_SUFF(pVM);
3301 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3302 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3303
3304 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3305 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3306
3307 /*
3308 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3309 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in
3310 * hmR0VmxExportHostMsrs().
3311 */
3312#if HC_ARCH_BITS == 64
3313 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3314 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3315#else
3316 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3317 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3318 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3319 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3320 {
3321 /* The switcher returns to long mode, EFER is managed by the switcher. */
3322 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3323 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3324 }
3325 else
3326 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3327#endif
3328
3329 /* If the newer VMCS fields for managing EFER exists, use it. */
3330 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3331 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3332 {
3333 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3334 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3335 Log4Func(("VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR and VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n"));
3336 }
3337
3338 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3339 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3340
3341 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3342 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3343 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3344
3345 /* Enable saving of the VMX preemption timer value on VM-exit. */
3346 if ( pVM->hm.s.vmx.fUsePreemptTimer
3347 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3348 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3349
3350 if ((fVal & fZap) != fVal)
3351 {
3352 LogRelFunc(("Invalid VM-exit controls combo! cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3353 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, fVal, fZap));
3354 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3355 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3356 }
3357
3358 /* Commit it to the VMCS and update our cache. */
3359 if (pVCpu->hm.s.vmx.u32ExitCtls != fVal)
3360 {
3361 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
3362 AssertRCReturn(rc, rc);
3363 pVCpu->hm.s.vmx.u32ExitCtls = fVal;
3364 }
3365
3366 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_EXIT_CTLS);
3367 }
3368 return VINF_SUCCESS;
3369}
3370
3371
3372/**
3373 * Sets the TPR threshold in the VMCS.
3374 *
3375 * @returns VBox status code.
3376 * @param pVCpu The cross context virtual CPU structure.
3377 * @param u32TprThreshold The TPR threshold (task-priority class only).
3378 */
3379DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3380{
3381 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3382 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3383 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3384}
3385
3386
3387/**
3388 * Exports the guest APIC TPR state into the VMCS.
3389 *
3390 * @returns VBox status code.
3391 * @param pVCpu The cross context virtual CPU structure.
3392 *
3393 * @remarks No-long-jump zone!!!
3394 */
3395static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu)
3396{
3397 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
3398 {
3399 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
3400
3401 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3402 && APICIsEnabled(pVCpu))
3403 {
3404 /*
3405 * Setup TPR shadowing.
3406 */
3407 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3408 {
3409 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3410
3411 bool fPendingIntr = false;
3412 uint8_t u8Tpr = 0;
3413 uint8_t u8PendingIntr = 0;
3414 int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3415 AssertRCReturn(rc, rc);
3416
3417 /*
3418 * If there are interrupts pending but masked by the TPR, instruct VT-x to
3419 * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
3420 * priority of the pending interrupt so we can deliver the interrupt. If there
3421 * are no interrupts pending, set threshold to 0 to not cause any
3422 * TPR-below-threshold VM-exits.
3423 */
3424 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3425 uint32_t u32TprThreshold = 0;
3426 if (fPendingIntr)
3427 {
3428 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3429 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3430 const uint8_t u8TprPriority = u8Tpr >> 4;
3431 if (u8PendingPriority <= u8TprPriority)
3432 u32TprThreshold = u8PendingPriority;
3433 }
3434
3435 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3436 AssertRCReturn(rc, rc);
3437 }
3438 }
3439 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
3440 }
3441 return VINF_SUCCESS;
3442}
3443
3444
3445/**
3446 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3447 *
3448 * @returns Guest's interruptibility-state.
3449 * @param pVCpu The cross context virtual CPU structure.
3450 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3451 * out-of-sync. Make sure to update the required fields
3452 * before using them.
3453 *
3454 * @remarks No-long-jump zone!!!
3455 */
3456static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3457{
3458 /*
3459 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3460 */
3461 uint32_t fIntrState = 0;
3462 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3463 {
3464 /* If inhibition is active, RIP & RFLAGS should've been accessed
3465 (i.e. read previously from the VMCS or from ring-3). */
3466#ifdef VBOX_STRICT
3467 uint64_t const fExtrn = ASMAtomicUoReadU64(&pMixedCtx->fExtrn);
3468 AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
3469#endif
3470 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3471 {
3472 if (pMixedCtx->eflags.Bits.u1IF)
3473 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3474 else
3475 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3476 }
3477 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3478 {
3479 /*
3480 * We can clear the inhibit force flag as even if we go back to the recompiler
3481 * without executing guest code in VT-x, the flag's condition to be cleared is
3482 * met and thus the cleared state is correct.
3483 */
3484 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3485 }
3486 }
3487
3488 /*
3489 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3490 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3491 * setting this would block host-NMIs and IRET will not clear the blocking.
3492 *
3493 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3494 */
3495 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3496 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3497 {
3498 fIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3499 }
3500
3501 return fIntrState;
3502}
3503
3504
3505/**
3506 * Exports the guest's interruptibility-state into the guest-state area in the
3507 * VMCS.
3508 *
3509 * @returns VBox status code.
3510 * @param pVCpu The cross context virtual CPU structure.
3511 * @param fIntrState The interruptibility-state to set.
3512 */
3513static int hmR0VmxExportGuestIntrState(PVMCPU pVCpu, uint32_t fIntrState)
3514{
3515 NOREF(pVCpu);
3516 AssertMsg(!(fIntrState & 0xfffffff0), ("%#x\n", fIntrState)); /* Bits 31:4 MBZ. */
3517 Assert((fIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3518 return VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, fIntrState);
3519}
3520
3521
3522/**
3523 * Exports the exception intercepts required for guest execution in the VMCS.
3524 *
3525 * @returns VBox status code.
3526 * @param pVCpu The cross context virtual CPU structure.
3527 *
3528 * @remarks No-long-jump zone!!!
3529 */
3530static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu)
3531{
3532 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
3533 {
3534 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxExportSharedCR0(). */
3535 if (pVCpu->hm.s.fGIMTrapXcptUD)
3536 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3537#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3538 else
3539 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3540#endif
3541
3542 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3543 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3544
3545 /** @todo Optimize by checking cache before writing to VMCS. */
3546 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3547 AssertRCReturn(rc, rc);
3548
3549 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3550 Log4Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64\n", pVCpu->hm.s.vmx.u32XcptBitmap));
3551 }
3552 return VINF_SUCCESS;
3553}
3554
3555
3556/**
3557 * Exports the guest's RIP into the guest-state area in the VMCS.
3558 *
3559 * @returns VBox status code.
3560 * @param pVCpu The cross context virtual CPU structure.
3561 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3562 * out-of-sync. Make sure to update the required fields
3563 * before using them.
3564 *
3565 * @remarks No-long-jump zone!!!
3566 */
3567static int hmR0VmxExportGuestRip(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3568{
3569 int rc = VINF_SUCCESS;
3570 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
3571 {
3572 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
3573
3574 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3575 AssertRCReturn(rc, rc);
3576
3577 /* Update the exit history entry with the correct CS.BASE + RIP or just RIP. */
3578 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
3579 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->cs.u64Base + pMixedCtx->rip, true);
3580 else
3581 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->rip, false);
3582
3583 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3584 Log4Func(("RIP=%#RX64\n", pMixedCtx->rip));
3585 }
3586 return rc;
3587}
3588
3589
3590/**
3591 * Exports the guest's RSP into the guest-state area in the VMCS.
3592 *
3593 * @returns VBox status code.
3594 * @param pVCpu The cross context virtual CPU structure.
3595 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3596 * out-of-sync. Make sure to update the required fields
3597 * before using them.
3598 *
3599 * @remarks No-long-jump zone!!!
3600 */
3601static int hmR0VmxExportGuestRsp(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3602{
3603 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3604 {
3605 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
3606
3607 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3608 AssertRCReturn(rc, rc);
3609
3610 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3611 }
3612 return VINF_SUCCESS;
3613}
3614
3615
3616/**
3617 * Exports the guest's RFLAGS into the guest-state area in the VMCS.
3618 *
3619 * @returns VBox status code.
3620 * @param pVCpu The cross context virtual CPU structure.
3621 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3622 * out-of-sync. Make sure to update the required fields
3623 * before using them.
3624 *
3625 * @remarks No-long-jump zone!!!
3626 */
3627static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3628{
3629 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3630 {
3631 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
3632
3633 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3634 Let us assert it as such and use 32-bit VMWRITE. */
3635 Assert(!RT_HI_U32(pMixedCtx->rflags.u64));
3636 X86EFLAGS fEFlags = pMixedCtx->eflags;
3637 Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3638 Assert(!(fEFlags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3639
3640 /*
3641 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3642 * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3643 * can run the real-mode guest code under Virtual 8086 mode.
3644 */
3645 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3646 {
3647 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3648 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3649 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3650 fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3651 fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3652 }
3653
3654 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3655 AssertRCReturn(rc, rc);
3656
3657 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3658 Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
3659 }
3660 return VINF_SUCCESS;
3661}
3662
3663
3664/**
3665 * Exports the guest CR0 control register into the guest-state area in the VMCS.
3666 *
3667 * The guest FPU state is always pre-loaded hence we don't need to bother about
3668 * sharing FPU related CR0 bits between the guest and host.
3669 *
3670 * @returns VBox status code.
3671 * @param pVCpu The cross context virtual CPU structure.
3672 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3673 * out-of-sync. Make sure to update the required fields
3674 * before using them.
3675 *
3676 * @remarks No-long-jump zone!!!
3677 */
3678static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3679{
3680 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
3681 {
3682 PVM pVM = pVCpu->CTX_SUFF(pVM);
3683 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
3684 Assert(!RT_HI_U32(pMixedCtx->cr0));
3685
3686 uint32_t const u32ShadowCr0 = pMixedCtx->cr0;
3687 uint32_t u32GuestCr0 = pMixedCtx->cr0;
3688
3689 /*
3690 * Setup VT-x's view of the guest CR0.
3691 * Minimize VM-exits due to CR3 changes when we have NestedPaging.
3692 */
3693 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
3694 if (pVM->hm.s.fNestedPaging)
3695 {
3696 if (CPUMIsGuestPagingEnabled(pVCpu))
3697 {
3698 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3699 uProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3700 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3701 }
3702 else
3703 {
3704 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3705 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3706 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3707 }
3708
3709 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3710 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3711 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3712 }
3713 else
3714 {
3715 /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3716 u32GuestCr0 |= X86_CR0_WP;
3717 }
3718
3719 /*
3720 * Guest FPU bits.
3721 *
3722 * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3723 * using CR0.TS.
3724 *
3725 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3726 * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3727 */
3728 u32GuestCr0 |= X86_CR0_NE;
3729
3730 /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3731 bool const fInterceptMF = !(u32ShadowCr0 & X86_CR0_NE);
3732
3733 /*
3734 * Update exception intercepts.
3735 */
3736 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3737 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3738 {
3739 Assert(PDMVmmDevHeapIsEnabled(pVM));
3740 Assert(pVM->hm.s.vmx.pRealModeTSS);
3741 uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3742 }
3743 else
3744 {
3745 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3746 uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3747 if (fInterceptMF)
3748 uXcptBitmap |= RT_BIT(X86_XCPT_MF);
3749 }
3750
3751 /* Additional intercepts for debugging, define these yourself explicitly. */
3752#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3753 uXcptBitmap |= 0
3754 | RT_BIT(X86_XCPT_BP)
3755 | RT_BIT(X86_XCPT_DE)
3756 | RT_BIT(X86_XCPT_NM)
3757 | RT_BIT(X86_XCPT_TS)
3758 | RT_BIT(X86_XCPT_UD)
3759 | RT_BIT(X86_XCPT_NP)
3760 | RT_BIT(X86_XCPT_SS)
3761 | RT_BIT(X86_XCPT_GP)
3762 | RT_BIT(X86_XCPT_PF)
3763 | RT_BIT(X86_XCPT_MF)
3764 ;
3765#elif defined(HMVMX_ALWAYS_TRAP_PF)
3766 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
3767#endif
3768 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3769 {
3770 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3771 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3772 }
3773 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3774
3775 /*
3776 * Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW).
3777 */
3778 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3779 uint32_t fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3780 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3781 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
3782 else
3783 Assert((fSetCr0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3784
3785 u32GuestCr0 |= fSetCr0;
3786 u32GuestCr0 &= fZapCr0;
3787 u32GuestCr0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3788
3789 /*
3790 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3791 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3792 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3793 */
3794 uint32_t u32Cr0Mask = X86_CR0_PE
3795 | X86_CR0_NE
3796 | (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
3797 | X86_CR0_PG
3798 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3799 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3800 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3801
3802 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3803 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3804 * and @bugref{6944}. */
3805#if 0
3806 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3807 u32Cr0Mask &= ~X86_CR0_PE;
3808#endif
3809 /*
3810 * Finally, update VMCS fields with the CR0 values.
3811 */
3812 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCr0);
3813 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32ShadowCr0);
3814 if (u32Cr0Mask != pVCpu->hm.s.vmx.u32Cr0Mask)
3815 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32Cr0Mask);
3816 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
3817 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3818 AssertRCReturn(rc, rc);
3819
3820 /* Update our caches. */
3821 pVCpu->hm.s.vmx.u32Cr0Mask = u32Cr0Mask;
3822 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
3823
3824 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3825
3826 Log4Func(("u32Cr0Mask=%#RX32 u32ShadowCr0=%#RX32 u32GuestCr0=%#RX32 (fSetCr0=%#RX32 fZapCr0=%#RX32\n", u32Cr0Mask,
3827 u32ShadowCr0, u32GuestCr0, fSetCr0, fZapCr0));
3828 }
3829
3830 return VINF_SUCCESS;
3831}
3832
3833
3834/**
3835 * Exports the guest control registers (CR3, CR4) into the guest-state area
3836 * in the VMCS.
3837 *
3838 * @returns VBox strict status code.
3839 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3840 * without unrestricted guest access and the VMMDev is not presently
3841 * mapped (e.g. EFI32).
3842 *
3843 * @param pVCpu The cross context virtual CPU structure.
3844 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3845 * out-of-sync. Make sure to update the required fields
3846 * before using them.
3847 *
3848 * @remarks No-long-jump zone!!!
3849 */
3850static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3851{
3852 int rc = VINF_SUCCESS;
3853 PVM pVM = pVCpu->CTX_SUFF(pVM);
3854
3855 /*
3856 * Guest CR2.
3857 * It's always loaded in the assembler code. Nothing to do here.
3858 */
3859
3860 /*
3861 * Guest CR3.
3862 */
3863 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
3864 {
3865 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
3866
3867 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3868 if (pVM->hm.s.fNestedPaging)
3869 {
3870 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3871
3872 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3873 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3874 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3875 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3876
3877 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3878 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3879 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3880
3881 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3882 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3883 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3884 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3885 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3886 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3887 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3888
3889 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3890 AssertRCReturn(rc, rc);
3891
3892 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3893 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3894 {
3895 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3896 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3897 {
3898 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3899 AssertRCReturn(rc, rc);
3900 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3901 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3902 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3903 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3904 AssertRCReturn(rc, rc);
3905 }
3906
3907 /*
3908 * The guest's view of its CR3 is unblemished with Nested Paging when the
3909 * guest is using paging or we have unrestricted guest execution to handle
3910 * the guest when it's not using paging.
3911 */
3912 GCPhysGuestCR3 = pMixedCtx->cr3;
3913 }
3914 else
3915 {
3916 /*
3917 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
3918 * thinks it accesses physical memory directly, we use our identity-mapped
3919 * page table to map guest-linear to guest-physical addresses. EPT takes care
3920 * of translating it to host-physical addresses.
3921 */
3922 RTGCPHYS GCPhys;
3923 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3924
3925 /* We obtain it here every time as the guest could have relocated this PCI region. */
3926 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3927 if (RT_SUCCESS(rc))
3928 { /* likely */ }
3929 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3930 {
3931 Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3932 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3933 }
3934 else
3935 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3936
3937 GCPhysGuestCR3 = GCPhys;
3938 }
3939
3940 Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
3941 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3942 AssertRCReturn(rc, rc);
3943 }
3944 else
3945 {
3946 /* Non-nested paging case, just use the hypervisor's CR3. */
3947 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3948
3949 Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
3950 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3951 AssertRCReturn(rc, rc);
3952 }
3953
3954 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3955 }
3956
3957 /*
3958 * Guest CR4.
3959 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3960 */
3961 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
3962 {
3963 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
3964 Assert(!RT_HI_U32(pMixedCtx->cr4));
3965
3966 uint32_t u32GuestCr4 = pMixedCtx->cr4;
3967 uint32_t const u32ShadowCr4 = pMixedCtx->cr4;
3968
3969 /*
3970 * Setup VT-x's view of the guest CR4.
3971 *
3972 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3973 * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3974 * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3975 *
3976 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3977 */
3978 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3979 {
3980 Assert(pVM->hm.s.vmx.pRealModeTSS);
3981 Assert(PDMVmmDevHeapIsEnabled(pVM));
3982 u32GuestCr4 &= ~X86_CR4_VME;
3983 }
3984
3985 if (pVM->hm.s.fNestedPaging)
3986 {
3987 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3988 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3989 {
3990 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3991 u32GuestCr4 |= X86_CR4_PSE;
3992 /* Our identity mapping is a 32-bit page directory. */
3993 u32GuestCr4 &= ~X86_CR4_PAE;
3994 }
3995 /* else use guest CR4.*/
3996 }
3997 else
3998 {
3999 /*
4000 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4001 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4002 */
4003 switch (pVCpu->hm.s.enmShadowMode)
4004 {
4005 case PGMMODE_REAL: /* Real-mode. */
4006 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4007 case PGMMODE_32_BIT: /* 32-bit paging. */
4008 {
4009 u32GuestCr4 &= ~X86_CR4_PAE;
4010 break;
4011 }
4012
4013 case PGMMODE_PAE: /* PAE paging. */
4014 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4015 {
4016 u32GuestCr4 |= X86_CR4_PAE;
4017 break;
4018 }
4019
4020 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4021 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4022#ifdef VBOX_ENABLE_64_BITS_GUESTS
4023 break;
4024#endif
4025 default:
4026 AssertFailed();
4027 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4028 }
4029 }
4030
4031 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4032 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4033 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4034 u32GuestCr4 |= fSetCr4;
4035 u32GuestCr4 &= fZapCr4;
4036
4037 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them,
4038 that would cause a VM-exit. */
4039 uint32_t u32Cr4Mask = X86_CR4_VME
4040 | X86_CR4_PAE
4041 | X86_CR4_PGE
4042 | X86_CR4_PSE
4043 | X86_CR4_VMXE;
4044 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4045 u32Cr4Mask |= X86_CR4_OSXSAVE;
4046 if (pVM->cpum.ro.GuestFeatures.fPcid)
4047 u32Cr4Mask |= X86_CR4_PCIDE;
4048
4049 /* Write VT-x's view of the guest CR4, the CR4 modify mask and the read-only CR4 shadow
4050 into the VMCS and update our cache. */
4051 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCr4);
4052 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32ShadowCr4);
4053 if (pVCpu->hm.s.vmx.u32Cr4Mask != u32Cr4Mask)
4054 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32Cr4Mask);
4055 AssertRCReturn(rc, rc);
4056 pVCpu->hm.s.vmx.u32Cr4Mask = u32Cr4Mask;
4057
4058 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4059 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4060
4061 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
4062
4063 Log4Func(("u32GuestCr4=%#RX32 u32ShadowCr4=%#RX32 (fSetCr4=%#RX32 fZapCr4=%#RX32)\n", u32GuestCr4, u32ShadowCr4, fSetCr4,
4064 fZapCr4));
4065 }
4066 return rc;
4067}
4068
4069
4070/**
4071 * Exports the guest debug registers into the guest-state area in the VMCS.
4072 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4073 *
4074 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4075 *
4076 * @returns VBox status code.
4077 * @param pVCpu The cross context virtual CPU structure.
4078 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4079 * out-of-sync. Make sure to update the required fields
4080 * before using them.
4081 *
4082 * @remarks No-long-jump zone!!!
4083 */
4084static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4085{
4086 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4087
4088#ifdef VBOX_STRICT
4089 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4090 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4091 {
4092 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4093 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4094 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4095 }
4096#endif
4097
4098 bool fSteppingDB = false;
4099 bool fInterceptMovDRx = false;
4100 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
4101 if (pVCpu->hm.s.fSingleInstruction)
4102 {
4103 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4104 PVM pVM = pVCpu->CTX_SUFF(pVM);
4105 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4106 {
4107 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4108 Assert(fSteppingDB == false);
4109 }
4110 else
4111 {
4112 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4113 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_RFLAGS;
4114 pVCpu->hm.s.fClearTrapFlag = true;
4115 fSteppingDB = true;
4116 }
4117 }
4118
4119 uint32_t u32GuestDr7;
4120 if ( fSteppingDB
4121 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4122 {
4123 /*
4124 * Use the combined guest and host DRx values found in the hypervisor register set
4125 * because the debugger has breakpoints active or someone is single stepping on the
4126 * host side without a monitor trap flag.
4127 *
4128 * Note! DBGF expects a clean DR6 state before executing guest code.
4129 */
4130#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4131 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4132 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4133 {
4134 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4135 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4136 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4137 }
4138 else
4139#endif
4140 if (!CPUMIsHyperDebugStateActive(pVCpu))
4141 {
4142 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4143 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4144 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4145 }
4146
4147 /* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
4148 u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
4149 pVCpu->hm.s.fUsingHyperDR7 = true;
4150 fInterceptMovDRx = true;
4151 }
4152 else
4153 {
4154 /*
4155 * If the guest has enabled debug registers, we need to load them prior to
4156 * executing guest code so they'll trigger at the right time.
4157 */
4158 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD))
4159 {
4160#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4161 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4162 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4163 {
4164 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4165 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4166 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4167 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4168 }
4169 else
4170#endif
4171 if (!CPUMIsGuestDebugStateActive(pVCpu))
4172 {
4173 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4174 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4175 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4176 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4177 }
4178 Assert(!fInterceptMovDRx);
4179 }
4180 /*
4181 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4182 * must intercept #DB in order to maintain a correct DR6 guest value, and
4183 * because we need to intercept it to prevent nested #DBs from hanging the
4184 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4185 */
4186#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4187 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4188 && !CPUMIsGuestDebugStateActive(pVCpu))
4189#else
4190 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4191#endif
4192 {
4193 fInterceptMovDRx = true;
4194 }
4195
4196 /* Update DR7 with the actual guest value. */
4197 u32GuestDr7 = pMixedCtx->dr[7];
4198 pVCpu->hm.s.fUsingHyperDR7 = false;
4199 }
4200
4201 if (fInterceptMovDRx)
4202 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4203 else
4204 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4205
4206 /*
4207 * Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
4208 * monitor-trap flag and update our cache.
4209 */
4210 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
4211 {
4212 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4213 AssertRCReturn(rc2, rc2);
4214 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
4215 }
4216
4217 /*
4218 * Update guest DR7.
4219 */
4220 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
4221 AssertRCReturn(rc, rc);
4222
4223 return VINF_SUCCESS;
4224}
4225
4226
4227#ifdef VBOX_STRICT
4228/**
4229 * Strict function to validate segment registers.
4230 *
4231 * @param pVCpu The cross context virtual CPU structure.
4232 * @param pCtx Pointer to the guest-CPU context.
4233 *
4234 * @remarks Will import guest CR0 on strict builds during validation of
4235 * segments.
4236 */
4237static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PCCPUMCTX pCtx)
4238{
4239 /*
4240 * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
4241 *
4242 * The reason we check for attribute value 0 in this function and not just the unusable bit is
4243 * because hmR0VmxExportGuestSegmentReg() only updates the VMCS' copy of the value with the unusable bit
4244 * and doesn't change the guest-context value.
4245 */
4246 PVM pVM = pVCpu->CTX_SUFF(pVM);
4247 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
4248 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4249 && ( !CPUMIsGuestInRealModeEx(pCtx)
4250 && !CPUMIsGuestInV86ModeEx(pCtx)))
4251 {
4252 /* Protected mode checks */
4253 /* CS */
4254 Assert(pCtx->cs.Attr.n.u1Present);
4255 Assert(!(pCtx->cs.Attr.u & 0xf00));
4256 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4257 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4258 || !(pCtx->cs.Attr.n.u1Granularity));
4259 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4260 || (pCtx->cs.Attr.n.u1Granularity));
4261 /* CS cannot be loaded with NULL in protected mode. */
4262 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4263 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4264 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4265 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4266 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4267 else
4268 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4269 /* SS */
4270 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4271 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4272 if ( !(pCtx->cr0 & X86_CR0_PE)
4273 || pCtx->cs.Attr.n.u4Type == 3)
4274 {
4275 Assert(!pCtx->ss.Attr.n.u2Dpl);
4276 }
4277 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4278 {
4279 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4280 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4281 Assert(pCtx->ss.Attr.n.u1Present);
4282 Assert(!(pCtx->ss.Attr.u & 0xf00));
4283 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4284 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4285 || !(pCtx->ss.Attr.n.u1Granularity));
4286 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4287 || (pCtx->ss.Attr.n.u1Granularity));
4288 }
4289 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmentReg(). */
4290 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4291 {
4292 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4293 Assert(pCtx->ds.Attr.n.u1Present);
4294 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4295 Assert(!(pCtx->ds.Attr.u & 0xf00));
4296 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4297 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4298 || !(pCtx->ds.Attr.n.u1Granularity));
4299 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4300 || (pCtx->ds.Attr.n.u1Granularity));
4301 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4302 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4303 }
4304 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4305 {
4306 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4307 Assert(pCtx->es.Attr.n.u1Present);
4308 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4309 Assert(!(pCtx->es.Attr.u & 0xf00));
4310 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4311 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4312 || !(pCtx->es.Attr.n.u1Granularity));
4313 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4314 || (pCtx->es.Attr.n.u1Granularity));
4315 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4316 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4317 }
4318 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4319 {
4320 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4321 Assert(pCtx->fs.Attr.n.u1Present);
4322 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4323 Assert(!(pCtx->fs.Attr.u & 0xf00));
4324 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4325 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4326 || !(pCtx->fs.Attr.n.u1Granularity));
4327 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4328 || (pCtx->fs.Attr.n.u1Granularity));
4329 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4330 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4331 }
4332 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4333 {
4334 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4335 Assert(pCtx->gs.Attr.n.u1Present);
4336 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4337 Assert(!(pCtx->gs.Attr.u & 0xf00));
4338 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4339 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4340 || !(pCtx->gs.Attr.n.u1Granularity));
4341 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4342 || (pCtx->gs.Attr.n.u1Granularity));
4343 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4344 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4345 }
4346 /* 64-bit capable CPUs. */
4347# if HC_ARCH_BITS == 64
4348 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4349 Assert(!pCtx->ss.Attr.u || !RT_HI_U32(pCtx->ss.u64Base));
4350 Assert(!pCtx->ds.Attr.u || !RT_HI_U32(pCtx->ds.u64Base));
4351 Assert(!pCtx->es.Attr.u || !RT_HI_U32(pCtx->es.u64Base));
4352# endif
4353 }
4354 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4355 || ( CPUMIsGuestInRealModeEx(pCtx)
4356 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4357 {
4358 /* Real and v86 mode checks. */
4359 /* hmR0VmxExportGuestSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4360 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4361 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4362 {
4363 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4364 }
4365 else
4366 {
4367 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4368 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4369 }
4370
4371 /* CS */
4372 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4373 Assert(pCtx->cs.u32Limit == 0xffff);
4374 Assert(u32CSAttr == 0xf3);
4375 /* SS */
4376 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4377 Assert(pCtx->ss.u32Limit == 0xffff);
4378 Assert(u32SSAttr == 0xf3);
4379 /* DS */
4380 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4381 Assert(pCtx->ds.u32Limit == 0xffff);
4382 Assert(u32DSAttr == 0xf3);
4383 /* ES */
4384 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4385 Assert(pCtx->es.u32Limit == 0xffff);
4386 Assert(u32ESAttr == 0xf3);
4387 /* FS */
4388 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4389 Assert(pCtx->fs.u32Limit == 0xffff);
4390 Assert(u32FSAttr == 0xf3);
4391 /* GS */
4392 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4393 Assert(pCtx->gs.u32Limit == 0xffff);
4394 Assert(u32GSAttr == 0xf3);
4395 /* 64-bit capable CPUs. */
4396# if HC_ARCH_BITS == 64
4397 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4398 Assert(!u32SSAttr || !RT_HI_U32(pCtx->ss.u64Base));
4399 Assert(!u32DSAttr || !RT_HI_U32(pCtx->ds.u64Base));
4400 Assert(!u32ESAttr || !RT_HI_U32(pCtx->es.u64Base));
4401# endif
4402 }
4403}
4404#endif /* VBOX_STRICT */
4405
4406
4407/**
4408 * Exports a guest segment register into the guest-state area in the VMCS.
4409 *
4410 * @returns VBox status code.
4411 * @param pVCpu The cross context virtual CPU structure.
4412 * @param idxSel Index of the selector in the VMCS.
4413 * @param idxLimit Index of the segment limit in the VMCS.
4414 * @param idxBase Index of the segment base in the VMCS.
4415 * @param idxAccess Index of the access rights of the segment in the VMCS.
4416 * @param pSelReg Pointer to the segment selector.
4417 *
4418 * @remarks No-long-jump zone!!!
4419 */
4420static int hmR0VmxExportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
4421 PCCPUMSELREG pSelReg)
4422{
4423 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4424 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4425 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4426 AssertRCReturn(rc, rc);
4427
4428 uint32_t u32Access = pSelReg->Attr.u;
4429 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4430 {
4431 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4432 u32Access = 0xf3;
4433 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4434 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4435 }
4436 else
4437 {
4438 /*
4439 * The way to differentiate between whether this is really a null selector or was just
4440 * a selector loaded with 0 in real-mode is using the segment attributes. A selector
4441 * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4442 * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4443 * NULL selectors loaded in protected-mode have their attribute as 0.
4444 */
4445 if (!u32Access)
4446 u32Access = X86DESCATTR_UNUSABLE;
4447 }
4448
4449 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4450 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4451 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4452
4453 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4454 AssertRCReturn(rc, rc);
4455 return rc;
4456}
4457
4458
4459/**
4460 * Exports the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4461 * into the guest-state area in the VMCS.
4462 *
4463 * @returns VBox status code.
4464 * @param pVCpu The cross context virtual CPU structure.
4465 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4466 * out-of-sync. Make sure to update the required fields
4467 * before using them.
4468 *
4469 * @remarks Will import guest CR0 on strict builds during validation of
4470 * segments.
4471 * @remarks No-long-jump zone!!!
4472 */
4473static int hmR0VmxExportGuestSegmentRegs(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4474{
4475 int rc = VERR_INTERNAL_ERROR_5;
4476 PVM pVM = pVCpu->CTX_SUFF(pVM);
4477
4478 /*
4479 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4480 */
4481 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4482 {
4483#ifdef VBOX_WITH_REM
4484 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4485 {
4486 Assert(pVM->hm.s.vmx.pRealModeTSS);
4487 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4488 if ( pVCpu->hm.s.vmx.fWasInRealMode
4489 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4490 {
4491 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4492 in real-mode (e.g. OpenBSD 4.0) */
4493 REMFlushTBs(pVM);
4494 Log4Func(("Switch to protected mode detected!\n"));
4495 pVCpu->hm.s.vmx.fWasInRealMode = false;
4496 }
4497 }
4498#endif
4499 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
4500 {
4501 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
4502 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4503 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4504 rc = HMVMX_EXPORT_SREG(CS, &pMixedCtx->cs);
4505 AssertRCReturn(rc, rc);
4506 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
4507 }
4508
4509 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
4510 {
4511 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
4512 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4513 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4514 rc = HMVMX_EXPORT_SREG(SS, &pMixedCtx->ss);
4515 AssertRCReturn(rc, rc);
4516 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
4517 }
4518
4519 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
4520 {
4521 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
4522 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4523 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4524 rc = HMVMX_EXPORT_SREG(DS, &pMixedCtx->ds);
4525 AssertRCReturn(rc, rc);
4526 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
4527 }
4528
4529 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
4530 {
4531 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
4532 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4533 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4534 rc = HMVMX_EXPORT_SREG(ES, &pMixedCtx->es);
4535 AssertRCReturn(rc, rc);
4536 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
4537 }
4538
4539 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
4540 {
4541 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
4542 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4543 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4544 rc = HMVMX_EXPORT_SREG(FS, &pMixedCtx->fs);
4545 AssertRCReturn(rc, rc);
4546 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
4547 }
4548
4549 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
4550 {
4551 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
4552 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4553 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4554 rc = HMVMX_EXPORT_SREG(GS, &pMixedCtx->gs);
4555 AssertRCReturn(rc, rc);
4556 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
4557 }
4558
4559#ifdef VBOX_STRICT
4560 hmR0VmxValidateSegmentRegs(pVCpu, pMixedCtx);
4561#endif
4562
4563 /* Update the exit history entry with the correct CS.BASE + RIP. */
4564 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4565 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->cs.u64Base + pMixedCtx->rip, true);
4566
4567 Log4Func(("CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4568 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4569 }
4570
4571 /*
4572 * Guest TR.
4573 */
4574 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
4575 {
4576 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
4577
4578 /*
4579 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
4580 * achieved using the interrupt redirection bitmap (all bits cleared to let the guest
4581 * handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4582 */
4583 uint16_t u16Sel = 0;
4584 uint32_t u32Limit = 0;
4585 uint64_t u64Base = 0;
4586 uint32_t u32AccessRights = 0;
4587
4588 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4589 {
4590 u16Sel = pMixedCtx->tr.Sel;
4591 u32Limit = pMixedCtx->tr.u32Limit;
4592 u64Base = pMixedCtx->tr.u64Base;
4593 u32AccessRights = pMixedCtx->tr.Attr.u;
4594 }
4595 else
4596 {
4597 Assert(pVM->hm.s.vmx.pRealModeTSS);
4598 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4599
4600 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4601 RTGCPHYS GCPhys;
4602 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4603 AssertRCReturn(rc, rc);
4604
4605 X86DESCATTR DescAttr;
4606 DescAttr.u = 0;
4607 DescAttr.n.u1Present = 1;
4608 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4609
4610 u16Sel = 0;
4611 u32Limit = HM_VTX_TSS_SIZE;
4612 u64Base = GCPhys; /* in real-mode phys = virt. */
4613 u32AccessRights = DescAttr.u;
4614 }
4615
4616 /* Validate. */
4617 Assert(!(u16Sel & RT_BIT(2)));
4618 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4619 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4620 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4621 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4622 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4623 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4624 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4625 Assert( (u32Limit & 0xfff) == 0xfff
4626 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4627 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4628 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4629
4630 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4631 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4632 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4633 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4634 AssertRCReturn(rc, rc);
4635
4636 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
4637 Log4Func(("TR base=%#RX64\n", pMixedCtx->tr.u64Base));
4638 }
4639
4640 /*
4641 * Guest GDTR.
4642 */
4643 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4644 {
4645 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
4646
4647 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4648 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4649 AssertRCReturn(rc, rc);
4650
4651 /* Validate. */
4652 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4653
4654 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4655 Log4Func(("GDTR base=%#RX64\n", pMixedCtx->gdtr.pGdt));
4656 }
4657
4658 /*
4659 * Guest LDTR.
4660 */
4661 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4662 {
4663 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
4664
4665 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4666 uint32_t u32Access = 0;
4667 if (!pMixedCtx->ldtr.Attr.u)
4668 u32Access = X86DESCATTR_UNUSABLE;
4669 else
4670 u32Access = pMixedCtx->ldtr.Attr.u;
4671
4672 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4673 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4674 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4675 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4676 AssertRCReturn(rc, rc);
4677
4678 /* Validate. */
4679 if (!(u32Access & X86DESCATTR_UNUSABLE))
4680 {
4681 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4682 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4683 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4684 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4685 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4686 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4687 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4688 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4689 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4690 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4691 }
4692
4693 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4694 Log4Func(("LDTR base=%#RX64\n", pMixedCtx->ldtr.u64Base));
4695 }
4696
4697 /*
4698 * Guest IDTR.
4699 */
4700 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4701 {
4702 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
4703
4704 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4705 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4706 AssertRCReturn(rc, rc);
4707
4708 /* Validate. */
4709 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4710
4711 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4712 Log4Func(("IDTR base=%#RX64\n", pMixedCtx->idtr.pIdt));
4713 }
4714
4715 return VINF_SUCCESS;
4716}
4717
4718
4719/**
4720 * Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4721 * areas.
4722 *
4723 * These MSRs will automatically be loaded to the host CPU on every successful
4724 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4725 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4726 * -not- updated here for performance reasons. See hmR0VmxExportHostMsrs().
4727 *
4728 * Also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4729 *
4730 * @returns VBox status code.
4731 * @param pVCpu The cross context virtual CPU structure.
4732 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4733 * out-of-sync. Make sure to update the required fields
4734 * before using them.
4735 *
4736 * @remarks No-long-jump zone!!!
4737 */
4738static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4739{
4740 AssertPtr(pVCpu);
4741 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4742
4743 /*
4744 * MSRs that we use the auto-load/store MSR area in the VMCS.
4745 * For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
4746 */
4747 PVM pVM = pVCpu->CTX_SUFF(pVM);
4748 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4749 {
4750 if (pVM->hm.s.fAllow64BitGuests)
4751 {
4752#if HC_ARCH_BITS == 32
4753 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS | CPUMCTX_EXTRN_KERNEL_GS_BASE);
4754
4755 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4756 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4757 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4758 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4759 AssertRCReturn(rc, rc);
4760# ifdef LOG_ENABLED
4761 PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4762 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4763 Log4Func(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4764# endif
4765#endif
4766 }
4767 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4768 }
4769
4770 /*
4771 * Guest Sysenter MSRs.
4772 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4773 * VM-exits on WRMSRs for these MSRs.
4774 */
4775 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4776 {
4777 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
4778
4779 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4780 {
4781 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs);
4782 AssertRCReturn(rc, rc);
4783 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4784 }
4785
4786 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4787 {
4788 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip);
4789 AssertRCReturn(rc, rc);
4790 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4791 }
4792
4793 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4794 {
4795 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp);
4796 AssertRCReturn(rc, rc);
4797 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4798 }
4799 }
4800
4801 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4802 {
4803 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
4804
4805 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4806 {
4807 /*
4808 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4809 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4810 */
4811 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4812 {
4813 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4814 AssertRCReturn(rc,rc);
4815 Log4Func(("EFER=%#RX64\n", pMixedCtx->msrEFER));
4816 }
4817 else
4818 {
4819 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4820 NULL /* pfAddedAndUpdated */);
4821 AssertRCReturn(rc, rc);
4822
4823 /* We need to intercept reads too, see @bugref{7386#c16}. */
4824 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4825 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4826 Log4Func(("MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pMixedCtx->msrEFER,
4827 pVCpu->hm.s.vmx.cMsrs));
4828 }
4829 }
4830 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4831 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4832 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4833 }
4834
4835 return VINF_SUCCESS;
4836}
4837
4838
4839#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4840/**
4841 * Check if guest state allows safe use of 32-bit switcher again.
4842 *
4843 * Segment bases and protected mode structures must be 32-bit addressable
4844 * because the 32-bit switcher will ignore high dword when writing these VMCS
4845 * fields. See @bugref{8432} for details.
4846 *
4847 * @returns true if safe, false if must continue to use the 64-bit switcher.
4848 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4849 * out-of-sync. Make sure to update the required fields
4850 * before using them.
4851 *
4852 * @remarks No-long-jump zone!!!
4853 */
4854static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pMixedCtx)
4855{
4856 if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4857 if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4858 if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4859 if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4860 if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4861 if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4862 if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4863 if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4864 if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4865 if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4866
4867 /* All good, bases are 32-bit. */
4868 return true;
4869}
4870#endif
4871
4872
4873/**
4874 * Selects up the appropriate function to run guest code.
4875 *
4876 * @returns VBox status code.
4877 * @param pVCpu The cross context virtual CPU structure.
4878 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4879 * out-of-sync. Make sure to update the required fields
4880 * before using them.
4881 *
4882 * @remarks No-long-jump zone!!!
4883 */
4884static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4885{
4886 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4887 {
4888#ifndef VBOX_ENABLE_64_BITS_GUESTS
4889 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4890#endif
4891 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4892#if HC_ARCH_BITS == 32
4893 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4894 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4895 {
4896#ifdef VBOX_STRICT
4897 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4898 {
4899 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4900 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4901 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4902 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4903 | HM_CHANGED_VMX_ENTRY_CTLS
4904 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4905 }
4906#endif
4907 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4908
4909 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4910 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4911 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4912 Log4Func(("Selected 64-bit switcher\n"));
4913 }
4914#else
4915 /* 64-bit host. */
4916 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4917#endif
4918 }
4919 else
4920 {
4921 /* Guest is not in long mode, use the 32-bit handler. */
4922#if HC_ARCH_BITS == 32
4923 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4924 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4925 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4926 {
4927# ifdef VBOX_STRICT
4928 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4929 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4930 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4931 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4932 | HM_CHANGED_VMX_ENTRY_CTLS
4933 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4934# endif
4935 }
4936# ifdef VBOX_ENABLE_64_BITS_GUESTS
4937 /*
4938 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
4939 * design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
4940 * switcher flag because now we know the guest is in a sane state where it's safe
4941 * to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4942 * the much faster 32-bit switcher again.
4943 */
4944 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4945 {
4946 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4947 Log4Func(("Selected 32-bit switcher\n"));
4948 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4949 }
4950 else
4951 {
4952 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4953 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4954 || hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
4955 {
4956 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4957 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4958 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
4959 | HM_CHANGED_VMX_ENTRY_CTLS
4960 | HM_CHANGED_VMX_EXIT_CTLS
4961 | HM_CHANGED_HOST_CONTEXT);
4962 Log4Func(("Selected 32-bit switcher (safe)\n"));
4963 }
4964 }
4965# else
4966 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4967# endif
4968#else
4969 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4970#endif
4971 }
4972 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4973 return VINF_SUCCESS;
4974}
4975
4976
4977/**
4978 * Wrapper for running the guest code in VT-x.
4979 *
4980 * @returns VBox status code, no informational status codes.
4981 * @param pVCpu The cross context virtual CPU structure.
4982 * @param pCtx Pointer to the guest-CPU context.
4983 *
4984 * @remarks No-long-jump zone!!!
4985 */
4986DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PCPUMCTX pCtx)
4987{
4988 /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4989 pCtx->fExtrn |= HMVMX_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
4990
4991 /*
4992 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4993 * floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
4994 * callee-saved and thus the need for this XMM wrapper.
4995 *
4996 * See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
4997 */
4998 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4999 /** @todo Add stats for resume vs launch. */
5000 PVM pVM = pVCpu->CTX_SUFF(pVM);
5001#ifdef VBOX_WITH_KERNEL_USING_XMM
5002 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
5003#else
5004 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
5005#endif
5006 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
5007 return rc;
5008}
5009
5010
5011/**
5012 * Reports world-switch error and dumps some useful debug info.
5013 *
5014 * @param pVCpu The cross context virtual CPU structure.
5015 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5016 * @param pCtx Pointer to the guest-CPU context.
5017 * @param pVmxTransient Pointer to the VMX transient structure (only
5018 * exitReason updated).
5019 */
5020static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
5021{
5022 Assert(pVCpu);
5023 Assert(pCtx);
5024 Assert(pVmxTransient);
5025 HMVMX_ASSERT_PREEMPT_SAFE();
5026
5027 Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
5028 switch (rcVMRun)
5029 {
5030 case VERR_VMX_INVALID_VMXON_PTR:
5031 AssertFailed();
5032 break;
5033 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5034 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5035 {
5036 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5037 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5038 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5039 AssertRC(rc);
5040
5041 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5042 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5043 Cannot do it here as we may have been long preempted. */
5044
5045#ifdef VBOX_STRICT
5046 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5047 pVmxTransient->uExitReason));
5048 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5049 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5050 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5051 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5052 else
5053 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5054 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5055 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5056
5057 /* VMX control bits. */
5058 uint32_t u32Val;
5059 uint64_t u64Val;
5060 RTHCUINTREG uHCReg;
5061 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5062 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5063 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5064 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5065 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5066 {
5067 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5068 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5069 }
5070 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5071 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5072 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5073 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5074 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5075 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5076 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5077 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5078 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5079 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5080 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5081 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5082 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5083 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5084 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5085 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5086 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5087 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5088 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5089 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5090 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5091 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5092 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5093 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5094 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5095 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5096 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5097 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5098 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5099 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5100 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5101 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5102 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5103 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5104 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5105 {
5106 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5107 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5108 }
5109
5110 /* Guest bits. */
5111 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5112 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5113 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5114 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5115 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5116 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5117 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
5118 {
5119 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5120 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5121 }
5122
5123 /* Host bits. */
5124 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5125 Log4(("Host CR0 %#RHr\n", uHCReg));
5126 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5127 Log4(("Host CR3 %#RHr\n", uHCReg));
5128 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5129 Log4(("Host CR4 %#RHr\n", uHCReg));
5130
5131 RTGDTR HostGdtr;
5132 PCX86DESCHC pDesc;
5133 ASMGetGDTR(&HostGdtr);
5134 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5135 Log4(("Host CS %#08x\n", u32Val));
5136 if (u32Val < HostGdtr.cbGdt)
5137 {
5138 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5139 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5140 }
5141
5142 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5143 Log4(("Host DS %#08x\n", u32Val));
5144 if (u32Val < HostGdtr.cbGdt)
5145 {
5146 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5147 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5148 }
5149
5150 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5151 Log4(("Host ES %#08x\n", u32Val));
5152 if (u32Val < HostGdtr.cbGdt)
5153 {
5154 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5155 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5156 }
5157
5158 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5159 Log4(("Host FS %#08x\n", u32Val));
5160 if (u32Val < HostGdtr.cbGdt)
5161 {
5162 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5163 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5164 }
5165
5166 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5167 Log4(("Host GS %#08x\n", u32Val));
5168 if (u32Val < HostGdtr.cbGdt)
5169 {
5170 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5171 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5172 }
5173
5174 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5175 Log4(("Host SS %#08x\n", u32Val));
5176 if (u32Val < HostGdtr.cbGdt)
5177 {
5178 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5179 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5180 }
5181
5182 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5183 Log4(("Host TR %#08x\n", u32Val));
5184 if (u32Val < HostGdtr.cbGdt)
5185 {
5186 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5187 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5188 }
5189
5190 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5191 Log4(("Host TR Base %#RHv\n", uHCReg));
5192 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5193 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5194 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5195 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5196 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5197 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5198 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5199 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5200 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5201 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5202 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5203 Log4(("Host RSP %#RHv\n", uHCReg));
5204 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5205 Log4(("Host RIP %#RHv\n", uHCReg));
5206# if HC_ARCH_BITS == 64
5207 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5208 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5209 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5210 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5211 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5212 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5213# endif
5214#endif /* VBOX_STRICT */
5215 break;
5216 }
5217
5218 default:
5219 /* Impossible */
5220 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5221 break;
5222 }
5223 NOREF(pCtx);
5224}
5225
5226
5227#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5228#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5229# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5230#endif
5231#ifdef VBOX_STRICT
5232static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5233{
5234 switch (idxField)
5235 {
5236 case VMX_VMCS_GUEST_RIP:
5237 case VMX_VMCS_GUEST_RSP:
5238 case VMX_VMCS_GUEST_SYSENTER_EIP:
5239 case VMX_VMCS_GUEST_SYSENTER_ESP:
5240 case VMX_VMCS_GUEST_GDTR_BASE:
5241 case VMX_VMCS_GUEST_IDTR_BASE:
5242 case VMX_VMCS_GUEST_CS_BASE:
5243 case VMX_VMCS_GUEST_DS_BASE:
5244 case VMX_VMCS_GUEST_ES_BASE:
5245 case VMX_VMCS_GUEST_FS_BASE:
5246 case VMX_VMCS_GUEST_GS_BASE:
5247 case VMX_VMCS_GUEST_SS_BASE:
5248 case VMX_VMCS_GUEST_LDTR_BASE:
5249 case VMX_VMCS_GUEST_TR_BASE:
5250 case VMX_VMCS_GUEST_CR3:
5251 return true;
5252 }
5253 return false;
5254}
5255
5256static bool hmR0VmxIsValidReadField(uint32_t idxField)
5257{
5258 switch (idxField)
5259 {
5260 /* Read-only fields. */
5261 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5262 return true;
5263 }
5264 /* Remaining readable fields should also be writable. */
5265 return hmR0VmxIsValidWriteField(idxField);
5266}
5267#endif /* VBOX_STRICT */
5268
5269
5270/**
5271 * Executes the specified handler in 64-bit mode.
5272 *
5273 * @returns VBox status code (no informational status codes).
5274 * @param pVCpu The cross context virtual CPU structure.
5275 * @param enmOp The operation to perform.
5276 * @param cParams Number of parameters.
5277 * @param paParam Array of 32-bit parameters.
5278 */
5279VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
5280{
5281 PVM pVM = pVCpu->CTX_SUFF(pVM);
5282 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5283 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5284 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5285 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5286
5287#ifdef VBOX_STRICT
5288 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5289 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5290
5291 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5292 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5293#endif
5294
5295 /* Disable interrupts. */
5296 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5297
5298#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5299 RTCPUID idHostCpu = RTMpCpuId();
5300 CPUMR0SetLApic(pVCpu, idHostCpu);
5301#endif
5302
5303 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5304 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5305
5306 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5307 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5308 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5309
5310 /* Leave VMX Root Mode. */
5311 VMXDisable();
5312
5313 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5314
5315 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5316 CPUMSetHyperEIP(pVCpu, enmOp);
5317 for (int i = (int)cParams - 1; i >= 0; i--)
5318 CPUMPushHyper(pVCpu, paParam[i]);
5319
5320 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5321
5322 /* Call the switcher. */
5323 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5324 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5325
5326 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5327 /* Make sure the VMX instructions don't cause #UD faults. */
5328 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5329
5330 /* Re-enter VMX Root Mode */
5331 int rc2 = VMXEnable(HCPhysCpuPage);
5332 if (RT_FAILURE(rc2))
5333 {
5334 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5335 ASMSetFlags(fOldEFlags);
5336 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5337 return rc2;
5338 }
5339
5340 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5341 AssertRC(rc2);
5342 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5343 Assert(!(ASMGetFlags() & X86_EFL_IF));
5344 ASMSetFlags(fOldEFlags);
5345 return rc;
5346}
5347
5348
5349/**
5350 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5351 * supporting 64-bit guests.
5352 *
5353 * @returns VBox status code.
5354 * @param fResume Whether to VMLAUNCH or VMRESUME.
5355 * @param pCtx Pointer to the guest-CPU context.
5356 * @param pCache Pointer to the VMCS cache.
5357 * @param pVM The cross context VM structure.
5358 * @param pVCpu The cross context virtual CPU structure.
5359 */
5360DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5361{
5362 NOREF(fResume);
5363
5364 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5365 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5366
5367#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5368 pCache->uPos = 1;
5369 pCache->interPD = PGMGetInterPaeCR3(pVM);
5370 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5371#endif
5372
5373#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5374 pCache->TestIn.HCPhysCpuPage = 0;
5375 pCache->TestIn.HCPhysVmcs = 0;
5376 pCache->TestIn.pCache = 0;
5377 pCache->TestOut.HCPhysVmcs = 0;
5378 pCache->TestOut.pCache = 0;
5379 pCache->TestOut.pCtx = 0;
5380 pCache->TestOut.eflags = 0;
5381#else
5382 NOREF(pCache);
5383#endif
5384
5385 uint32_t aParam[10];
5386 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5387 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5388 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5389 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5390 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5391 aParam[5] = 0;
5392 aParam[6] = VM_RC_ADDR(pVM, pVM);
5393 aParam[7] = 0;
5394 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5395 aParam[9] = 0;
5396
5397#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5398 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5399 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5400#endif
5401 int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5402
5403#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5404 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5405 Assert(pCtx->dr[4] == 10);
5406 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5407#endif
5408
5409#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5410 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5411 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5412 pVCpu->hm.s.vmx.HCPhysVmcs));
5413 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5414 pCache->TestOut.HCPhysVmcs));
5415 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5416 pCache->TestOut.pCache));
5417 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5418 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5419 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5420 pCache->TestOut.pCtx));
5421 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5422#endif
5423 NOREF(pCtx);
5424 return rc;
5425}
5426
5427
5428/**
5429 * Initialize the VMCS-Read cache.
5430 *
5431 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5432 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5433 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5434 * (those that have a 32-bit FULL & HIGH part).
5435 *
5436 * @returns VBox status code.
5437 * @param pVCpu The cross context virtual CPU structure.
5438 */
5439static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
5440{
5441#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5442 do { \
5443 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5444 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5445 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5446 ++cReadFields; \
5447 } while (0)
5448
5449 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5450 uint32_t cReadFields = 0;
5451
5452 /*
5453 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5454 * and serve to indicate exceptions to the rules.
5455 */
5456
5457 /* Guest-natural selector base fields. */
5458#if 0
5459 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5460 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5461 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5462#endif
5463 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5464 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5465 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5466 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5467 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5468 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5469 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5470 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5471 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5472 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5473 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5474 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5475#if 0
5476 /* Unused natural width guest-state fields. */
5477 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5478 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5479#endif
5480 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5481 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5482
5483 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
5484 these 64-bit fields (using "FULL" and "HIGH" fields). */
5485#if 0
5486 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5487 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5488 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5489 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5490 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5491 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5492 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5493 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5494 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5495#endif
5496
5497 /* Natural width guest-state fields. */
5498 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5499#if 0
5500 /* Currently unused field. */
5501 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5502#endif
5503
5504 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5505 {
5506 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5507 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5508 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5509 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5510 }
5511 else
5512 {
5513 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5514 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5515 }
5516
5517#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5518 return VINF_SUCCESS;
5519}
5520
5521
5522/**
5523 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5524 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5525 * darwin, running 64-bit guests).
5526 *
5527 * @returns VBox status code.
5528 * @param pVCpu The cross context virtual CPU structure.
5529 * @param idxField The VMCS field encoding.
5530 * @param u64Val 16, 32 or 64-bit value.
5531 */
5532VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5533{
5534 int rc;
5535 switch (idxField)
5536 {
5537 /*
5538 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5539 */
5540 /* 64-bit Control fields. */
5541 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5542 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5543 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5544 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5545 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5546 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5547 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5548 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5549 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5550 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5551 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5552 case VMX_VMCS64_CTRL_EPTP_FULL:
5553 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5554 /* 64-bit Guest-state fields. */
5555 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5556 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5557 case VMX_VMCS64_GUEST_PAT_FULL:
5558 case VMX_VMCS64_GUEST_EFER_FULL:
5559 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5560 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5561 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5562 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5563 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5564 /* 64-bit Host-state fields. */
5565 case VMX_VMCS64_HOST_PAT_FULL:
5566 case VMX_VMCS64_HOST_EFER_FULL:
5567 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5568 {
5569 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5570 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5571 break;
5572 }
5573
5574 /*
5575 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5576 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5577 */
5578 /* Natural-width Guest-state fields. */
5579 case VMX_VMCS_GUEST_CR3:
5580 case VMX_VMCS_GUEST_ES_BASE:
5581 case VMX_VMCS_GUEST_CS_BASE:
5582 case VMX_VMCS_GUEST_SS_BASE:
5583 case VMX_VMCS_GUEST_DS_BASE:
5584 case VMX_VMCS_GUEST_FS_BASE:
5585 case VMX_VMCS_GUEST_GS_BASE:
5586 case VMX_VMCS_GUEST_LDTR_BASE:
5587 case VMX_VMCS_GUEST_TR_BASE:
5588 case VMX_VMCS_GUEST_GDTR_BASE:
5589 case VMX_VMCS_GUEST_IDTR_BASE:
5590 case VMX_VMCS_GUEST_RSP:
5591 case VMX_VMCS_GUEST_RIP:
5592 case VMX_VMCS_GUEST_SYSENTER_ESP:
5593 case VMX_VMCS_GUEST_SYSENTER_EIP:
5594 {
5595 if (!(RT_HI_U32(u64Val)))
5596 {
5597 /* If this field is 64-bit, VT-x will zero out the top bits. */
5598 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5599 }
5600 else
5601 {
5602 /* Assert that only the 32->64 switcher case should ever come here. */
5603 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5604 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5605 }
5606 break;
5607 }
5608
5609 default:
5610 {
5611 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5612 rc = VERR_INVALID_PARAMETER;
5613 break;
5614 }
5615 }
5616 AssertRCReturn(rc, rc);
5617 return rc;
5618}
5619
5620
5621/**
5622 * Queue up a VMWRITE by using the VMCS write cache.
5623 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5624 *
5625 * @param pVCpu The cross context virtual CPU structure.
5626 * @param idxField The VMCS field encoding.
5627 * @param u64Val 16, 32 or 64-bit value.
5628 */
5629VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5630{
5631 AssertPtr(pVCpu);
5632 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5633
5634 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5635 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5636
5637 /* Make sure there are no duplicates. */
5638 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5639 {
5640 if (pCache->Write.aField[i] == idxField)
5641 {
5642 pCache->Write.aFieldVal[i] = u64Val;
5643 return VINF_SUCCESS;
5644 }
5645 }
5646
5647 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5648 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5649 pCache->Write.cValidEntries++;
5650 return VINF_SUCCESS;
5651}
5652#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5653
5654
5655/**
5656 * Sets up the usage of TSC-offsetting and updates the VMCS.
5657 *
5658 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5659 * VMX preemption timer.
5660 *
5661 * @returns VBox status code.
5662 * @param pVCpu The cross context virtual CPU structure.
5663 *
5664 * @remarks No-long-jump zone!!!
5665 */
5666static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5667{
5668 bool fOffsettedTsc;
5669 bool fParavirtTsc;
5670 PVM pVM = pVCpu->CTX_SUFF(pVM);
5671 uint64_t uTscOffset;
5672 if (pVM->hm.s.vmx.fUsePreemptTimer)
5673 {
5674 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
5675
5676 /* Make sure the returned values have sane upper and lower boundaries. */
5677 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5678 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5679 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5680 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5681
5682 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5683 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
5684 AssertRC(rc);
5685 }
5686 else
5687 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
5688
5689 /** @todo later optimize this to be done elsewhere and not before every
5690 * VM-entry. */
5691 if (fParavirtTsc)
5692 {
5693 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5694 information before every VM-entry, hence disable it for performance sake. */
5695#if 0
5696 int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5697 AssertRC(rc);
5698#endif
5699 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5700 }
5701
5702 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
5703 if ( fOffsettedTsc
5704 && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5705 {
5706 if (pVCpu->hm.s.vmx.u64TscOffset != uTscOffset)
5707 {
5708 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
5709 AssertRC(rc);
5710 pVCpu->hm.s.vmx.u64TscOffset = uTscOffset;
5711 }
5712
5713 if (uProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT)
5714 {
5715 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5716 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5717 AssertRC(rc);
5718 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5719 }
5720 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5721 }
5722 else
5723 {
5724 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5725 if (!(uProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
5726 {
5727 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5728 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5729 AssertRC(rc);
5730 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5731 }
5732 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5733 }
5734}
5735
5736
5737/**
5738 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5739 * VM-exit interruption info type.
5740 *
5741 * @returns The IEM exception flags.
5742 * @param uVector The event vector.
5743 * @param uVmxVectorType The VMX event type.
5744 *
5745 * @remarks This function currently only constructs flags required for
5746 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5747 * and CR2 aspects of an exception are not included).
5748 */
5749static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5750{
5751 uint32_t fIemXcptFlags;
5752 switch (uVmxVectorType)
5753 {
5754 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5755 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5756 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5757 break;
5758
5759 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5760 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5761 break;
5762
5763 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5764 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5765 break;
5766
5767 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5768 {
5769 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5770 if (uVector == X86_XCPT_BP)
5771 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5772 else if (uVector == X86_XCPT_OF)
5773 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5774 else
5775 {
5776 fIemXcptFlags = 0;
5777 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5778 }
5779 break;
5780 }
5781
5782 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5783 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5784 break;
5785
5786 default:
5787 fIemXcptFlags = 0;
5788 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5789 break;
5790 }
5791 return fIemXcptFlags;
5792}
5793
5794
5795/**
5796 * Sets an event as a pending event to be injected into the guest.
5797 *
5798 * @param pVCpu The cross context virtual CPU structure.
5799 * @param u32IntInfo The VM-entry interruption-information field.
5800 * @param cbInstr The VM-entry instruction length in bytes (for software
5801 * interrupts, exceptions and privileged software
5802 * exceptions).
5803 * @param u32ErrCode The VM-entry exception error code.
5804 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5805 * page-fault.
5806 *
5807 * @remarks Statistics counter assumes this is a guest event being injected or
5808 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5809 * always incremented.
5810 */
5811DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5812 RTGCUINTPTR GCPtrFaultAddress)
5813{
5814 Assert(!pVCpu->hm.s.Event.fPending);
5815 pVCpu->hm.s.Event.fPending = true;
5816 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5817 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5818 pVCpu->hm.s.Event.cbInstr = cbInstr;
5819 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5820}
5821
5822
5823/**
5824 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5825 *
5826 * @param pVCpu The cross context virtual CPU structure.
5827 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5828 * out-of-sync. Make sure to update the required fields
5829 * before using them.
5830 */
5831DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5832{
5833 NOREF(pMixedCtx);
5834 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5835 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5836 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5837 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5838}
5839
5840
5841/**
5842 * Handle a condition that occurred while delivering an event through the guest
5843 * IDT.
5844 *
5845 * @returns Strict VBox status code (i.e. informational status codes too).
5846 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5847 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5848 * to continue execution of the guest which will delivery the \#DF.
5849 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5850 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5851 *
5852 * @param pVCpu The cross context virtual CPU structure.
5853 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5854 * out-of-sync. Make sure to update the required fields
5855 * before using them.
5856 * @param pVmxTransient Pointer to the VMX transient structure.
5857 *
5858 * @remarks No-long-jump zone!!!
5859 */
5860static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5861{
5862 uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5863
5864 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5865 rc2 |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5866 AssertRCReturn(rc2, rc2);
5867
5868 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5869 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5870 {
5871 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5872 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5873
5874 /*
5875 * If the event was a software interrupt (generated with INT n) or a software exception
5876 * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
5877 * can handle the VM-exit and continue guest execution which will re-execute the
5878 * instruction rather than re-injecting the exception, as that can cause premature
5879 * trips to ring-3 before injection and involve TRPM which currently has no way of
5880 * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
5881 * the problem).
5882 */
5883 IEMXCPTRAISE enmRaise;
5884 IEMXCPTRAISEINFO fRaiseInfo;
5885 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5886 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5887 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5888 {
5889 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5890 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5891 }
5892 else if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5893 {
5894 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5895 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5896 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5897 /** @todo Make AssertMsgReturn as just AssertMsg later. */
5898 AssertMsgReturn(uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT,
5899 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5900 uExitVectorType), VERR_VMX_IPE_5);
5901
5902 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5903
5904 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5905 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
5906 {
5907 pVmxTransient->fVectoringPF = true;
5908 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5909 }
5910 }
5911 else
5912 {
5913 /*
5914 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5915 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5916 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
5917 */
5918 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5919 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5920 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5921 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5922 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5923 }
5924
5925 /*
5926 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5927 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5928 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5929 * subsequent VM-entry would fail.
5930 *
5931 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5932 */
5933 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5934 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5935 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
5936 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
5937 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5938 {
5939 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5940 }
5941
5942 switch (enmRaise)
5943 {
5944 case IEMXCPTRAISE_CURRENT_XCPT:
5945 {
5946 Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
5947 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
5948 Assert(rcStrict == VINF_SUCCESS);
5949 break;
5950 }
5951
5952 case IEMXCPTRAISE_PREV_EVENT:
5953 {
5954 uint32_t u32ErrCode;
5955 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5956 {
5957 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5958 AssertRCReturn(rc2, rc2);
5959 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5960 }
5961 else
5962 u32ErrCode = 0;
5963
5964 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
5965 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5966 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5967 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5968
5969 Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
5970 pVCpu->hm.s.Event.u32ErrCode));
5971 Assert(rcStrict == VINF_SUCCESS);
5972 break;
5973 }
5974
5975 case IEMXCPTRAISE_REEXEC_INSTR:
5976 Assert(rcStrict == VINF_SUCCESS);
5977 break;
5978
5979 case IEMXCPTRAISE_DOUBLE_FAULT:
5980 {
5981 /*
5982 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
5983 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
5984 */
5985 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
5986 {
5987 pVmxTransient->fVectoringDoublePF = true;
5988 Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
5989 pMixedCtx->cr2));
5990 rcStrict = VINF_SUCCESS;
5991 }
5992 else
5993 {
5994 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5995 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5996 Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
5997 uIdtVector, uExitVector));
5998 rcStrict = VINF_HM_DOUBLE_FAULT;
5999 }
6000 break;
6001 }
6002
6003 case IEMXCPTRAISE_TRIPLE_FAULT:
6004 {
6005 Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
6006 rcStrict = VINF_EM_RESET;
6007 break;
6008 }
6009
6010 case IEMXCPTRAISE_CPU_HANG:
6011 {
6012 Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
6013 rcStrict = VERR_EM_GUEST_CPU_HANG;
6014 break;
6015 }
6016
6017 default:
6018 {
6019 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6020 rcStrict = VERR_VMX_IPE_2;
6021 break;
6022 }
6023 }
6024 }
6025 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6026 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6027 && uExitVector != X86_XCPT_DF
6028 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6029 {
6030 /*
6031 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6032 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6033 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6034 */
6035 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6036 {
6037 Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
6038 VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6039 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6040 }
6041 }
6042
6043 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
6044 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
6045 return rcStrict;
6046}
6047
6048
6049/**
6050 * Imports a guest segment register from the current VMCS into
6051 * the guest-CPU context.
6052 *
6053 * @returns VBox status code.
6054 * @param pVCpu The cross context virtual CPU structure.
6055 * @param idxSel Index of the selector in the VMCS.
6056 * @param idxLimit Index of the segment limit in the VMCS.
6057 * @param idxBase Index of the segment base in the VMCS.
6058 * @param idxAccess Index of the access rights of the segment in the VMCS.
6059 * @param pSelReg Pointer to the segment selector.
6060 *
6061 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6062 * do not log!
6063 *
6064 * @remarks Never call this function directly!!! Use the
6065 * HMVMX_IMPORT_SREG() macro as that takes care
6066 * of whether to read from the VMCS cache or not.
6067 */
6068static int hmR0VmxImportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6069 PCPUMSELREG pSelReg)
6070{
6071 NOREF(pVCpu);
6072
6073 uint32_t u32Sel;
6074 uint32_t u32Limit;
6075 uint32_t u32Attr;
6076 uint64_t u64Base;
6077 int rc = VMXReadVmcs32(idxSel, &u32Sel);
6078 rc |= VMXReadVmcs32(idxLimit, &u32Limit);
6079 rc |= VMXReadVmcs32(idxAccess, &u32Attr);
6080 rc |= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
6081 AssertRCReturn(rc, rc);
6082
6083 pSelReg->Sel = (uint16_t)u32Sel;
6084 pSelReg->ValidSel = (uint16_t)u32Sel;
6085 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6086 pSelReg->u32Limit = u32Limit;
6087 pSelReg->u64Base = u64Base;
6088 pSelReg->Attr.u = u32Attr;
6089
6090 /*
6091 * If VT-x marks the segment as unusable, most other bits remain undefined:
6092 * - For CS the L, D and G bits have meaning.
6093 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6094 * - For the remaining data segments no bits are defined.
6095 *
6096 * The present bit and the unusable bit has been observed to be set at the
6097 * same time (the selector was supposed to be invalid as we started executing
6098 * a V8086 interrupt in ring-0).
6099 *
6100 * What should be important for the rest of the VBox code, is that the P bit is
6101 * cleared. Some of the other VBox code recognizes the unusable bit, but
6102 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6103 * safe side here, we'll strip off P and other bits we don't care about. If
6104 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6105 *
6106 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6107 */
6108 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6109 {
6110 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6111
6112 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6113 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6114 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6115#ifdef VBOX_STRICT
6116 VMMRZCallRing3Disable(pVCpu);
6117 Log4Func(("Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Sel, pSelReg->Attr.u));
6118# ifdef DEBUG_bird
6119 AssertMsg((u32Attr & ~X86DESCATTR_P) == pSelReg->Attr.u,
6120 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6121 idxSel, u32Sel, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6122# endif
6123 VMMRZCallRing3Enable(pVCpu);
6124#endif
6125 }
6126 return VINF_SUCCESS;
6127}
6128
6129
6130/**
6131 * Imports the guest RIP from the VMCS back into the guest-CPU context.
6132 *
6133 * @returns VBox status code.
6134 * @param pVCpu The cross context virtual CPU structure.
6135 *
6136 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6137 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6138 * instead!!!
6139 */
6140DECLINLINE(int) hmR0VmxImportGuestRip(PVMCPU pVCpu)
6141{
6142 uint64_t u64Val;
6143 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6144 if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
6145 {
6146 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6147 if (RT_SUCCESS(rc))
6148 {
6149 pCtx->rip = u64Val;
6150 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
6151 }
6152 return rc;
6153 }
6154 return VINF_SUCCESS;
6155}
6156
6157
6158/**
6159 * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
6160 *
6161 * @returns VBox status code.
6162 * @param pVCpu The cross context virtual CPU structure.
6163 *
6164 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6165 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6166 * instead!!!
6167 */
6168DECLINLINE(int) hmR0VmxImportGuestRFlags(PVMCPU pVCpu)
6169{
6170 uint32_t u32Val;
6171 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6172 if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
6173 {
6174 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
6175 if (RT_SUCCESS(rc))
6176 {
6177 pCtx->eflags.u32 = u32Val;
6178
6179 /* Restore eflags for real-on-v86-mode hack. */
6180 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6181 {
6182 pCtx->eflags.Bits.u1VM = 0;
6183 pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6184 }
6185 }
6186 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
6187 return rc;
6188 }
6189 return VINF_SUCCESS;
6190}
6191
6192
6193/**
6194 * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
6195 * context.
6196 *
6197 * @returns VBox status code.
6198 * @param pVCpu The cross context virtual CPU structure.
6199 *
6200 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6201 * do not log!
6202 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6203 * instead!!!
6204 */
6205DECLINLINE(int) hmR0VmxImportGuestIntrState(PVMCPU pVCpu)
6206{
6207 uint32_t u32Val;
6208 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6209 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32Val);
6210 if (RT_SUCCESS(rc))
6211 {
6212 /*
6213 * We additionally have a requirement to import RIP, RFLAGS depending on whether we
6214 * might need them in hmR0VmxEvaluatePendingEvent().
6215 */
6216 if (!u32Val)
6217 {
6218 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6219 {
6220 rc = hmR0VmxImportGuestRip(pVCpu);
6221 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6222 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6223 }
6224
6225 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6226 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6227 }
6228 else
6229 {
6230 rc = hmR0VmxImportGuestRip(pVCpu);
6231 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6232
6233 if (u32Val & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6234 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6235 {
6236 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
6237 }
6238 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6239 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6240
6241 if (u32Val & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6242 {
6243 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6244 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6245 }
6246 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6247 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6248 }
6249 }
6250 return rc;
6251}
6252
6253
6254/**
6255 * Worker for VMXR0ImportStateOnDemand.
6256 *
6257 * @returns VBox status code.
6258 * @param pVCpu The cross context virtual CPU structure.
6259 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6260 */
6261static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
6262{
6263#define VMXLOCAL_BREAK_RC(a_rc) \
6264 if (RT_FAILURE(a_rc)) \
6265 break
6266
6267 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
6268
6269 int rc = VINF_SUCCESS;
6270 PVM pVM = pVCpu->CTX_SUFF(pVM);
6271 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6272 uint64_t u64Val;
6273 uint32_t u32Val;
6274
6275 Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
6276
6277 /*
6278 * We disable interrupts to make the updating of the state and in particular
6279 * the fExtrn modification atomic wrt to preemption hooks.
6280 */
6281 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
6282
6283 fWhat &= pCtx->fExtrn;
6284 if (fWhat)
6285 {
6286 do
6287 {
6288 if (fWhat & CPUMCTX_EXTRN_RIP)
6289 {
6290 rc = hmR0VmxImportGuestRip(pVCpu);
6291 VMXLOCAL_BREAK_RC(rc);
6292 }
6293
6294 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
6295 {
6296 rc = hmR0VmxImportGuestRFlags(pVCpu);
6297 VMXLOCAL_BREAK_RC(rc);
6298 }
6299
6300 if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
6301 {
6302 rc = hmR0VmxImportGuestIntrState(pVCpu);
6303 VMXLOCAL_BREAK_RC(rc);
6304 }
6305
6306 if (fWhat & CPUMCTX_EXTRN_RSP)
6307 {
6308 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6309 VMXLOCAL_BREAK_RC(rc);
6310 pCtx->rsp = u64Val;
6311 }
6312
6313 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
6314 {
6315 if (fWhat & CPUMCTX_EXTRN_CS)
6316 {
6317 rc = HMVMX_IMPORT_SREG(CS, &pCtx->cs);
6318 VMXLOCAL_BREAK_RC(rc);
6319 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6320 pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6321 }
6322 if (fWhat & CPUMCTX_EXTRN_SS)
6323 {
6324 rc = HMVMX_IMPORT_SREG(SS, &pCtx->ss);
6325 VMXLOCAL_BREAK_RC(rc);
6326 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6327 pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6328 }
6329 if (fWhat & CPUMCTX_EXTRN_DS)
6330 {
6331 rc = HMVMX_IMPORT_SREG(DS, &pCtx->ds);
6332 VMXLOCAL_BREAK_RC(rc);
6333 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6334 pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6335 }
6336 if (fWhat & CPUMCTX_EXTRN_ES)
6337 {
6338 rc = HMVMX_IMPORT_SREG(ES, &pCtx->es);
6339 VMXLOCAL_BREAK_RC(rc);
6340 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6341 pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6342 }
6343 if (fWhat & CPUMCTX_EXTRN_FS)
6344 {
6345 rc = HMVMX_IMPORT_SREG(FS, &pCtx->fs);
6346 VMXLOCAL_BREAK_RC(rc);
6347 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6348 pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6349 }
6350 if (fWhat & CPUMCTX_EXTRN_GS)
6351 {
6352 rc = HMVMX_IMPORT_SREG(GS, &pCtx->gs);
6353 VMXLOCAL_BREAK_RC(rc);
6354 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6355 pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6356 }
6357 }
6358
6359 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
6360 {
6361 if (fWhat & CPUMCTX_EXTRN_LDTR)
6362 {
6363 rc = HMVMX_IMPORT_SREG(LDTR, &pCtx->ldtr);
6364 VMXLOCAL_BREAK_RC(rc);
6365 }
6366
6367 if (fWhat & CPUMCTX_EXTRN_GDTR)
6368 {
6369 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6370 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
6371 VMXLOCAL_BREAK_RC(rc);
6372 pCtx->gdtr.pGdt = u64Val;
6373 pCtx->gdtr.cbGdt = u32Val;
6374 }
6375
6376 /* Guest IDTR. */
6377 if (fWhat & CPUMCTX_EXTRN_IDTR)
6378 {
6379 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6380 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
6381 VMXLOCAL_BREAK_RC(rc);
6382 pCtx->idtr.pIdt = u64Val;
6383 pCtx->idtr.cbIdt = u32Val;
6384 }
6385
6386 /* Guest TR. */
6387 if (fWhat & CPUMCTX_EXTRN_TR)
6388 {
6389 /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
6390 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6391 {
6392 rc = HMVMX_IMPORT_SREG(TR, &pCtx->tr);
6393 VMXLOCAL_BREAK_RC(rc);
6394 }
6395 }
6396 }
6397
6398 if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
6399 {
6400 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
6401 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
6402 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
6403 pCtx->SysEnter.cs = u32Val;
6404 VMXLOCAL_BREAK_RC(rc);
6405 }
6406
6407#if HC_ARCH_BITS == 64
6408 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
6409 {
6410 if ( pVM->hm.s.fAllow64BitGuests
6411 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6412 pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
6413 }
6414
6415 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
6416 {
6417 if ( pVM->hm.s.fAllow64BitGuests
6418 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6419 {
6420 pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
6421 pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
6422 pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
6423 }
6424 }
6425#endif
6426
6427 if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
6428#if HC_ARCH_BITS == 32
6429 || (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS))
6430#endif
6431 )
6432 {
6433 PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6434 uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
6435 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6436 {
6437 switch (pMsr->u32Msr)
6438 {
6439#if HC_ARCH_BITS == 32
6440 case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
6441 case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
6442 case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
6443 case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6444#endif
6445 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6446 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6447 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit */ break;
6448 default:
6449 {
6450 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6451 ASMSetFlags(fEFlags);
6452 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
6453 cMsrs));
6454 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6455 }
6456 }
6457 }
6458 }
6459
6460 if (fWhat & CPUMCTX_EXTRN_DR7)
6461 {
6462 if (!pVCpu->hm.s.fUsingHyperDR7)
6463 {
6464 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6465 rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
6466 VMXLOCAL_BREAK_RC(rc);
6467 pCtx->dr[7] = u32Val;
6468 }
6469 }
6470
6471 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
6472 {
6473 uint32_t u32Shadow;
6474 if (fWhat & CPUMCTX_EXTRN_CR0)
6475 {
6476 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
6477 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
6478 VMXLOCAL_BREAK_RC(rc);
6479 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr0Mask)
6480 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr0Mask);
6481 VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
6482 CPUMSetGuestCR0(pVCpu, u32Val);
6483 VMMRZCallRing3Enable(pVCpu);
6484 }
6485
6486 if (fWhat & CPUMCTX_EXTRN_CR4)
6487 {
6488 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
6489 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
6490 VMXLOCAL_BREAK_RC(rc);
6491 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr4Mask)
6492 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr4Mask);
6493 CPUMSetGuestCR4(pVCpu, u32Val);
6494 }
6495
6496 if (fWhat & CPUMCTX_EXTRN_CR3)
6497 {
6498 /* CR0.PG bit changes are always intercepted, so it's up to date. */
6499 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6500 || ( pVM->hm.s.fNestedPaging
6501 && CPUMIsGuestPagingEnabledEx(pCtx)))
6502 {
6503 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6504 if (pCtx->cr3 != u64Val)
6505 {
6506 CPUMSetGuestCR3(pVCpu, u64Val);
6507 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6508 }
6509
6510 /* If the guest is in PAE mode, sync back the PDPE's into the guest state.
6511 Note: CR4.PAE, CR0.PG, EFER bit changes are always intercepted, so they're up to date. */
6512 if (CPUMIsGuestInPAEModeEx(pCtx))
6513 {
6514 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6515 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6516 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6517 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6518 VMXLOCAL_BREAK_RC(rc);
6519 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6520 }
6521 }
6522 }
6523 }
6524 } while (0);
6525
6526 if (RT_SUCCESS(rc))
6527 {
6528 /* Update fExtrn. */
6529 pCtx->fExtrn &= ~fWhat;
6530
6531 /* If everything has been imported, clear the HM keeper bit. */
6532 if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
6533 {
6534 pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
6535 Assert(!pCtx->fExtrn);
6536 }
6537 }
6538 }
6539 else
6540 AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
6541
6542 ASMSetFlags(fEFlags);
6543
6544 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
6545
6546 /*
6547 * Honor any pending CR3 updates.
6548 *
6549 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6550 * -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6551 * -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
6552 *
6553 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6554 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6555 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6556 * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
6557 *
6558 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6559 */
6560 if (VMMRZCallRing3IsEnabled(pVCpu))
6561 {
6562 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6563 {
6564 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
6565 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6566 }
6567
6568 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6569 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6570
6571 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6572 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6573 }
6574
6575 return VINF_SUCCESS;
6576#undef VMXLOCAL_BREAK_RC
6577}
6578
6579
6580/**
6581 * Saves the guest state from the VMCS into the guest-CPU context.
6582 *
6583 * @returns VBox status code.
6584 * @param pVCpu The cross context virtual CPU structure.
6585 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6586 */
6587VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
6588{
6589 return hmR0VmxImportGuestState(pVCpu, fWhat);
6590}
6591
6592
6593/**
6594 * Check per-VM and per-VCPU force flag actions that require us to go back to
6595 * ring-3 for one reason or another.
6596 *
6597 * @returns Strict VBox status code (i.e. informational status codes too)
6598 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6599 * ring-3.
6600 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6601 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6602 * interrupts)
6603 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6604 * all EMTs to be in ring-3.
6605 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6606 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6607 * to the EM loop.
6608 *
6609 * @param pVCpu The cross context virtual CPU structure.
6610 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6611 * out-of-sync. Make sure to update the required fields
6612 * before using them.
6613 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6614 */
6615static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6616{
6617 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6618
6619 /*
6620 * Anything pending? Should be more likely than not if we're doing a good job.
6621 */
6622 PVM pVM = pVCpu->CTX_SUFF(pVM);
6623 if ( !fStepping
6624 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6625 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6626 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6627 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6628 return VINF_SUCCESS;
6629
6630 /* Pending PGM C3 sync. */
6631 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6632 {
6633 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
6634 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6635 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6636 if (rcStrict2 != VINF_SUCCESS)
6637 {
6638 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6639 Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6640 return rcStrict2;
6641 }
6642 }
6643
6644 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6645 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6646 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6647 {
6648 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6649 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6650 Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6651 return rc2;
6652 }
6653
6654 /* Pending VM request packets, such as hardware interrupts. */
6655 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6656 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6657 {
6658 Log4Func(("Pending VM request forcing us back to ring-3\n"));
6659 return VINF_EM_PENDING_REQUEST;
6660 }
6661
6662 /* Pending PGM pool flushes. */
6663 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6664 {
6665 Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
6666 return VINF_PGM_POOL_FLUSH_PENDING;
6667 }
6668
6669 /* Pending DMA requests. */
6670 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6671 {
6672 Log4Func(("Pending DMA request forcing us back to ring-3\n"));
6673 return VINF_EM_RAW_TO_R3;
6674 }
6675
6676 return VINF_SUCCESS;
6677}
6678
6679
6680/**
6681 * Converts any TRPM trap into a pending HM event. This is typically used when
6682 * entering from ring-3 (not longjmp returns).
6683 *
6684 * @param pVCpu The cross context virtual CPU structure.
6685 */
6686static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6687{
6688 Assert(TRPMHasTrap(pVCpu));
6689 Assert(!pVCpu->hm.s.Event.fPending);
6690
6691 uint8_t uVector;
6692 TRPMEVENT enmTrpmEvent;
6693 RTGCUINT uErrCode;
6694 RTGCUINTPTR GCPtrFaultAddress;
6695 uint8_t cbInstr;
6696
6697 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6698 AssertRC(rc);
6699
6700 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6701 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6702 if (enmTrpmEvent == TRPM_TRAP)
6703 {
6704 switch (uVector)
6705 {
6706 case X86_XCPT_NMI:
6707 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6708 break;
6709
6710 case X86_XCPT_BP:
6711 case X86_XCPT_OF:
6712 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6713 break;
6714
6715 case X86_XCPT_PF:
6716 case X86_XCPT_DF:
6717 case X86_XCPT_TS:
6718 case X86_XCPT_NP:
6719 case X86_XCPT_SS:
6720 case X86_XCPT_GP:
6721 case X86_XCPT_AC:
6722 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6723 RT_FALL_THRU();
6724 default:
6725 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6726 break;
6727 }
6728 }
6729 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6730 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6731 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6732 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6733 else
6734 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6735
6736 rc = TRPMResetTrap(pVCpu);
6737 AssertRC(rc);
6738 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6739 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6740
6741 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6742}
6743
6744
6745/**
6746 * Converts the pending HM event into a TRPM trap.
6747 *
6748 * @param pVCpu The cross context virtual CPU structure.
6749 */
6750static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6751{
6752 Assert(pVCpu->hm.s.Event.fPending);
6753
6754 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6755 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6756 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6757 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6758
6759 /* If a trap was already pending, we did something wrong! */
6760 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6761
6762 TRPMEVENT enmTrapType;
6763 switch (uVectorType)
6764 {
6765 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6766 enmTrapType = TRPM_HARDWARE_INT;
6767 break;
6768
6769 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6770 enmTrapType = TRPM_SOFTWARE_INT;
6771 break;
6772
6773 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6774 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6775 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6776 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6777 enmTrapType = TRPM_TRAP;
6778 break;
6779
6780 default:
6781 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6782 enmTrapType = TRPM_32BIT_HACK;
6783 break;
6784 }
6785
6786 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6787
6788 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6789 AssertRC(rc);
6790
6791 if (fErrorCodeValid)
6792 TRPMSetErrorCode(pVCpu, uErrorCode);
6793
6794 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6795 && uVector == X86_XCPT_PF)
6796 {
6797 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6798 }
6799 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6800 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6801 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6802 {
6803 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6804 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6805 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6806 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6807 }
6808
6809 /* Clear any pending events from the VMCS. */
6810 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
6811 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
6812
6813 /* We're now done converting the pending event. */
6814 pVCpu->hm.s.Event.fPending = false;
6815}
6816
6817
6818/**
6819 * Does the necessary state syncing before returning to ring-3 for any reason
6820 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6821 *
6822 * @returns VBox status code.
6823 * @param pVCpu The cross context virtual CPU structure.
6824 * @param fImportState Whether to import the guest state from the VMCS back
6825 * to the guest-CPU context.
6826 *
6827 * @remarks No-long-jmp zone!!!
6828 */
6829static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
6830{
6831 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6832 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6833
6834 RTCPUID idCpu = RTMpCpuId();
6835 Log4Func(("HostCpuId=%u\n", idCpu));
6836
6837 /*
6838 * !!! IMPORTANT !!!
6839 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
6840 */
6841
6842 /* Save the guest state if necessary. */
6843 if (fImportState)
6844 {
6845 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
6846 AssertRCReturn(rc, rc);
6847 }
6848
6849 /* Restore host FPU state if necessary. We will resync on next R0 reentry. */
6850 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
6851 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6852
6853 /* Restore host debug registers if necessary. We will resync on next R0 reentry. */
6854#ifdef VBOX_STRICT
6855 if (CPUMIsHyperDebugStateActive(pVCpu))
6856 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6857#endif
6858 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6859 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6860 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6861
6862#if HC_ARCH_BITS == 64
6863 /* Restore host-state bits that VT-x only restores partially. */
6864 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6865 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6866 {
6867 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6868 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6869 }
6870 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6871#endif
6872
6873 /* Restore the lazy host MSRs as we're leaving VT-x context. */
6874 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
6875 {
6876 /* We shouldn't restore the host MSRs without saving the guest MSRs first. */
6877 if (!fImportState)
6878 {
6879 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE
6880 | CPUMCTX_EXTRN_SYSCALL_MSRS);
6881 AssertRCReturn(rc, rc);
6882 }
6883 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6884 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
6885 }
6886 else
6887 pVCpu->hm.s.vmx.fLazyMsrs = 0;
6888
6889 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6890 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6891
6892 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6893 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
6894 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
6895 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
6896 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
6897 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6898 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6899 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6900 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6901
6902 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6903
6904 /** @todo This partially defeats the purpose of having preemption hooks.
6905 * The problem is, deregistering the hooks should be moved to a place that
6906 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6907 * context.
6908 */
6909 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6910 {
6911 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6912 AssertRCReturn(rc, rc);
6913
6914 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6915 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6916 }
6917 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6918 NOREF(idCpu);
6919
6920 return VINF_SUCCESS;
6921}
6922
6923
6924/**
6925 * Leaves the VT-x session.
6926 *
6927 * @returns VBox status code.
6928 * @param pVCpu The cross context virtual CPU structure.
6929 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6930 * out-of-sync. Make sure to update the required fields
6931 * before using them.
6932 *
6933 * @remarks No-long-jmp zone!!!
6934 */
6935static int hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6936{
6937 HM_DISABLE_PREEMPT();
6938 HMVMX_ASSERT_CPU_SAFE();
6939 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6940 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6941
6942 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6943 and done this from the VMXR0ThreadCtxCallback(). */
6944 if (!pVCpu->hm.s.fLeaveDone)
6945 {
6946 int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
6947 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
6948 pVCpu->hm.s.fLeaveDone = true;
6949 }
6950 Assert(!pMixedCtx->fExtrn); NOREF(pMixedCtx);
6951
6952 /*
6953 * !!! IMPORTANT !!!
6954 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6955 */
6956
6957 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6958 /** @todo Deregistering here means we need to VMCLEAR always
6959 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6960 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
6961 VMMR0ThreadCtxHookDisable(pVCpu);
6962
6963 /* Leave HM context. This takes care of local init (term). */
6964 int rc = HMR0LeaveCpu(pVCpu);
6965
6966 HM_RESTORE_PREEMPT();
6967 return rc;
6968}
6969
6970
6971/**
6972 * Does the necessary state syncing before doing a longjmp to ring-3.
6973 *
6974 * @returns VBox status code.
6975 * @param pVCpu The cross context virtual CPU structure.
6976 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6977 * out-of-sync. Make sure to update the required fields
6978 * before using them.
6979 *
6980 * @remarks No-long-jmp zone!!!
6981 */
6982DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6983{
6984 return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
6985}
6986
6987
6988/**
6989 * Take necessary actions before going back to ring-3.
6990 *
6991 * An action requires us to go back to ring-3. This function does the necessary
6992 * steps before we can safely return to ring-3. This is not the same as longjmps
6993 * to ring-3, this is voluntary and prepares the guest so it may continue
6994 * executing outside HM (recompiler/IEM).
6995 *
6996 * @returns VBox status code.
6997 * @param pVCpu The cross context virtual CPU structure.
6998 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6999 * out-of-sync. Make sure to update the required fields
7000 * before using them.
7001 * @param rcExit The reason for exiting to ring-3. Can be
7002 * VINF_VMM_UNKNOWN_RING3_CALL.
7003 */
7004static int hmR0VmxExitToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7005{
7006 Assert(pVCpu);
7007 Assert(pMixedCtx);
7008 HMVMX_ASSERT_PREEMPT_SAFE();
7009
7010 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7011 {
7012 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7013 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7014 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7015 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7016 }
7017
7018 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7019 VMMRZCallRing3Disable(pVCpu);
7020 Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
7021
7022 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7023 if (pVCpu->hm.s.Event.fPending)
7024 {
7025 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7026 Assert(!pVCpu->hm.s.Event.fPending);
7027 }
7028
7029 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7030 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7031
7032 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7033 and if we're injecting an event we should have a TRPM trap pending. */
7034 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7035#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
7036 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7037#endif
7038
7039 /* Save guest state and restore host state bits. */
7040 int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7041 AssertRCReturn(rc, rc);
7042 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7043 /* Thread-context hooks are unregistered at this point!!! */
7044
7045 /* Sync recompiler state. */
7046 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7047 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7048 | CPUM_CHANGED_LDTR
7049 | CPUM_CHANGED_GDTR
7050 | CPUM_CHANGED_IDTR
7051 | CPUM_CHANGED_TR
7052 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7053 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
7054 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7055 {
7056 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7057 }
7058
7059 Assert(!pVCpu->hm.s.fClearTrapFlag);
7060
7061 /* Update the exit-to-ring 3 reason. */
7062 pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
7063
7064 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7065 if (rcExit != VINF_EM_RAW_INTERRUPT)
7066 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7067
7068 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7069
7070 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7071 VMMRZCallRing3RemoveNotification(pVCpu);
7072 VMMRZCallRing3Enable(pVCpu);
7073
7074 return rc;
7075}
7076
7077
7078/**
7079 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7080 * longjump to ring-3 and possibly get preempted.
7081 *
7082 * @returns VBox status code.
7083 * @param pVCpu The cross context virtual CPU structure.
7084 * @param enmOperation The operation causing the ring-3 longjump.
7085 * @param pvUser Opaque pointer to the guest-CPU context. The data
7086 * may be out-of-sync. Make sure to update the required
7087 * fields before using them.
7088 */
7089static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7090{
7091 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7092 {
7093 /*
7094 * !!! IMPORTANT !!!
7095 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7096 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7097 */
7098 VMMRZCallRing3RemoveNotification(pVCpu);
7099 VMMRZCallRing3Disable(pVCpu);
7100 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7101 RTThreadPreemptDisable(&PreemptState);
7102
7103 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7104 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7105
7106#if HC_ARCH_BITS == 64
7107 /* Restore host-state bits that VT-x only restores partially. */
7108 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7109 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7110 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7111 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7112#endif
7113
7114 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7115 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7116 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7117
7118 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7119 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7120 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7121 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7122 {
7123 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7124 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7125 }
7126
7127 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7128 VMMR0ThreadCtxHookDisable(pVCpu);
7129 HMR0LeaveCpu(pVCpu);
7130 RTThreadPreemptRestore(&PreemptState);
7131 return VINF_SUCCESS;
7132 }
7133
7134 Assert(pVCpu);
7135 Assert(pvUser);
7136 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7137 HMVMX_ASSERT_PREEMPT_SAFE();
7138
7139 VMMRZCallRing3Disable(pVCpu);
7140 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7141
7142 Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
7143
7144 int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7145 AssertRCReturn(rc, rc);
7146
7147 VMMRZCallRing3Enable(pVCpu);
7148 return VINF_SUCCESS;
7149}
7150
7151
7152/**
7153 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7154 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7155 *
7156 * @param pVCpu The cross context virtual CPU structure.
7157 */
7158DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7159{
7160 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7161 {
7162 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7163 {
7164 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7165 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7166 AssertRC(rc);
7167 Log4Func(("Setup interrupt-window exiting\n"));
7168 }
7169 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7170}
7171
7172
7173/**
7174 * Clears the interrupt-window exiting control in the VMCS.
7175 *
7176 * @param pVCpu The cross context virtual CPU structure.
7177 */
7178DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7179{
7180 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7181 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7182 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7183 AssertRC(rc);
7184 Log4Func(("Cleared interrupt-window exiting\n"));
7185}
7186
7187
7188/**
7189 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7190 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7191 *
7192 * @param pVCpu The cross context virtual CPU structure.
7193 */
7194DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7195{
7196 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7197 {
7198 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7199 {
7200 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7201 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7202 AssertRC(rc);
7203 Log4Func(("Setup NMI-window exiting\n"));
7204 }
7205 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7206}
7207
7208
7209/**
7210 * Clears the NMI-window exiting control in the VMCS.
7211 *
7212 * @param pVCpu The cross context virtual CPU structure.
7213 */
7214DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7215{
7216 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7217 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7218 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7219 AssertRC(rc);
7220 Log4Func(("Cleared NMI-window exiting\n"));
7221}
7222
7223
7224/**
7225 * Evaluates the event to be delivered to the guest and sets it as the pending
7226 * event.
7227 *
7228 * @returns The VT-x guest-interruptibility state.
7229 * @param pVCpu The cross context virtual CPU structure.
7230 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7231 * out-of-sync. Make sure to update the required fields
7232 * before using them.
7233 */
7234static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7235{
7236 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7237 uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7238 bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7239 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7240 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7241
7242 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7243 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7244 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7245 Assert(!TRPMHasTrap(pVCpu));
7246
7247 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7248 APICUpdatePendingInterrupts(pVCpu);
7249
7250 /*
7251 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7252 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7253 */
7254 /** @todo SMI. SMIs take priority over NMIs. */
7255 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7256 {
7257 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7258 if ( !pVCpu->hm.s.Event.fPending
7259 && !fBlockNmi
7260 && !fBlockSti
7261 && !fBlockMovSS)
7262 {
7263 Log4Func(("Pending NMI\n"));
7264 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7265 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7266
7267 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7268 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7269 }
7270 else
7271 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7272 }
7273 /*
7274 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7275 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7276 */
7277 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7278 && !pVCpu->hm.s.fSingleInstruction)
7279 {
7280 Assert(!DBGFIsStepping(pVCpu));
7281 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7282 AssertRCReturn(rc, 0);
7283 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7284 if ( !pVCpu->hm.s.Event.fPending
7285 && !fBlockInt
7286 && !fBlockSti
7287 && !fBlockMovSS)
7288 {
7289 uint8_t u8Interrupt;
7290 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7291 if (RT_SUCCESS(rc))
7292 {
7293 Log4Func(("Pending external interrupt u8Interrupt=%#x\n", u8Interrupt));
7294 uint32_t u32IntInfo = u8Interrupt
7295 | VMX_EXIT_INTERRUPTION_INFO_VALID
7296 | (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7297
7298 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7299 }
7300 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7301 {
7302 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7303 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7304 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7305
7306 /*
7307 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7308 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7309 * need to re-set this force-flag here.
7310 */
7311 }
7312 else
7313 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7314 }
7315 else
7316 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7317 }
7318
7319 return fIntrState;
7320}
7321
7322
7323/**
7324 * Sets a pending-debug exception to be delivered to the guest if the guest is
7325 * single-stepping in the VMCS.
7326 *
7327 * @param pVCpu The cross context virtual CPU structure.
7328 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7329 * out-of-sync. Make sure to update the required fields
7330 * before using them.
7331 */
7332DECLINLINE(int) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7333{
7334 RT_NOREF(pVCpu);
7335 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS)); NOREF(pMixedCtx);
7336 return VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7337}
7338
7339
7340/**
7341 * Injects any pending events into the guest if the guest is in a state to
7342 * receive them.
7343 *
7344 * @returns Strict VBox status code (i.e. informational status codes too).
7345 * @param pVCpu The cross context virtual CPU structure.
7346 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7347 * out-of-sync. Make sure to update the required fields
7348 * before using them.
7349 * @param fIntrState The VT-x guest-interruptibility state.
7350 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7351 * return VINF_EM_DBG_STEPPED if the event was
7352 * dispatched directly.
7353 */
7354static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t fIntrState, bool fStepping)
7355{
7356 HMVMX_ASSERT_PREEMPT_SAFE();
7357 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7358
7359 bool fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7360 bool fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7361
7362 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7363 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7364 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7365 Assert(!TRPMHasTrap(pVCpu));
7366
7367 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7368 if (pVCpu->hm.s.Event.fPending)
7369 {
7370 /*
7371 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7372 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7373 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7374 *
7375 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7376 */
7377 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7378#ifdef VBOX_STRICT
7379 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7380 {
7381 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7382 Assert(!fBlockInt);
7383 Assert(!fBlockSti);
7384 Assert(!fBlockMovSS);
7385 }
7386 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7387 {
7388 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7389 Assert(!fBlockSti);
7390 Assert(!fBlockMovSS);
7391 Assert(!fBlockNmi);
7392 }
7393#endif
7394 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7395 uIntType));
7396 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7397 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping,
7398 &fIntrState);
7399 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7400
7401 /* Update the interruptibility-state as it could have been changed by
7402 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7403 fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7404 fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7405
7406 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7407 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7408 else
7409 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7410 }
7411
7412 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7413 if ( fBlockSti
7414 || fBlockMovSS)
7415 {
7416 if (!pVCpu->hm.s.fSingleInstruction)
7417 {
7418 /*
7419 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7420 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7421 * See Intel spec. 27.3.4 "Saving Non-Register State".
7422 */
7423 Assert(!DBGFIsStepping(pVCpu));
7424 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7425 AssertRCReturn(rc, rc);
7426 if (pMixedCtx->eflags.Bits.u1TF)
7427 {
7428 int rc2 = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
7429 AssertRCReturn(rc2, rc2);
7430 }
7431 }
7432 else if (pMixedCtx->eflags.Bits.u1TF)
7433 {
7434 /*
7435 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7436 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7437 */
7438 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7439 fIntrState = 0;
7440 }
7441 }
7442
7443 /*
7444 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7445 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7446 */
7447 int rc3 = hmR0VmxExportGuestIntrState(pVCpu, fIntrState);
7448 AssertRCReturn(rc3, rc3);
7449
7450 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7451 NOREF(fBlockMovSS); NOREF(fBlockSti);
7452 return rcStrict;
7453}
7454
7455
7456/**
7457 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7458 *
7459 * @param pVCpu The cross context virtual CPU structure.
7460 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7461 * out-of-sync. Make sure to update the required fields
7462 * before using them.
7463 */
7464DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7465{
7466 NOREF(pMixedCtx);
7467 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7468 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7469}
7470
7471
7472/**
7473 * Injects a double-fault (\#DF) exception into the VM.
7474 *
7475 * @returns Strict VBox status code (i.e. informational status codes too).
7476 * @param pVCpu The cross context virtual CPU structure.
7477 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7478 * out-of-sync. Make sure to update the required fields
7479 * before using them.
7480 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7481 * and should return VINF_EM_DBG_STEPPED if the event
7482 * is injected directly (register modified by us, not
7483 * by hardware on VM-entry).
7484 * @param pfIntrState Pointer to the current guest interruptibility-state.
7485 * This interruptibility-state will be updated if
7486 * necessary. This cannot not be NULL.
7487 */
7488DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCCPUMCTX pMixedCtx, bool fStepping, uint32_t *pfIntrState)
7489{
7490 NOREF(pMixedCtx);
7491 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7492 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7493 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7494 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */, fStepping,
7495 pfIntrState);
7496}
7497
7498
7499/**
7500 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7501 *
7502 * @param pVCpu The cross context virtual CPU structure.
7503 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7504 * out-of-sync. Make sure to update the required fields
7505 * before using them.
7506 */
7507DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7508{
7509 NOREF(pMixedCtx);
7510 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7511 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7512 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7513}
7514
7515
7516/**
7517 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7518 *
7519 * @param pVCpu The cross context virtual CPU structure.
7520 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7521 * out-of-sync. Make sure to update the required fields
7522 * before using them.
7523 * @param cbInstr The value of RIP that is to be pushed on the guest
7524 * stack.
7525 */
7526DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7527{
7528 NOREF(pMixedCtx);
7529 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7530 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7531 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7532}
7533
7534
7535/**
7536 * Injects a general-protection (\#GP) fault into the VM.
7537 *
7538 * @returns Strict VBox status code (i.e. informational status codes too).
7539 * @param pVCpu The cross context virtual CPU structure.
7540 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7541 * out-of-sync. Make sure to update the required fields
7542 * before using them.
7543 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7544 * mode, i.e. in real-mode it's not valid).
7545 * @param u32ErrorCode The error code associated with the \#GP.
7546 * @param fStepping Whether we're running in
7547 * hmR0VmxRunGuestCodeStep() and should return
7548 * VINF_EM_DBG_STEPPED if the event is injected
7549 * directly (register modified by us, not by
7550 * hardware on VM-entry).
7551 * @param pfIntrState Pointer to the current guest interruptibility-state.
7552 * This interruptibility-state will be updated if
7553 * necessary. This cannot not be NULL.
7554 */
7555DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7556 bool fStepping, uint32_t *pfIntrState)
7557{
7558 NOREF(pMixedCtx);
7559 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7560 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7561 if (fErrorCodeValid)
7562 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7563 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */, fStepping,
7564 pfIntrState);
7565}
7566
7567
7568#if 0 /* unused */
7569/**
7570 * Sets a general-protection (\#GP) exception as pending-for-injection into the
7571 * VM.
7572 *
7573 * @param pVCpu The cross context virtual CPU structure.
7574 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7575 * out-of-sync. Make sure to update the required fields
7576 * before using them.
7577 * @param u32ErrorCode The error code associated with the \#GP.
7578 */
7579DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7580{
7581 NOREF(pMixedCtx);
7582 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7583 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7584 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7585 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7586}
7587#endif /* unused */
7588
7589
7590/**
7591 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7592 *
7593 * @param pVCpu The cross context virtual CPU structure.
7594 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7595 * out-of-sync. Make sure to update the required fields
7596 * before using them.
7597 * @param uVector The software interrupt vector number.
7598 * @param cbInstr The value of RIP that is to be pushed on the guest
7599 * stack.
7600 */
7601DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7602{
7603 NOREF(pMixedCtx);
7604 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7605 if ( uVector == X86_XCPT_BP
7606 || uVector == X86_XCPT_OF)
7607 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7608 else
7609 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7610 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7611}
7612
7613
7614/**
7615 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7616 * stack.
7617 *
7618 * @returns Strict VBox status code (i.e. informational status codes too).
7619 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7620 * @param pVM The cross context VM structure.
7621 * @param pMixedCtx Pointer to the guest-CPU context.
7622 * @param uValue The value to push to the guest stack.
7623 */
7624DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7625{
7626 /*
7627 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7628 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7629 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7630 */
7631 if (pMixedCtx->sp == 1)
7632 return VINF_EM_RESET;
7633 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7634 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7635 AssertRC(rc);
7636 return rc;
7637}
7638
7639
7640/**
7641 * Injects an event into the guest upon VM-entry by updating the relevant fields
7642 * in the VM-entry area in the VMCS.
7643 *
7644 * @returns Strict VBox status code (i.e. informational status codes too).
7645 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7646 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7647 *
7648 * @param pVCpu The cross context virtual CPU structure.
7649 * @param u64IntInfo The VM-entry interruption-information field.
7650 * @param cbInstr The VM-entry instruction length in bytes (for
7651 * software interrupts, exceptions and privileged
7652 * software exceptions).
7653 * @param u32ErrCode The VM-entry exception error code.
7654 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7655 * @param pfIntrState Pointer to the current guest interruptibility-state.
7656 * This interruptibility-state will be updated if
7657 * necessary. This cannot not be NULL.
7658 * @param fStepping Whether we're running in
7659 * hmR0VmxRunGuestCodeStep() and should return
7660 * VINF_EM_DBG_STEPPED if the event is injected
7661 * directly (register modified by us, not by
7662 * hardware on VM-entry).
7663 */
7664static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7665 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState)
7666{
7667 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7668 AssertMsg(!RT_HI_U32(u64IntInfo), ("%#RX64\n", u64IntInfo));
7669 Assert(pfIntrState);
7670
7671 PCPUMCTX pMixedCtx = &pVCpu->cpum.GstCtx;
7672 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7673 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7674 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7675
7676#ifdef VBOX_STRICT
7677 /*
7678 * Validate the error-code-valid bit for hardware exceptions.
7679 * No error codes for exceptions in real-mode.
7680 *
7681 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7682 */
7683 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7684 && !CPUMIsGuestInRealModeEx(pMixedCtx))
7685 {
7686 switch (uVector)
7687 {
7688 case X86_XCPT_PF:
7689 case X86_XCPT_DF:
7690 case X86_XCPT_TS:
7691 case X86_XCPT_NP:
7692 case X86_XCPT_SS:
7693 case X86_XCPT_GP:
7694 case X86_XCPT_AC:
7695 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7696 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7697 RT_FALL_THRU();
7698 default:
7699 break;
7700 }
7701 }
7702#endif
7703
7704 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7705 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7706 || !(*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7707
7708 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7709
7710 /*
7711 * Hardware interrupts & exceptions cannot be delivered through the software interrupt
7712 * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
7713 * interrupt handler in the (real-mode) guest.
7714 *
7715 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
7716 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7717 */
7718 if (CPUMIsGuestInRealModeEx(pMixedCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
7719 {
7720 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
7721 {
7722 /*
7723 * For unrestricted execution enabled CPUs running real-mode guests, we must not
7724 * set the deliver-error-code bit.
7725 *
7726 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7727 */
7728 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7729 }
7730 else
7731 {
7732 PVM pVM = pVCpu->CTX_SUFF(pVM);
7733 Assert(PDMVmmDevHeapIsEnabled(pVM));
7734 Assert(pVM->hm.s.vmx.pRealModeTSS);
7735
7736 /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
7737 int rc2 = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK
7738 | CPUMCTX_EXTRN_TABLE_MASK
7739 | CPUMCTX_EXTRN_RIP
7740 | CPUMCTX_EXTRN_RSP
7741 | CPUMCTX_EXTRN_RFLAGS);
7742 AssertRCReturn(rc2, rc2);
7743
7744 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7745 size_t const cbIdtEntry = sizeof(X86IDTR16);
7746 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7747 {
7748 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7749 if (uVector == X86_XCPT_DF)
7750 return VINF_EM_RESET;
7751
7752 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7753 if (uVector == X86_XCPT_GP)
7754 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, pfIntrState);
7755
7756 /*
7757 * If we're injecting an event with no valid IDT entry, inject a #GP.
7758 * No error codes for exceptions in real-mode.
7759 *
7760 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7761 */
7762 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, fStepping,
7763 pfIntrState);
7764 }
7765
7766 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7767 uint16_t uGuestIp = pMixedCtx->ip;
7768 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7769 {
7770 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7771 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7772 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7773 }
7774 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7775 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7776
7777 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7778 X86IDTR16 IdtEntry;
7779 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7780 rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7781 AssertRCReturn(rc2, rc2);
7782
7783 /* Construct the stack frame for the interrupt/exception handler. */
7784 VBOXSTRICTRC rcStrict;
7785 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7786 if (rcStrict == VINF_SUCCESS)
7787 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7788 if (rcStrict == VINF_SUCCESS)
7789 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7790
7791 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7792 if (rcStrict == VINF_SUCCESS)
7793 {
7794 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7795 pMixedCtx->rip = IdtEntry.offSel;
7796 pMixedCtx->cs.Sel = IdtEntry.uSel;
7797 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7798 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7799 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7800 && uVector == X86_XCPT_PF)
7801 pMixedCtx->cr2 = GCPtrFaultAddress;
7802
7803 /* If any other guest-state bits are changed here, make sure to update
7804 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7805 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS
7806 | HM_CHANGED_GUEST_CR2
7807 | HM_CHANGED_GUEST_RIP
7808 | HM_CHANGED_GUEST_RFLAGS
7809 | HM_CHANGED_GUEST_RSP);
7810
7811 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7812 if (*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7813 {
7814 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7815 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7816 Log4Func(("Clearing inhibition due to STI\n"));
7817 *pfIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7818 }
7819 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7820 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
7821
7822 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7823 it, if we are returning to ring-3 before executing guest code. */
7824 pVCpu->hm.s.Event.fPending = false;
7825
7826 /* Make hmR0VmxPreRunGuest() return if we're stepping since we've changed cs:rip. */
7827 if (fStepping)
7828 rcStrict = VINF_EM_DBG_STEPPED;
7829 }
7830 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7831 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7832 return rcStrict;
7833 }
7834 }
7835
7836 /* Validate. */
7837 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7838 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7839 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7840
7841 /* Inject. */
7842 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7843 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7844 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7845 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7846 AssertRCReturn(rc, rc);
7847
7848 /* Update CR2. */
7849 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7850 && uVector == X86_XCPT_PF)
7851 pMixedCtx->cr2 = GCPtrFaultAddress;
7852
7853 Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7854
7855 return VINF_SUCCESS;
7856}
7857
7858
7859/**
7860 * Clears the interrupt-window exiting control in the VMCS and if necessary
7861 * clears the current event in the VMCS as well.
7862 *
7863 * @returns VBox status code.
7864 * @param pVCpu The cross context virtual CPU structure.
7865 *
7866 * @remarks Use this function only to clear events that have not yet been
7867 * delivered to the guest but are injected in the VMCS!
7868 * @remarks No-long-jump zone!!!
7869 */
7870static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
7871{
7872 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7873 {
7874 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7875 Log4Func(("Cleared interrupt widow\n"));
7876 }
7877
7878 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
7879 {
7880 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
7881 Log4Func(("Cleared interrupt widow\n"));
7882 }
7883}
7884
7885
7886/**
7887 * Enters the VT-x session.
7888 *
7889 * @returns VBox status code.
7890 * @param pVCpu The cross context virtual CPU structure.
7891 * @param pHostCpu Pointer to the global CPU info struct.
7892 */
7893VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu, PHMGLOBALCPUINFO pHostCpu)
7894{
7895 AssertPtr(pVCpu);
7896 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
7897 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7898 RT_NOREF(pHostCpu);
7899
7900 LogFlowFunc(("pVCpu=%p\n", pVCpu));
7901 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7902 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7903
7904#ifdef VBOX_STRICT
7905 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
7906 RTCCUINTREG uHostCR4 = ASMGetCR4();
7907 if (!(uHostCR4 & X86_CR4_VMXE))
7908 {
7909 LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
7910 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7911 }
7912#endif
7913
7914 /*
7915 * Load the VCPU's VMCS as the current (and active) one.
7916 */
7917 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7918 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7919 if (RT_FAILURE(rc))
7920 return rc;
7921
7922 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7923 pVCpu->hm.s.fLeaveDone = false;
7924 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7925
7926 return VINF_SUCCESS;
7927}
7928
7929
7930/**
7931 * The thread-context callback (only on platforms which support it).
7932 *
7933 * @param enmEvent The thread-context event.
7934 * @param pVCpu The cross context virtual CPU structure.
7935 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7936 * @thread EMT(pVCpu)
7937 */
7938VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7939{
7940 NOREF(fGlobalInit);
7941
7942 switch (enmEvent)
7943 {
7944 case RTTHREADCTXEVENT_OUT:
7945 {
7946 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7947 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7948 VMCPU_ASSERT_EMT(pVCpu);
7949
7950 /* No longjmps (logger flushes, locks) in this fragile context. */
7951 VMMRZCallRing3Disable(pVCpu);
7952 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7953
7954 /*
7955 * Restore host-state (FPU, debug etc.)
7956 */
7957 if (!pVCpu->hm.s.fLeaveDone)
7958 {
7959 /*
7960 * Do -not- import the guest-state here as we might already be in the middle of importing
7961 * it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
7962 */
7963 hmR0VmxLeave(pVCpu, false /* fImportState */);
7964 pVCpu->hm.s.fLeaveDone = true;
7965 }
7966
7967 /* Leave HM context, takes care of local init (term). */
7968 int rc = HMR0LeaveCpu(pVCpu);
7969 AssertRC(rc); NOREF(rc);
7970
7971 /* Restore longjmp state. */
7972 VMMRZCallRing3Enable(pVCpu);
7973 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
7974 break;
7975 }
7976
7977 case RTTHREADCTXEVENT_IN:
7978 {
7979 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7980 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7981 VMCPU_ASSERT_EMT(pVCpu);
7982
7983 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7984 VMMRZCallRing3Disable(pVCpu);
7985 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7986
7987 /* Initialize the bare minimum state required for HM. This takes care of
7988 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7989 int rc = hmR0EnterCpu(pVCpu);
7990 AssertRC(rc);
7991 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7992 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7993
7994 /* Load the active VMCS as the current one. */
7995 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7996 {
7997 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7998 AssertRC(rc); NOREF(rc);
7999 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8000 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8001 }
8002 pVCpu->hm.s.fLeaveDone = false;
8003
8004 /* Restore longjmp state. */
8005 VMMRZCallRing3Enable(pVCpu);
8006 break;
8007 }
8008
8009 default:
8010 break;
8011 }
8012}
8013
8014
8015/**
8016 * Exports the host state into the VMCS host-state area.
8017 * Sets up the VM-exit MSR-load area.
8018 *
8019 * The CPU state will be loaded from these fields on every successful VM-exit.
8020 *
8021 * @returns VBox status code.
8022 * @param pVCpu The cross context virtual CPU structure.
8023 *
8024 * @remarks No-long-jump zone!!!
8025 */
8026static int hmR0VmxExportHostState(PVMCPU pVCpu)
8027{
8028 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8029
8030 int rc = VINF_SUCCESS;
8031 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8032 {
8033 rc = hmR0VmxExportHostControlRegs();
8034 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8035
8036 rc = hmR0VmxExportHostSegmentRegs(pVCpu);
8037 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8038
8039 rc = hmR0VmxExportHostMsrs(pVCpu);
8040 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8041
8042 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
8043 }
8044 return rc;
8045}
8046
8047
8048/**
8049 * Saves the host state in the VMCS host-state.
8050 *
8051 * @returns VBox status code.
8052 * @param pVCpu The cross context virtual CPU structure.
8053 *
8054 * @remarks No-long-jump zone!!!
8055 */
8056VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
8057{
8058 AssertPtr(pVCpu);
8059 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8060
8061 /*
8062 * Export the host state here while entering HM context.
8063 * When thread-context hooks are used, we might get preempted and have to re-save the host
8064 * state but most of the time we won't be, so do it here before we disable interrupts.
8065 */
8066 return hmR0VmxExportHostState(pVCpu);
8067}
8068
8069
8070/**
8071 * Exports the guest state into the VMCS guest-state area.
8072 *
8073 * The will typically be done before VM-entry when the guest-CPU state and the
8074 * VMCS state may potentially be out of sync.
8075 *
8076 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8077 * VM-entry controls.
8078 * Sets up the appropriate VMX non-root function to execute guest code based on
8079 * the guest CPU mode.
8080 *
8081 * @returns VBox strict status code.
8082 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8083 * without unrestricted guest access and the VMMDev is not presently
8084 * mapped (e.g. EFI32).
8085 *
8086 * @param pVCpu The cross context virtual CPU structure.
8087 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8088 * out-of-sync. Make sure to update the required fields
8089 * before using them.
8090 *
8091 * @remarks No-long-jump zone!!!
8092 */
8093static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
8094{
8095 AssertPtr(pVCpu);
8096 AssertPtr(pMixedCtx);
8097 HMVMX_ASSERT_PREEMPT_SAFE();
8098
8099 LogFlowFunc(("pVCpu=%p\n", pVCpu));
8100
8101 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
8102
8103 /* Determine real-on-v86 mode. */
8104 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8105 if ( !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
8106 && CPUMIsGuestInRealModeEx(pMixedCtx))
8107 {
8108 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8109 }
8110
8111 /*
8112 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8113 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8114 */
8115 int rc = hmR0VmxSelectVMRunHandler(pVCpu, pMixedCtx);
8116 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8117
8118 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8119 rc = hmR0VmxExportGuestEntryCtls(pVCpu, pMixedCtx);
8120 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8121
8122 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8123 rc = hmR0VmxExportGuestExitCtls(pVCpu, pMixedCtx);
8124 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8125
8126 rc = hmR0VmxExportGuestCR0(pVCpu, pMixedCtx);
8127 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8128
8129 VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pMixedCtx);
8130 if (rcStrict == VINF_SUCCESS)
8131 { /* likely */ }
8132 else
8133 {
8134 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8135 return rcStrict;
8136 }
8137
8138 rc = hmR0VmxExportGuestSegmentRegs(pVCpu, pMixedCtx);
8139 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8140
8141 /* This needs to be done after hmR0VmxExportGuestEntryCtls() and hmR0VmxExportGuestExitCtls() as it
8142 may alter controls if we determine we don't have to swap EFER after all. */
8143 rc = hmR0VmxExportGuestMsrs(pVCpu, pMixedCtx);
8144 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8145
8146 rc = hmR0VmxExportGuestApicTpr(pVCpu);
8147 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8148
8149 /* This needs to be done after hmR0VmxExportGuestCR0() as it may alter intercepted exceptions. */
8150 rc = hmR0VmxExportGuestXcptIntercepts(pVCpu);
8151 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8152
8153 /* Exporting RFLAGS here is fine, even though RFLAGS.TF might depend on guest debug state which is
8154 not exported here. It is re-evaluated and updated if necessary in hmR0VmxExportSharedState(). */
8155 rc = hmR0VmxExportGuestRip(pVCpu, pMixedCtx);
8156 rc |= hmR0VmxExportGuestRsp(pVCpu, pMixedCtx);
8157 rc |= hmR0VmxExportGuestRflags(pVCpu, pMixedCtx);
8158 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8159
8160 /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
8161 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
8162 | HM_CHANGED_GUEST_CR2
8163 | (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
8164 | HM_CHANGED_GUEST_X87
8165 | HM_CHANGED_GUEST_SSE_AVX
8166 | HM_CHANGED_GUEST_OTHER_XSAVE
8167 | HM_CHANGED_GUEST_XCRx
8168 | HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
8169 | HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
8170 | HM_CHANGED_GUEST_TSC_AUX
8171 | HM_CHANGED_GUEST_OTHER_MSRS
8172 | HM_CHANGED_GUEST_HWVIRT
8173 | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
8174
8175 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
8176 return rc;
8177}
8178
8179
8180/**
8181 * Exports the state shared between the host and guest into the VMCS.
8182 *
8183 * @param pVCpu The cross context virtual CPU structure.
8184 * @param pCtx Pointer to the guest-CPU context.
8185 *
8186 * @remarks No-long-jump zone!!!
8187 */
8188static void hmR0VmxExportSharedState(PVMCPU pVCpu, PCPUMCTX pCtx)
8189{
8190 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8191 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8192
8193 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
8194 {
8195 int rc = hmR0VmxExportSharedDebugState(pVCpu, pCtx);
8196 AssertRC(rc);
8197 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
8198
8199 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8200 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
8201 {
8202 rc = hmR0VmxExportGuestRflags(pVCpu, pCtx);
8203 AssertRC(rc);
8204 }
8205 }
8206
8207 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
8208 {
8209 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8210 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
8211 }
8212
8213 AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
8214 ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8215}
8216
8217
8218/**
8219 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8220 *
8221 * @returns Strict VBox status code (i.e. informational status codes too).
8222 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8223 * without unrestricted guest access and the VMMDev is not presently
8224 * mapped (e.g. EFI32).
8225 *
8226 * @param pVCpu The cross context virtual CPU structure.
8227 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8228 * out-of-sync. Make sure to update the required fields
8229 * before using them.
8230 *
8231 * @remarks No-long-jump zone!!!
8232 */
8233static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
8234{
8235 HMVMX_ASSERT_PREEMPT_SAFE();
8236 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8237 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8238
8239#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8240 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8241#endif
8242
8243 /*
8244 * For many exits it's only RIP that changes and hence try to export it first
8245 * without going through a lot of change flag checks.
8246 */
8247 VBOXSTRICTRC rcStrict;
8248 uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8249 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8250 if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
8251 {
8252 rcStrict = hmR0VmxExportGuestRip(pVCpu, pMixedCtx);
8253 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8254 { /* likely */}
8255 else
8256 AssertMsgFailedReturn(("hmR0VmxExportGuestRip failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8257 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
8258 }
8259 else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8260 {
8261 rcStrict = hmR0VmxExportGuestState(pVCpu, pMixedCtx);
8262 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8263 { /* likely */}
8264 else
8265 {
8266 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("hmR0VmxExportGuestState failed! rc=%Rrc\n",
8267 VBOXSTRICTRC_VAL(rcStrict)));
8268 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8269 return rcStrict;
8270 }
8271 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
8272 }
8273 else
8274 rcStrict = VINF_SUCCESS;
8275
8276#ifdef VBOX_STRICT
8277 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8278 fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8279 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8280 AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
8281 ("fCtxChanged=%#RX64\n", fCtxChanged));
8282#endif
8283 return rcStrict;
8284}
8285
8286
8287/**
8288 * Does the preparations before executing guest code in VT-x.
8289 *
8290 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8291 * recompiler/IEM. We must be cautious what we do here regarding committing
8292 * guest-state information into the VMCS assuming we assuredly execute the
8293 * guest in VT-x mode.
8294 *
8295 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8296 * the common-state (TRPM/forceflags), we must undo those changes so that the
8297 * recompiler/IEM can (and should) use them when it resumes guest execution.
8298 * Otherwise such operations must be done when we can no longer exit to ring-3.
8299 *
8300 * @returns Strict VBox status code (i.e. informational status codes too).
8301 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8302 * have been disabled.
8303 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8304 * double-fault into the guest.
8305 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8306 * dispatched directly.
8307 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8308 *
8309 * @param pVCpu The cross context virtual CPU structure.
8310 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8311 * out-of-sync. Make sure to update the required fields
8312 * before using them.
8313 * @param pVmxTransient Pointer to the VMX transient structure.
8314 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8315 * us ignore some of the reasons for returning to
8316 * ring-3, and return VINF_EM_DBG_STEPPED if event
8317 * dispatching took place.
8318 */
8319static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8320{
8321 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8322
8323#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8324 PGMRZDynMapFlushAutoSet(pVCpu);
8325#endif
8326
8327 /* Check force flag actions that might require us to go back to ring-3. */
8328 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, pMixedCtx, fStepping);
8329 if (rcStrict == VINF_SUCCESS)
8330 { /* FFs doesn't get set all the time. */ }
8331 else
8332 return rcStrict;
8333
8334 /*
8335 * Setup the virtualized-APIC accesses.
8336 *
8337 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8338 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8339 *
8340 * This is the reason we do it here and not in hmR0VmxExportGuestState().
8341 */
8342 PVM pVM = pVCpu->CTX_SUFF(pVM);
8343 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8344 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8345 && PDMHasApic(pVM))
8346 {
8347 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8348 Assert(u64MsrApicBase);
8349 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8350
8351 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8352
8353 /* Unalias any existing mapping. */
8354 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8355 AssertRCReturn(rc, rc);
8356
8357 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8358 Log4Func(("Mapped HC APIC-access page at %#RGp\n", GCPhysApicBase));
8359 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8360 AssertRCReturn(rc, rc);
8361
8362 /* Update the per-VCPU cache of the APIC base MSR. */
8363 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8364 }
8365
8366 if (TRPMHasTrap(pVCpu))
8367 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8368 uint32_t fIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8369
8370 /*
8371 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
8372 * needs to be done with longjmps or interrupts + preemption enabled. Event injection might
8373 * also result in triple-faulting the VM.
8374 */
8375 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, fIntrState, fStepping);
8376 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8377 { /* likely */ }
8378 else
8379 {
8380 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8381 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8382 return rcStrict;
8383 }
8384
8385 /*
8386 * A longjump might result in importing CR3 even for VM-exits that don't necessarily
8387 * import CR3 themselves. We will need to update them here as even as late as the above
8388 * hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
8389 * the below force flags to be set.
8390 */
8391 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8392 {
8393 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
8394 int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8395 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
8396 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
8397 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8398 }
8399 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8400 {
8401 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8402 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8403 }
8404
8405 /*
8406 * No longjmps to ring-3 from this point on!!!
8407 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8408 * This also disables flushing of the R0-logger instance (if any).
8409 */
8410 VMMRZCallRing3Disable(pVCpu);
8411
8412 /*
8413 * Export the guest state bits.
8414 *
8415 * We cannot perform longjmps while loading the guest state because we do not preserve the
8416 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8417 * CPU migration.
8418 *
8419 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8420 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8421 * Hence, loading of the guest state needs to be done -after- injection of events.
8422 */
8423 rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pMixedCtx);
8424 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8425 { /* likely */ }
8426 else
8427 {
8428 VMMRZCallRing3Enable(pVCpu);
8429 return rcStrict;
8430 }
8431
8432 /*
8433 * We disable interrupts so that we don't miss any interrupts that would flag preemption
8434 * (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
8435 * preemption disabled for a while. Since this is purly to aid the
8436 * RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
8437 * disable interrupt on NT.
8438 *
8439 * We need to check for force-flags that could've possible been altered since we last
8440 * checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
8441 * see @bugref{6398}).
8442 *
8443 * We also check a couple of other force-flags as a last opportunity to get the EMT back
8444 * to ring-3 before executing guest code.
8445 */
8446 pVmxTransient->fEFlags = ASMIntDisableFlags();
8447
8448 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8449 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8450 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8451 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8452 {
8453 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8454 {
8455 pVCpu->hm.s.Event.fPending = false;
8456
8457 /*
8458 * We've injected any pending events. This is really the point of no return (to ring-3).
8459 *
8460 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
8461 * returns from this function, so don't enable them here.
8462 */
8463 return VINF_SUCCESS;
8464 }
8465
8466 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8467 rcStrict = VINF_EM_RAW_INTERRUPT;
8468 }
8469 else
8470 {
8471 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8472 rcStrict = VINF_EM_RAW_TO_R3;
8473 }
8474
8475 ASMSetFlags(pVmxTransient->fEFlags);
8476 VMMRZCallRing3Enable(pVCpu);
8477
8478 return rcStrict;
8479}
8480
8481
8482/**
8483 * Prepares to run guest code in VT-x and we've committed to doing so. This
8484 * means there is no backing out to ring-3 or anywhere else at this
8485 * point.
8486 *
8487 * @param pVCpu The cross context virtual CPU structure.
8488 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8489 * out-of-sync. Make sure to update the required fields
8490 * before using them.
8491 * @param pVmxTransient Pointer to the VMX transient structure.
8492 *
8493 * @remarks Called with preemption disabled.
8494 * @remarks No-long-jump zone!!!
8495 */
8496static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8497{
8498 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8499 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8500 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8501
8502 /*
8503 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8504 */
8505 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8506 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8507
8508 PVM pVM = pVCpu->CTX_SUFF(pVM);
8509 if (!CPUMIsGuestFPUStateActive(pVCpu))
8510 {
8511 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8512 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8513 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT;
8514 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8515 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
8516 }
8517
8518 /*
8519 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8520 */
8521 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8522 && pVCpu->hm.s.vmx.cMsrs > 0)
8523 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8524
8525 /*
8526 * Re-save the host state bits as we may've been preempted (only happens when
8527 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8528 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8529 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8530 * See @bugref{8432}.
8531 */
8532 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8533 {
8534 int rc = hmR0VmxExportHostState(pVCpu);
8535 AssertRC(rc);
8536 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
8537 }
8538 Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
8539
8540 /*
8541 * Export the state shared between host and guest (FPU, debug, lazy MSRs).
8542 */
8543 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
8544 hmR0VmxExportSharedState(pVCpu, pMixedCtx);
8545 AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8546
8547 /* Store status of the shared guest-host state at the time of VM-entry. */
8548#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8549 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8550 {
8551 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8552 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8553 }
8554 else
8555#endif
8556 {
8557 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8558 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8559 }
8560
8561 /*
8562 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8563 */
8564 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8565 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8566
8567 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8568 RTCPUID idCurrentCpu = pCpu->idCpu;
8569 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8570 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8571 {
8572 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8573 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8574 }
8575
8576 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8577 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8578 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8579 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8580
8581 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8582
8583 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8584 to start executing. */
8585
8586 /*
8587 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8588 */
8589 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8590 {
8591 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8592 {
8593 bool fMsrUpdated;
8594 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
8595 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8596 &fMsrUpdated);
8597 AssertRC(rc2);
8598 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8599 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8600 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8601 }
8602 else
8603 {
8604 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8605 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8606 }
8607 }
8608
8609 if (pVM->cpum.ro.GuestFeatures.fIbrs)
8610 {
8611 bool fMsrUpdated;
8612 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_OTHER_MSRS);
8613 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
8614 &fMsrUpdated);
8615 AssertRC(rc2);
8616 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8617 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8618 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8619 }
8620
8621#ifdef VBOX_STRICT
8622 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8623 hmR0VmxCheckHostEferMsr(pVCpu);
8624 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8625#endif
8626#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8627 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
8628 {
8629 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pMixedCtx);
8630 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8631 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8632 }
8633#endif
8634}
8635
8636
8637/**
8638 * Performs some essential restoration of state after running guest code in
8639 * VT-x.
8640 *
8641 * @param pVCpu The cross context virtual CPU structure.
8642 * @param pVmxTransient Pointer to the VMX transient structure.
8643 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8644 *
8645 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8646 *
8647 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8648 * unconditionally when it is safe to do so.
8649 */
8650static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8651{
8652 uint64_t const uHostTsc = ASMReadTSC();
8653 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8654
8655 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8656 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8657 pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8658 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8659 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8660 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8661
8662 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8663 TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TscOffset);
8664
8665 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
8666 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8667 Assert(!ASMIntAreEnabled());
8668 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8669
8670#if HC_ARCH_BITS == 64
8671 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8672#endif
8673#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8674 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8675 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8676 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8677#else
8678 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8679#endif
8680#ifdef VBOX_STRICT
8681 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8682#endif
8683 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8684
8685 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8686 uint32_t uExitReason;
8687 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8688 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8689 AssertRC(rc);
8690 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8691 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8692
8693 if (rcVMRun == VINF_SUCCESS)
8694 {
8695 /*
8696 * Update the VM-exit history array here even if the VM-entry failed due to:
8697 * - Invalid guest state.
8698 * - MSR loading.
8699 * - Machine-check event.
8700 *
8701 * In any of the above cases we will still have a "valid" VM-exit reason
8702 * despite @a fVMEntryFailed being false.
8703 *
8704 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8705 *
8706 * Note! We don't have CS or RIP at this point. Will probably address that later
8707 * by amending the history entry added here.
8708 */
8709 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
8710 UINT64_MAX, uHostTsc);
8711
8712 if (!pVmxTransient->fVMEntryFailed)
8713 {
8714 VMMRZCallRing3Enable(pVCpu);
8715
8716 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8717 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8718
8719#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8720 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
8721 AssertRC(rc);
8722#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8723 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_RFLAGS);
8724 AssertRC(rc);
8725#else
8726 /*
8727 * Import the guest-interruptibility state always as we need it while evaluating
8728 * injecting events on re-entry.
8729 *
8730 * We don't import CR0 (when Unrestricted guest execution is unavailable) despite
8731 * checking for real-mode while exporting the state because all bits that cause
8732 * mode changes wrt CR0 are intercepted.
8733 */
8734 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
8735 AssertRC(rc);
8736#endif
8737
8738 /*
8739 * Sync the TPR shadow with our APIC state.
8740 */
8741 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8742 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8743 {
8744 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8745 AssertRC(rc);
8746 ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8747 }
8748
8749 return;
8750 }
8751 }
8752 else
8753 {
8754 Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
8755 }
8756
8757 VMMRZCallRing3Enable(pVCpu);
8758}
8759
8760
8761/**
8762 * Runs the guest code using VT-x the normal way.
8763 *
8764 * @returns VBox status code.
8765 * @param pVCpu The cross context virtual CPU structure.
8766 * @param pCtx Pointer to the guest-CPU context.
8767 *
8768 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8769 */
8770static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, PCPUMCTX pCtx)
8771{
8772 VMXTRANSIENT VmxTransient;
8773 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8774 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8775 uint32_t cLoops = 0;
8776
8777 for (;; cLoops++)
8778 {
8779 Assert(!HMR0SuspendPending());
8780 HMVMX_ASSERT_CPU_SAFE();
8781
8782 /* Preparatory work for running guest code, this may force us to return
8783 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8784 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8785 rcStrict = hmR0VmxPreRunGuest(pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8786 if (rcStrict != VINF_SUCCESS)
8787 break;
8788
8789 hmR0VmxPreRunGuestCommitted(pVCpu, pCtx, &VmxTransient);
8790 int rcRun = hmR0VmxRunGuest(pVCpu, pCtx);
8791 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8792
8793 /* Restore any residual host-state and save any bits shared between host
8794 and guest into the guest-CPU state. Re-enables interrupts! */
8795 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
8796
8797 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8798 if (RT_SUCCESS(rcRun))
8799 { /* very likely */ }
8800 else
8801 {
8802 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
8803 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, pCtx, &VmxTransient);
8804 return rcRun;
8805 }
8806
8807 /* Profile the VM-exit. */
8808 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8809 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8810 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8811 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
8812 HMVMX_START_EXIT_DISPATCH_PROF();
8813
8814 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8815
8816 /* Handle the VM-exit. */
8817#ifdef HMVMX_USE_FUNCTION_TABLE
8818 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8819#else
8820 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8821#endif
8822 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
8823 if (rcStrict == VINF_SUCCESS)
8824 {
8825 if (cLoops <= pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
8826 continue; /* likely */
8827 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8828 rcStrict = VINF_EM_RAW_INTERRUPT;
8829 }
8830 break;
8831 }
8832
8833 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8834 return rcStrict;
8835}
8836
8837
8838
8839/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8840 * probes.
8841 *
8842 * The following few functions and associated structure contains the bloat
8843 * necessary for providing detailed debug events and dtrace probes as well as
8844 * reliable host side single stepping. This works on the principle of
8845 * "subclassing" the normal execution loop and workers. We replace the loop
8846 * method completely and override selected helpers to add necessary adjustments
8847 * to their core operation.
8848 *
8849 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8850 * any performance for debug and analysis features.
8851 *
8852 * @{
8853 */
8854
8855/**
8856 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
8857 * the debug run loop.
8858 */
8859typedef struct VMXRUNDBGSTATE
8860{
8861 /** The RIP we started executing at. This is for detecting that we stepped. */
8862 uint64_t uRipStart;
8863 /** The CS we started executing with. */
8864 uint16_t uCsStart;
8865
8866 /** Whether we've actually modified the 1st execution control field. */
8867 bool fModifiedProcCtls : 1;
8868 /** Whether we've actually modified the 2nd execution control field. */
8869 bool fModifiedProcCtls2 : 1;
8870 /** Whether we've actually modified the exception bitmap. */
8871 bool fModifiedXcptBitmap : 1;
8872
8873 /** We desire the modified the CR0 mask to be cleared. */
8874 bool fClearCr0Mask : 1;
8875 /** We desire the modified the CR4 mask to be cleared. */
8876 bool fClearCr4Mask : 1;
8877 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
8878 uint32_t fCpe1Extra;
8879 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
8880 uint32_t fCpe1Unwanted;
8881 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
8882 uint32_t fCpe2Extra;
8883 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
8884 uint32_t bmXcptExtra;
8885 /** The sequence number of the Dtrace provider settings the state was
8886 * configured against. */
8887 uint32_t uDtraceSettingsSeqNo;
8888 /** VM-exits to check (one bit per VM-exit). */
8889 uint32_t bmExitsToCheck[3];
8890
8891 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
8892 uint32_t fProcCtlsInitial;
8893 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
8894 uint32_t fProcCtls2Initial;
8895 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
8896 uint32_t bmXcptInitial;
8897} VMXRUNDBGSTATE;
8898AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
8899typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
8900
8901
8902/**
8903 * Initializes the VMXRUNDBGSTATE structure.
8904 *
8905 * @param pVCpu The cross context virtual CPU structure of the
8906 * calling EMT.
8907 * @param pCtx The CPU register context to go with @a pVCpu.
8908 * @param pDbgState The structure to initialize.
8909 */
8910static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
8911{
8912 pDbgState->uRipStart = pCtx->rip;
8913 pDbgState->uCsStart = pCtx->cs.Sel;
8914
8915 pDbgState->fModifiedProcCtls = false;
8916 pDbgState->fModifiedProcCtls2 = false;
8917 pDbgState->fModifiedXcptBitmap = false;
8918 pDbgState->fClearCr0Mask = false;
8919 pDbgState->fClearCr4Mask = false;
8920 pDbgState->fCpe1Extra = 0;
8921 pDbgState->fCpe1Unwanted = 0;
8922 pDbgState->fCpe2Extra = 0;
8923 pDbgState->bmXcptExtra = 0;
8924 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
8925 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
8926 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
8927}
8928
8929
8930/**
8931 * Updates the VMSC fields with changes requested by @a pDbgState.
8932 *
8933 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
8934 * immediately before executing guest code, i.e. when interrupts are disabled.
8935 * We don't check status codes here as we cannot easily assert or return in the
8936 * latter case.
8937 *
8938 * @param pVCpu The cross context virtual CPU structure.
8939 * @param pDbgState The debug state.
8940 */
8941static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
8942{
8943 /*
8944 * Ensure desired flags in VMCS control fields are set.
8945 * (Ignoring write failure here, as we're committed and it's just debug extras.)
8946 *
8947 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
8948 * there should be no stale data in pCtx at this point.
8949 */
8950 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
8951 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
8952 {
8953 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
8954 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
8955 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8956 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
8957 pDbgState->fModifiedProcCtls = true;
8958 }
8959
8960 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
8961 {
8962 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
8963 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
8964 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
8965 pDbgState->fModifiedProcCtls2 = true;
8966 }
8967
8968 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
8969 {
8970 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
8971 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8972 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
8973 pDbgState->fModifiedXcptBitmap = true;
8974 }
8975
8976 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32Cr0Mask != 0)
8977 {
8978 pVCpu->hm.s.vmx.u32Cr0Mask = 0;
8979 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
8980 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
8981 }
8982
8983 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32Cr4Mask != 0)
8984 {
8985 pVCpu->hm.s.vmx.u32Cr4Mask = 0;
8986 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
8987 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
8988 }
8989}
8990
8991
8992static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
8993{
8994 /*
8995 * Restore VM-exit control settings as we may not reenter this function the
8996 * next time around.
8997 */
8998 /* We reload the initial value, trigger what we can of recalculations the
8999 next time around. From the looks of things, that's all that's required atm. */
9000 if (pDbgState->fModifiedProcCtls)
9001 {
9002 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9003 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9004 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9005 AssertRCReturn(rc2, rc2);
9006 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9007 }
9008
9009 /* We're currently the only ones messing with this one, so just restore the
9010 cached value and reload the field. */
9011 if ( pDbgState->fModifiedProcCtls2
9012 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9013 {
9014 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9015 AssertRCReturn(rc2, rc2);
9016 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9017 }
9018
9019 /* If we've modified the exception bitmap, we restore it and trigger
9020 reloading and partial recalculation the next time around. */
9021 if (pDbgState->fModifiedXcptBitmap)
9022 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9023
9024 return rcStrict;
9025}
9026
9027
9028/**
9029 * Configures VM-exit controls for current DBGF and DTrace settings.
9030 *
9031 * This updates @a pDbgState and the VMCS execution control fields to reflect
9032 * the necessary VM-exits demanded by DBGF and DTrace.
9033 *
9034 * @param pVCpu The cross context virtual CPU structure.
9035 * @param pDbgState The debug state.
9036 * @param pVmxTransient Pointer to the VMX transient structure. May update
9037 * fUpdateTscOffsettingAndPreemptTimer.
9038 */
9039static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9040{
9041 /*
9042 * Take down the dtrace serial number so we can spot changes.
9043 */
9044 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9045 ASMCompilerBarrier();
9046
9047 /*
9048 * We'll rebuild most of the middle block of data members (holding the
9049 * current settings) as we go along here, so start by clearing it all.
9050 */
9051 pDbgState->bmXcptExtra = 0;
9052 pDbgState->fCpe1Extra = 0;
9053 pDbgState->fCpe1Unwanted = 0;
9054 pDbgState->fCpe2Extra = 0;
9055 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9056 pDbgState->bmExitsToCheck[i] = 0;
9057
9058 /*
9059 * Software interrupts (INT XXh) - no idea how to trigger these...
9060 */
9061 PVM pVM = pVCpu->CTX_SUFF(pVM);
9062 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9063 || VBOXVMM_INT_SOFTWARE_ENABLED())
9064 {
9065 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9066 }
9067
9068 /*
9069 * INT3 breakpoints - triggered by #BP exceptions.
9070 */
9071 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9072 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9073
9074 /*
9075 * Exception bitmap and XCPT events+probes.
9076 */
9077 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9078 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9079 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9080
9081 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9082 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9083 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9084 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9085 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9086 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9087 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9088 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9089 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9090 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9091 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9092 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9093 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9094 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9095 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9096 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9097 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9098 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9099
9100 if (pDbgState->bmXcptExtra)
9101 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9102
9103 /*
9104 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9105 *
9106 * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
9107 * So, when adding/changing/removing please don't forget to update it.
9108 *
9109 * Some of the macros are picking up local variables to save horizontal space,
9110 * (being able to see it in a table is the lesser evil here).
9111 */
9112#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9113 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9114 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9115#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9116 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9117 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9118 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9119 } else do { } while (0)
9120#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9121 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9122 { \
9123 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9124 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9125 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9126 } else do { } while (0)
9127#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9128 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9129 { \
9130 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9131 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9132 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9133 } else do { } while (0)
9134#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9135 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9136 { \
9137 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9138 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9139 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9140 } else do { } while (0)
9141
9142 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9143 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9144 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9145 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9146 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9147
9148 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9149 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9150 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9151 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9152 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9153 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9154 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9155 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9156 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9157 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9158 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9159 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9160 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9161 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9162 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9163 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9164 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9165 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9166 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9167 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9168 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9169 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9170 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9171 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9172 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9173 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9174 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9175 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9176 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9177 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9178 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9179 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9180 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9181 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9182 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9183 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9184
9185 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9186 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9187 {
9188 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
9189 | CPUMCTX_EXTRN_CR4
9190 | CPUMCTX_EXTRN_APIC_TPR);
9191 AssertRC(rc);
9192
9193#if 0 /** @todo fix me */
9194 pDbgState->fClearCr0Mask = true;
9195 pDbgState->fClearCr4Mask = true;
9196#endif
9197 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9198 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9199 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9200 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9201 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9202 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9203 require clearing here and in the loop if we start using it. */
9204 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9205 }
9206 else
9207 {
9208 if (pDbgState->fClearCr0Mask)
9209 {
9210 pDbgState->fClearCr0Mask = false;
9211 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
9212 }
9213 if (pDbgState->fClearCr4Mask)
9214 {
9215 pDbgState->fClearCr4Mask = false;
9216 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
9217 }
9218 }
9219 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9220 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9221
9222 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9223 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9224 {
9225 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9226 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9227 }
9228 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9229 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9230
9231 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9232 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9233 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9234 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9235 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9236 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9237 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9238 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9239#if 0 /** @todo too slow, fix handler. */
9240 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9241#endif
9242 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9243
9244 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9245 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9246 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9247 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9248 {
9249 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9250 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9251 }
9252 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9253 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9254 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9255 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9256
9257 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9258 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9259 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9260 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9261 {
9262 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9263 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9264 }
9265 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9266 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9267 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9268 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9269
9270 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9271 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9272 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9273 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9274 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9275 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9276 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9277 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9278 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9279 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9280 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9281 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9282 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9283 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9284 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9285 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9286 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9287 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9288 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9289 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9290 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9291 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9292
9293#undef IS_EITHER_ENABLED
9294#undef SET_ONLY_XBM_IF_EITHER_EN
9295#undef SET_CPE1_XBM_IF_EITHER_EN
9296#undef SET_CPEU_XBM_IF_EITHER_EN
9297#undef SET_CPE2_XBM_IF_EITHER_EN
9298
9299 /*
9300 * Sanitize the control stuff.
9301 */
9302 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9303 if (pDbgState->fCpe2Extra)
9304 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9305 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9306 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9307 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9308 {
9309 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9310 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9311 }
9312
9313 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9314 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9315 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9316 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9317}
9318
9319
9320/**
9321 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9322 * appropriate.
9323 *
9324 * The caller has checked the VM-exit against the
9325 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9326 * already, so we don't have to do that either.
9327 *
9328 * @returns Strict VBox status code (i.e. informational status codes too).
9329 * @param pVCpu The cross context virtual CPU structure.
9330 * @param pMixedCtx Pointer to the guest-CPU context.
9331 * @param pVmxTransient Pointer to the VMX-transient structure.
9332 * @param uExitReason The VM-exit reason.
9333 *
9334 * @remarks The name of this function is displayed by dtrace, so keep it short
9335 * and to the point. No longer than 33 chars long, please.
9336 */
9337static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9338 uint32_t uExitReason)
9339{
9340 /*
9341 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9342 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9343 *
9344 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9345 * does. Must add/change/remove both places. Same ordering, please.
9346 *
9347 * Added/removed events must also be reflected in the next section
9348 * where we dispatch dtrace events.
9349 */
9350 bool fDtrace1 = false;
9351 bool fDtrace2 = false;
9352 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9353 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9354 uint32_t uEventArg = 0;
9355#define SET_EXIT(a_EventSubName) \
9356 do { \
9357 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9358 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9359 } while (0)
9360#define SET_BOTH(a_EventSubName) \
9361 do { \
9362 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9363 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9364 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9365 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9366 } while (0)
9367 switch (uExitReason)
9368 {
9369 case VMX_EXIT_MTF:
9370 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9371
9372 case VMX_EXIT_XCPT_OR_NMI:
9373 {
9374 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9375 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9376 {
9377 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9378 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9379 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9380 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9381 {
9382 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9383 {
9384 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9385 uEventArg = pVmxTransient->uExitIntErrorCode;
9386 }
9387 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9388 switch (enmEvent1)
9389 {
9390 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9391 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9392 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9393 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9394 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9395 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9396 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9397 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9398 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9399 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9400 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9401 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9402 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9403 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9404 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9405 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9406 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9407 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9408 default: break;
9409 }
9410 }
9411 else
9412 AssertFailed();
9413 break;
9414
9415 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9416 uEventArg = idxVector;
9417 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9418 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9419 break;
9420 }
9421 break;
9422 }
9423
9424 case VMX_EXIT_TRIPLE_FAULT:
9425 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9426 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9427 break;
9428 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9429 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9430 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9431 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9432 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9433
9434 /* Instruction specific VM-exits: */
9435 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9436 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9437 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9438 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9439 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9440 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9441 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9442 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9443 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9444 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9445 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9446 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9447 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9448 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9449 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9450 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9451 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9452 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9453 case VMX_EXIT_MOV_CRX:
9454 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9455 if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
9456 SET_BOTH(CRX_READ);
9457 else
9458 SET_BOTH(CRX_WRITE);
9459 uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQualification);
9460 break;
9461 case VMX_EXIT_MOV_DRX:
9462 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9463 if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification)
9464 == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
9465 SET_BOTH(DRX_READ);
9466 else
9467 SET_BOTH(DRX_WRITE);
9468 uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification);
9469 break;
9470 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9471 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9472 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9473 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9474 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9475 case VMX_EXIT_XDTR_ACCESS:
9476 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9477 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9478 {
9479 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9480 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9481 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9482 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9483 }
9484 break;
9485
9486 case VMX_EXIT_TR_ACCESS:
9487 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9488 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9489 {
9490 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9491 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9492 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9493 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9494 }
9495 break;
9496
9497 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9498 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9499 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9500 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9501 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9502 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9503 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9504 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9505 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9506 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9507 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9508
9509 /* Events that aren't relevant at this point. */
9510 case VMX_EXIT_EXT_INT:
9511 case VMX_EXIT_INT_WINDOW:
9512 case VMX_EXIT_NMI_WINDOW:
9513 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9514 case VMX_EXIT_PREEMPT_TIMER:
9515 case VMX_EXIT_IO_INSTR:
9516 break;
9517
9518 /* Errors and unexpected events. */
9519 case VMX_EXIT_INIT_SIGNAL:
9520 case VMX_EXIT_SIPI:
9521 case VMX_EXIT_IO_SMI:
9522 case VMX_EXIT_SMI:
9523 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9524 case VMX_EXIT_ERR_MSR_LOAD:
9525 case VMX_EXIT_ERR_MACHINE_CHECK:
9526 break;
9527
9528 default:
9529 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9530 break;
9531 }
9532#undef SET_BOTH
9533#undef SET_EXIT
9534
9535 /*
9536 * Dtrace tracepoints go first. We do them here at once so we don't
9537 * have to copy the guest state saving and stuff a few dozen times.
9538 * Down side is that we've got to repeat the switch, though this time
9539 * we use enmEvent since the probes are a subset of what DBGF does.
9540 */
9541 if (fDtrace1 || fDtrace2)
9542 {
9543 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9544 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9545 switch (enmEvent1)
9546 {
9547 /** @todo consider which extra parameters would be helpful for each probe. */
9548 case DBGFEVENT_END: break;
9549 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9550 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9551 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9552 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9553 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9554 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9555 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9556 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9557 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9558 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9559 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9560 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9561 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9562 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9563 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9564 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9565 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9566 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9567 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9568 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9569 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9570 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9571 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9572 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9573 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9574 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9575 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9576 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9577 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9578 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9579 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9580 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9581 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9582 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9583 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9584 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9585 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9586 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9587 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9588 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9589 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9590 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9591 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9592 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9593 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9594 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9595 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9596 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9597 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9598 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9599 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9600 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9601 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9602 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9603 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9604 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9605 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9606 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9607 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9608 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9609 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9610 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9611 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9612 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9613 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9614 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9615 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9616 }
9617 switch (enmEvent2)
9618 {
9619 /** @todo consider which extra parameters would be helpful for each probe. */
9620 case DBGFEVENT_END: break;
9621 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9622 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9623 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9624 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9625 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9626 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9627 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9628 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9629 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9630 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9631 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9632 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9633 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9634 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9635 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9636 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9637 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9638 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9639 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9640 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9641 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9642 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9643 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9644 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9645 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9646 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9647 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9648 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9649 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9650 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9651 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9652 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9653 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9654 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9655 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9656 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9657 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9658 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9659 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9660 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9661 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9662 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9663 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9664 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9665 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9666 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9667 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9668 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9669 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9670 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9671 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9672 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9673 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9674 }
9675 }
9676
9677 /*
9678 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9679 * the DBGF call will do a full check).
9680 *
9681 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9682 * Note! If we have to events, we prioritize the first, i.e. the instruction
9683 * one, in order to avoid event nesting.
9684 */
9685 PVM pVM = pVCpu->CTX_SUFF(pVM);
9686 if ( enmEvent1 != DBGFEVENT_END
9687 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9688 {
9689 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9690 if (rcStrict != VINF_SUCCESS)
9691 return rcStrict;
9692 }
9693 else if ( enmEvent2 != DBGFEVENT_END
9694 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9695 {
9696 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9697 if (rcStrict != VINF_SUCCESS)
9698 return rcStrict;
9699 }
9700
9701 return VINF_SUCCESS;
9702}
9703
9704
9705/**
9706 * Single-stepping VM-exit filtering.
9707 *
9708 * This is preprocessing the VM-exits and deciding whether we've gotten far
9709 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9710 * handling is performed.
9711 *
9712 * @returns Strict VBox status code (i.e. informational status codes too).
9713 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9714 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9715 * out-of-sync. Make sure to update the required
9716 * fields before using them.
9717 * @param pVmxTransient Pointer to the VMX-transient structure.
9718 * @param uExitReason The VM-exit reason.
9719 * @param pDbgState The debug state.
9720 */
9721DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9722 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9723{
9724 /*
9725 * Expensive (saves context) generic dtrace VM-exit probe.
9726 */
9727 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9728 { /* more likely */ }
9729 else
9730 {
9731 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9732 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9733 AssertRC(rc);
9734 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9735 }
9736
9737 /*
9738 * Check for host NMI, just to get that out of the way.
9739 */
9740 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9741 { /* normally likely */ }
9742 else
9743 {
9744 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9745 AssertRCReturn(rc2, rc2);
9746 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9747 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9748 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9749 }
9750
9751 /*
9752 * Check for single stepping event if we're stepping.
9753 */
9754 if (pVCpu->hm.s.fSingleInstruction)
9755 {
9756 switch (uExitReason)
9757 {
9758 case VMX_EXIT_MTF:
9759 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9760
9761 /* Various events: */
9762 case VMX_EXIT_XCPT_OR_NMI:
9763 case VMX_EXIT_EXT_INT:
9764 case VMX_EXIT_TRIPLE_FAULT:
9765 case VMX_EXIT_INT_WINDOW:
9766 case VMX_EXIT_NMI_WINDOW:
9767 case VMX_EXIT_TASK_SWITCH:
9768 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9769 case VMX_EXIT_APIC_ACCESS:
9770 case VMX_EXIT_EPT_VIOLATION:
9771 case VMX_EXIT_EPT_MISCONFIG:
9772 case VMX_EXIT_PREEMPT_TIMER:
9773
9774 /* Instruction specific VM-exits: */
9775 case VMX_EXIT_CPUID:
9776 case VMX_EXIT_GETSEC:
9777 case VMX_EXIT_HLT:
9778 case VMX_EXIT_INVD:
9779 case VMX_EXIT_INVLPG:
9780 case VMX_EXIT_RDPMC:
9781 case VMX_EXIT_RDTSC:
9782 case VMX_EXIT_RSM:
9783 case VMX_EXIT_VMCALL:
9784 case VMX_EXIT_VMCLEAR:
9785 case VMX_EXIT_VMLAUNCH:
9786 case VMX_EXIT_VMPTRLD:
9787 case VMX_EXIT_VMPTRST:
9788 case VMX_EXIT_VMREAD:
9789 case VMX_EXIT_VMRESUME:
9790 case VMX_EXIT_VMWRITE:
9791 case VMX_EXIT_VMXOFF:
9792 case VMX_EXIT_VMXON:
9793 case VMX_EXIT_MOV_CRX:
9794 case VMX_EXIT_MOV_DRX:
9795 case VMX_EXIT_IO_INSTR:
9796 case VMX_EXIT_RDMSR:
9797 case VMX_EXIT_WRMSR:
9798 case VMX_EXIT_MWAIT:
9799 case VMX_EXIT_MONITOR:
9800 case VMX_EXIT_PAUSE:
9801 case VMX_EXIT_XDTR_ACCESS:
9802 case VMX_EXIT_TR_ACCESS:
9803 case VMX_EXIT_INVEPT:
9804 case VMX_EXIT_RDTSCP:
9805 case VMX_EXIT_INVVPID:
9806 case VMX_EXIT_WBINVD:
9807 case VMX_EXIT_XSETBV:
9808 case VMX_EXIT_RDRAND:
9809 case VMX_EXIT_INVPCID:
9810 case VMX_EXIT_VMFUNC:
9811 case VMX_EXIT_RDSEED:
9812 case VMX_EXIT_XSAVES:
9813 case VMX_EXIT_XRSTORS:
9814 {
9815 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
9816 | CPUMCTX_EXTRN_CS);
9817 AssertRCReturn(rc, rc);
9818 if ( pMixedCtx->rip != pDbgState->uRipStart
9819 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
9820 return VINF_EM_DBG_STEPPED;
9821 break;
9822 }
9823
9824 /* Errors and unexpected events: */
9825 case VMX_EXIT_INIT_SIGNAL:
9826 case VMX_EXIT_SIPI:
9827 case VMX_EXIT_IO_SMI:
9828 case VMX_EXIT_SMI:
9829 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9830 case VMX_EXIT_ERR_MSR_LOAD:
9831 case VMX_EXIT_ERR_MACHINE_CHECK:
9832 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9833 break;
9834
9835 default:
9836 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9837 break;
9838 }
9839 }
9840
9841 /*
9842 * Check for debugger event breakpoints and dtrace probes.
9843 */
9844 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9845 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9846 {
9847 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9848 if (rcStrict != VINF_SUCCESS)
9849 return rcStrict;
9850 }
9851
9852 /*
9853 * Normal processing.
9854 */
9855#ifdef HMVMX_USE_FUNCTION_TABLE
9856 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9857#else
9858 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9859#endif
9860}
9861
9862
9863/**
9864 * Single steps guest code using VT-x.
9865 *
9866 * @returns Strict VBox status code (i.e. informational status codes too).
9867 * @param pVCpu The cross context virtual CPU structure.
9868 * @param pCtx Pointer to the guest-CPU context.
9869 *
9870 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
9871 */
9872static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, PCPUMCTX pCtx)
9873{
9874 VMXTRANSIENT VmxTransient;
9875 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9876
9877 /* Set HMCPU indicators. */
9878 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
9879 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
9880 pVCpu->hm.s.fDebugWantRdTscExit = false;
9881 pVCpu->hm.s.fUsingDebugLoop = true;
9882
9883 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
9884 VMXRUNDBGSTATE DbgState;
9885 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
9886 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9887
9888 /*
9889 * The loop.
9890 */
9891 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9892 for (uint32_t cLoops = 0; ; cLoops++)
9893 {
9894 Assert(!HMR0SuspendPending());
9895 HMVMX_ASSERT_CPU_SAFE();
9896 bool fStepping = pVCpu->hm.s.fSingleInstruction;
9897
9898 /*
9899 * Preparatory work for running guest code, this may force us to return
9900 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
9901 */
9902 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9903 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
9904 rcStrict = hmR0VmxPreRunGuest(pVCpu, pCtx, &VmxTransient, fStepping);
9905 if (rcStrict != VINF_SUCCESS)
9906 break;
9907
9908 hmR0VmxPreRunGuestCommitted(pVCpu, pCtx, &VmxTransient);
9909 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
9910
9911 /*
9912 * Now we can run the guest code.
9913 */
9914 int rcRun = hmR0VmxRunGuest(pVCpu, pCtx);
9915
9916 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9917
9918 /*
9919 * Restore any residual host-state and save any bits shared between host
9920 * and guest into the guest-CPU state. Re-enables interrupts!
9921 */
9922 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
9923
9924 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9925 if (RT_SUCCESS(rcRun))
9926 { /* very likely */ }
9927 else
9928 {
9929 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
9930 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, pCtx, &VmxTransient);
9931 return rcRun;
9932 }
9933
9934 /* Profile the VM-exit. */
9935 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9936 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9937 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9938 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
9939 HMVMX_START_EXIT_DISPATCH_PROF();
9940
9941 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9942
9943 /*
9944 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
9945 */
9946 rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
9947 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
9948 if (rcStrict != VINF_SUCCESS)
9949 break;
9950 if (cLoops > pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
9951 {
9952 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9953 rcStrict = VINF_EM_RAW_INTERRUPT;
9954 break;
9955 }
9956
9957 /*
9958 * Stepping: Did the RIP change, if so, consider it a single step.
9959 * Otherwise, make sure one of the TFs gets set.
9960 */
9961 if (fStepping)
9962 {
9963 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
9964 | CPUMCTX_EXTRN_CS);
9965 AssertRC(rc);
9966 if ( pCtx->rip != DbgState.uRipStart
9967 || pCtx->cs.Sel != DbgState.uCsStart)
9968 {
9969 rcStrict = VINF_EM_DBG_STEPPED;
9970 break;
9971 }
9972 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
9973 }
9974
9975 /*
9976 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
9977 */
9978 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
9979 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9980 }
9981
9982 /*
9983 * Clear the X86_EFL_TF if necessary.
9984 */
9985 if (pVCpu->hm.s.fClearTrapFlag)
9986 {
9987 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
9988 AssertRC(rc);
9989 pVCpu->hm.s.fClearTrapFlag = false;
9990 pCtx->eflags.Bits.u1TF = 0;
9991 }
9992 /** @todo there seems to be issues with the resume flag when the monitor trap
9993 * flag is pending without being used. Seen early in bios init when
9994 * accessing APIC page in protected mode. */
9995
9996 /*
9997 * Restore VM-exit control settings as we may not reenter this function the
9998 * next time around.
9999 */
10000 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10001
10002 /* Restore HMCPU indicators. */
10003 pVCpu->hm.s.fUsingDebugLoop = false;
10004 pVCpu->hm.s.fDebugWantRdTscExit = false;
10005 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10006
10007 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10008 return rcStrict;
10009}
10010
10011
10012/** @} */
10013
10014
10015/**
10016 * Checks if any expensive dtrace probes are enabled and we should go to the
10017 * debug loop.
10018 *
10019 * @returns true if we should use debug loop, false if not.
10020 */
10021static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10022{
10023 /* It's probably faster to OR the raw 32-bit counter variables together.
10024 Since the variables are in an array and the probes are next to one
10025 another (more or less), we have good locality. So, better read
10026 eight-nine cache lines ever time and only have one conditional, than
10027 128+ conditionals, right? */
10028 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10029 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10030 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10031 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10032 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10033 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10034 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10035 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10036 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10037 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10038 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10039 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10040 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10041 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10042 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10043 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10044 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10045 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10046 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10047 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10048 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10049 ) != 0
10050 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10051 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10052 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10053 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10054 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10055 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10056 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10057 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10058 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10059 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10060 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10061 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10062 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10063 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10064 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10065 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10066 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10067 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10068 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10069 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10070 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10071 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10072 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10073 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10074 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10075 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10076 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10077 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10078 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10079 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10080 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10081 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10082 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10083 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10084 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10085 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10086 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10087 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10088 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10089 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10090 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10091 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10092 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10093 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10094 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10095 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10096 ) != 0
10097 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10098 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10099 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10100 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10101 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10102 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10103 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10104 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10105 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10106 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10107 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10108 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10109 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10110 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10111 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10112 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10113 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10114 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10115 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10116 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10117 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10118 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10119 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10120 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10121 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10122 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10123 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10124 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10125 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10126 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10127 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10128 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10129 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10130 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10131 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10132 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10133 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10134 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10135 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10136 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10137 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10138 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10139 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10140 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10141 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10142 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10143 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10144 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10145 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10146 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10147 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10148 ) != 0;
10149}
10150
10151
10152/**
10153 * Runs the guest code using VT-x.
10154 *
10155 * @returns Strict VBox status code (i.e. informational status codes too).
10156 * @param pVCpu The cross context virtual CPU structure.
10157 * @param pCtx Pointer to the guest-CPU context.
10158 */
10159VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu, PCPUMCTX pCtx)
10160{
10161 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10162 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10163 HMVMX_ASSERT_PREEMPT_SAFE();
10164
10165 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10166
10167 VBOXSTRICTRC rcStrict;
10168 if ( !pVCpu->hm.s.fUseDebugLoop
10169 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10170 && !DBGFIsStepping(pVCpu)
10171 && !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
10172 rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, pCtx);
10173 else
10174 rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, pCtx);
10175
10176 if (rcStrict == VERR_EM_INTERPRETER)
10177 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10178 else if (rcStrict == VINF_EM_RESET)
10179 rcStrict = VINF_EM_TRIPLE_FAULT;
10180
10181 int rc2 = hmR0VmxExitToRing3(pVCpu, pCtx, rcStrict);
10182 if (RT_FAILURE(rc2))
10183 {
10184 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10185 rcStrict = rc2;
10186 }
10187 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10188 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10189 return rcStrict;
10190}
10191
10192
10193#ifndef HMVMX_USE_FUNCTION_TABLE
10194DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10195{
10196#ifdef DEBUG_ramshankar
10197#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
10198 do { \
10199 if (a_fSave != 0) \
10200 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
10201 VBOXSTRICTRC rcStrict = a_CallExpr; \
10202 if (a_fSave != 0) \
10203 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
10204 return rcStrict; \
10205 } while (0)
10206#else
10207# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
10208#endif
10209 switch (rcReason)
10210 {
10211 case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10212 case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10213 case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10214 case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10215 case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10216 case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10217 case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10218 case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10219 case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10220 case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10221 case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10222 case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10223 case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10224 case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10225 case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10226 case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10227 case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10228 case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10229 case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10230 case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10231 case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10232 case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10233 case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10234 case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10235 case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10236 case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10237 case VMX_EXIT_XDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10238 case VMX_EXIT_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10239 case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10240 case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10241 case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10242 case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10243 case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10244 case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10245
10246 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10247 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10248 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10249 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10250 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10251 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10252 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10253 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10254 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10255
10256 case VMX_EXIT_VMCLEAR:
10257 case VMX_EXIT_VMLAUNCH:
10258 case VMX_EXIT_VMPTRLD:
10259 case VMX_EXIT_VMPTRST:
10260 case VMX_EXIT_VMREAD:
10261 case VMX_EXIT_VMRESUME:
10262 case VMX_EXIT_VMWRITE:
10263 case VMX_EXIT_VMXOFF:
10264 case VMX_EXIT_VMXON:
10265 case VMX_EXIT_INVEPT:
10266 case VMX_EXIT_INVVPID:
10267 case VMX_EXIT_VMFUNC:
10268 case VMX_EXIT_XSAVES:
10269 case VMX_EXIT_XRSTORS:
10270 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10271
10272 case VMX_EXIT_ENCLS:
10273 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10274 case VMX_EXIT_PML_FULL:
10275 default:
10276 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10277 }
10278#undef VMEXIT_CALL_RET
10279}
10280#endif /* !HMVMX_USE_FUNCTION_TABLE */
10281
10282
10283#ifdef VBOX_STRICT
10284/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10285# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10286 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10287
10288# define HMVMX_ASSERT_PREEMPT_CPUID() \
10289 do { \
10290 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10291 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10292 } while (0)
10293
10294# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10295 do { \
10296 AssertPtr(pVCpu); \
10297 AssertPtr(pMixedCtx); \
10298 AssertPtr(pVmxTransient); \
10299 Assert(pVmxTransient->fVMEntryFailed == false); \
10300 Assert(ASMIntAreEnabled()); \
10301 HMVMX_ASSERT_PREEMPT_SAFE(); \
10302 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10303 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10304 HMVMX_ASSERT_PREEMPT_SAFE(); \
10305 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10306 HMVMX_ASSERT_PREEMPT_CPUID(); \
10307 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10308 } while (0)
10309
10310# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10311 do { \
10312 Log4Func(("\n")); \
10313 } while (0)
10314#else /* nonstrict builds: */
10315# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10316 do { \
10317 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10318 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10319 } while (0)
10320# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10321#endif
10322
10323
10324/**
10325 * Advances the guest RIP by the specified number of bytes.
10326 *
10327 * @param pVCpu The cross context virtual CPU structure.
10328 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10329 * out-of-sync. Make sure to update the required fields
10330 * before using them.
10331 * @param cbInstr Number of bytes to advance the RIP by.
10332 *
10333 * @remarks No-long-jump zone!!!
10334 */
10335DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10336{
10337 /* Advance the RIP. */
10338 pMixedCtx->rip += cbInstr;
10339 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
10340
10341 /* Update interrupt inhibition. */
10342 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10343 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10344 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10345}
10346
10347
10348/**
10349 * Advances the guest RIP after reading it from the VMCS.
10350 *
10351 * @returns VBox status code, no informational status codes.
10352 * @param pVCpu The cross context virtual CPU structure.
10353 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10354 * out-of-sync. Make sure to update the required fields
10355 * before using them.
10356 * @param pVmxTransient Pointer to the VMX transient structure.
10357 *
10358 * @remarks No-long-jump zone!!!
10359 */
10360static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10361{
10362 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10363 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
10364 | CPUMCTX_EXTRN_RFLAGS);
10365 AssertRCReturn(rc, rc);
10366
10367 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10368
10369 /*
10370 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10371 * pending debug exception field as it takes care of priority of events.
10372 *
10373 * See Intel spec. 32.2.1 "Debug Exceptions".
10374 */
10375 if ( !pVCpu->hm.s.fSingleInstruction
10376 && pMixedCtx->eflags.Bits.u1TF)
10377 {
10378 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
10379 AssertRCReturn(rc, rc);
10380 }
10381
10382 return VINF_SUCCESS;
10383}
10384
10385
10386/**
10387 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10388 * and update error record fields accordingly.
10389 *
10390 * @return VMX_IGS_* return codes.
10391 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10392 * wrong with the guest state.
10393 *
10394 * @param pVCpu The cross context virtual CPU structure.
10395 * @param pCtx Pointer to the guest-CPU state.
10396 *
10397 * @remarks This function assumes our cache of the VMCS controls
10398 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10399 */
10400static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCPUMCTX pCtx)
10401{
10402#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10403#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10404 uError = (err); \
10405 break; \
10406 } else do { } while (0)
10407
10408 int rc;
10409 PVM pVM = pVCpu->CTX_SUFF(pVM);
10410 uint32_t uError = VMX_IGS_ERROR;
10411 uint32_t u32Val;
10412 bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10413
10414 do
10415 {
10416 /*
10417 * CR0.
10418 */
10419 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10420 uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10421 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10422 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10423 if (fUnrestrictedGuest)
10424 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
10425
10426 uint32_t u32GuestCr0;
10427 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10428 AssertRCBreak(rc);
10429 HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10430 HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10431 if ( !fUnrestrictedGuest
10432 && (u32GuestCr0 & X86_CR0_PG)
10433 && !(u32GuestCr0 & X86_CR0_PE))
10434 {
10435 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10436 }
10437
10438 /*
10439 * CR4.
10440 */
10441 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10442 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10443
10444 uint32_t u32GuestCr4;
10445 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10446 AssertRCBreak(rc);
10447 HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10448 HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10449
10450 /*
10451 * IA32_DEBUGCTL MSR.
10452 */
10453 uint64_t u64Val;
10454 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10455 AssertRCBreak(rc);
10456 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10457 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10458 {
10459 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10460 }
10461 uint64_t u64DebugCtlMsr = u64Val;
10462
10463#ifdef VBOX_STRICT
10464 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10465 AssertRCBreak(rc);
10466 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10467#endif
10468 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10469
10470 /*
10471 * RIP and RFLAGS.
10472 */
10473 uint32_t u32Eflags;
10474#if HC_ARCH_BITS == 64
10475 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10476 AssertRCBreak(rc);
10477 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10478 if ( !fLongModeGuest
10479 || !pCtx->cs.Attr.n.u1Long)
10480 {
10481 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10482 }
10483 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10484 * must be identical if the "IA-32e mode guest" VM-entry
10485 * control is 1 and CS.L is 1. No check applies if the
10486 * CPU supports 64 linear-address bits. */
10487
10488 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10489 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10490 AssertRCBreak(rc);
10491 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10492 VMX_IGS_RFLAGS_RESERVED);
10493 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10494 u32Eflags = u64Val;
10495#else
10496 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10497 AssertRCBreak(rc);
10498 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10499 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10500#endif
10501
10502 if ( fLongModeGuest
10503 || ( fUnrestrictedGuest
10504 && !(u32GuestCr0 & X86_CR0_PE)))
10505 {
10506 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10507 }
10508
10509 uint32_t u32EntryInfo;
10510 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10511 AssertRCBreak(rc);
10512 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10513 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10514 {
10515 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10516 }
10517
10518 /*
10519 * 64-bit checks.
10520 */
10521#if HC_ARCH_BITS == 64
10522 if (fLongModeGuest)
10523 {
10524 HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10525 HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10526 }
10527
10528 if ( !fLongModeGuest
10529 && (u32GuestCr4 & X86_CR4_PCIDE))
10530 {
10531 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10532 }
10533
10534 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10535 * 51:32 beyond the processor's physical-address width are 0. */
10536
10537 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10538 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10539 {
10540 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10541 }
10542
10543 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10544 AssertRCBreak(rc);
10545 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10546
10547 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10548 AssertRCBreak(rc);
10549 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10550#endif
10551
10552 /*
10553 * PERF_GLOBAL MSR.
10554 */
10555 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10556 {
10557 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10558 AssertRCBreak(rc);
10559 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10560 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10561 }
10562
10563 /*
10564 * PAT MSR.
10565 */
10566 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10567 {
10568 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10569 AssertRCBreak(rc);
10570 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10571 for (unsigned i = 0; i < 8; i++)
10572 {
10573 uint8_t u8Val = (u64Val & 0xff);
10574 if ( u8Val != 0 /* UC */
10575 && u8Val != 1 /* WC */
10576 && u8Val != 4 /* WT */
10577 && u8Val != 5 /* WP */
10578 && u8Val != 6 /* WB */
10579 && u8Val != 7 /* UC- */)
10580 {
10581 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10582 }
10583 u64Val >>= 8;
10584 }
10585 }
10586
10587 /*
10588 * EFER MSR.
10589 */
10590 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10591 {
10592 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10593 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10594 AssertRCBreak(rc);
10595 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10596 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10597 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10598 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10599 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10600 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10601 || !(u32GuestCr0 & X86_CR0_PG)
10602 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10603 VMX_IGS_EFER_LMA_LME_MISMATCH);
10604 }
10605
10606 /*
10607 * Segment registers.
10608 */
10609 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10610 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10611 if (!(u32Eflags & X86_EFL_VM))
10612 {
10613 /* CS */
10614 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10615 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10616 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10617 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10618 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10619 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10620 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10621 /* CS cannot be loaded with NULL in protected mode. */
10622 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10623 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10624 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10625 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10626 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10627 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10628 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10629 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10630 else
10631 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10632
10633 /* SS */
10634 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10635 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10636 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10637 if ( !(pCtx->cr0 & X86_CR0_PE)
10638 || pCtx->cs.Attr.n.u4Type == 3)
10639 {
10640 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10641 }
10642 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10643 {
10644 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10645 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10646 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10647 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10648 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10649 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10650 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10651 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10652 }
10653
10654 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmenReg(). */
10655 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10656 {
10657 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10658 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10659 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10660 || pCtx->ds.Attr.n.u4Type > 11
10661 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10662 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10663 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10664 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10665 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10666 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10667 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10668 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10669 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10670 }
10671 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10672 {
10673 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10674 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10675 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10676 || pCtx->es.Attr.n.u4Type > 11
10677 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10678 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10679 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10680 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10681 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10682 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10683 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10684 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10685 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10686 }
10687 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10688 {
10689 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10690 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10691 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10692 || pCtx->fs.Attr.n.u4Type > 11
10693 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10694 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10695 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10696 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10697 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10698 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10699 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10700 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10701 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10702 }
10703 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10704 {
10705 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10706 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10707 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10708 || pCtx->gs.Attr.n.u4Type > 11
10709 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10710 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10711 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10712 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10713 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10714 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10715 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10716 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10717 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10718 }
10719 /* 64-bit capable CPUs. */
10720#if HC_ARCH_BITS == 64
10721 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10722 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10723 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10724 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10725 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10726 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10727 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10728 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10729 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10730 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10731 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10732#endif
10733 }
10734 else
10735 {
10736 /* V86 mode checks. */
10737 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10738 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10739 {
10740 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10741 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10742 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10743 }
10744 else
10745 {
10746 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10747 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10748 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10749 }
10750
10751 /* CS */
10752 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10753 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10754 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10755 /* SS */
10756 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10757 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10758 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10759 /* DS */
10760 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10761 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10762 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10763 /* ES */
10764 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10765 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10766 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10767 /* FS */
10768 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10769 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10770 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10771 /* GS */
10772 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10773 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10774 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10775 /* 64-bit capable CPUs. */
10776#if HC_ARCH_BITS == 64
10777 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10778 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10779 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10780 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10781 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10782 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10783 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10784 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10785 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10786 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10787 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10788#endif
10789 }
10790
10791 /*
10792 * TR.
10793 */
10794 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10795 /* 64-bit capable CPUs. */
10796#if HC_ARCH_BITS == 64
10797 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10798#endif
10799 if (fLongModeGuest)
10800 {
10801 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10802 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10803 }
10804 else
10805 {
10806 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10807 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10808 VMX_IGS_TR_ATTR_TYPE_INVALID);
10809 }
10810 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10811 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10812 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10813 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10814 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10815 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10816 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10817 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10818
10819 /*
10820 * GDTR and IDTR.
10821 */
10822#if HC_ARCH_BITS == 64
10823 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10824 AssertRCBreak(rc);
10825 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10826
10827 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10828 AssertRCBreak(rc);
10829 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10830#endif
10831
10832 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10833 AssertRCBreak(rc);
10834 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10835
10836 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10837 AssertRCBreak(rc);
10838 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10839
10840 /*
10841 * Guest Non-Register State.
10842 */
10843 /* Activity State. */
10844 uint32_t u32ActivityState;
10845 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10846 AssertRCBreak(rc);
10847 HMVMX_CHECK_BREAK( !u32ActivityState
10848 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
10849 VMX_IGS_ACTIVITY_STATE_INVALID);
10850 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10851 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10852 uint32_t u32IntrState;
10853 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
10854 AssertRCBreak(rc);
10855 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
10856 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10857 {
10858 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10859 }
10860
10861 /** @todo Activity state and injecting interrupts. Left as a todo since we
10862 * currently don't use activity states but ACTIVE. */
10863
10864 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10865 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10866
10867 /* Guest interruptibility-state. */
10868 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10869 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10870 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
10871 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10872 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10873 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10874 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10875 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10876 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10877 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
10878 {
10879 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10880 {
10881 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10882 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10883 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10884 }
10885 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10886 {
10887 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10888 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10889 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10890 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10891 }
10892 }
10893 /** @todo Assumes the processor is not in SMM. */
10894 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10895 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10896 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10897 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10898 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10899 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
10900 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10901 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10902 {
10903 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
10904 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10905 }
10906
10907 /* Pending debug exceptions. */
10908#if HC_ARCH_BITS == 64
10909 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
10910 AssertRCBreak(rc);
10911 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10912 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10913 u32Val = u64Val; /* For pending debug exceptions checks below. */
10914#else
10915 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
10916 AssertRCBreak(rc);
10917 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10918 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10919#endif
10920
10921 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10922 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
10923 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10924 {
10925 if ( (u32Eflags & X86_EFL_TF)
10926 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10927 {
10928 /* Bit 14 is PendingDebug.BS. */
10929 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10930 }
10931 if ( !(u32Eflags & X86_EFL_TF)
10932 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10933 {
10934 /* Bit 14 is PendingDebug.BS. */
10935 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10936 }
10937 }
10938
10939 /* VMCS link pointer. */
10940 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10941 AssertRCBreak(rc);
10942 if (u64Val != UINT64_C(0xffffffffffffffff))
10943 {
10944 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10945 /** @todo Bits beyond the processor's physical-address width MBZ. */
10946 /** @todo 32-bit located in memory referenced by value of this field (as a
10947 * physical address) must contain the processor's VMCS revision ID. */
10948 /** @todo SMM checks. */
10949 }
10950
10951 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10952 * not using Nested Paging? */
10953 if ( pVM->hm.s.fNestedPaging
10954 && !fLongModeGuest
10955 && CPUMIsGuestInPAEModeEx(pCtx))
10956 {
10957 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10958 AssertRCBreak(rc);
10959 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10960
10961 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10962 AssertRCBreak(rc);
10963 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10964
10965 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10966 AssertRCBreak(rc);
10967 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10968
10969 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10970 AssertRCBreak(rc);
10971 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10972 }
10973
10974 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10975 if (uError == VMX_IGS_ERROR)
10976 uError = VMX_IGS_REASON_NOT_FOUND;
10977 } while (0);
10978
10979 pVCpu->hm.s.u32HMError = uError;
10980 return uError;
10981
10982#undef HMVMX_ERROR_BREAK
10983#undef HMVMX_CHECK_BREAK
10984}
10985
10986/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10987/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
10988/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10989
10990/** @name VM-exit handlers.
10991 * @{
10992 */
10993
10994/**
10995 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
10996 */
10997HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10998{
10999 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11000 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11001 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11002 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11003 return VINF_SUCCESS;
11004 return VINF_EM_RAW_INTERRUPT;
11005}
11006
11007
11008/**
11009 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11010 */
11011HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11012{
11013 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11014 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11015
11016 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11017 AssertRCReturn(rc, rc);
11018
11019 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11020 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11021 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11022 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11023
11024 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11025 {
11026 /*
11027 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
11028 * injected it ourselves and anything we inject is not going to cause a VM-exit directly
11029 * for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
11030 *
11031 * [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11032 * [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
11033 */
11034 VMXDispatchHostNmi();
11035 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11036 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11037 return VINF_SUCCESS;
11038 }
11039
11040 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11041 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11042 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11043 { /* likely */ }
11044 else
11045 {
11046 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11047 rcStrictRc1 = VINF_SUCCESS;
11048 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11049 return rcStrictRc1;
11050 }
11051
11052 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11053 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11054 switch (uIntType)
11055 {
11056 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11057 Assert(uVector == X86_XCPT_DB);
11058 RT_FALL_THRU();
11059 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11060 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11061 RT_FALL_THRU();
11062 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11063 {
11064 /*
11065 * If there's any exception caused as a result of event injection, the resulting
11066 * secondary/final execption will be pending, we shall continue guest execution
11067 * after injecting the event. The page-fault case is complicated and we manually
11068 * handle any currently pending event in hmR0VmxExitXcptPF.
11069 */
11070 if (!pVCpu->hm.s.Event.fPending)
11071 { /* likely */ }
11072 else if (uVector != X86_XCPT_PF)
11073 {
11074 rc = VINF_SUCCESS;
11075 break;
11076 }
11077
11078 switch (uVector)
11079 {
11080 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11081 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11082 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11083 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11084 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11085 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11086
11087 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11088 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11089 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11090 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11091 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11092 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11093 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11094 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11095 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11096 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11097 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11098 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11099 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11100 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11101 default:
11102 {
11103 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11104 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11105 {
11106 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11107 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11108 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11109
11110 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
11111 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11112 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11113 AssertRCReturn(rc, rc);
11114 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11115 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11116 0 /* GCPtrFaultAddress */);
11117 }
11118 else
11119 {
11120 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11121 pVCpu->hm.s.u32HMError = uVector;
11122 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11123 }
11124 break;
11125 }
11126 }
11127 break;
11128 }
11129
11130 default:
11131 {
11132 pVCpu->hm.s.u32HMError = uExitIntInfo;
11133 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11134 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11135 break;
11136 }
11137 }
11138 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11139 return rc;
11140}
11141
11142
11143/**
11144 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11145 */
11146HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11147{
11148 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11149
11150 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11151 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11152
11153 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11154 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11155 return VINF_SUCCESS;
11156}
11157
11158
11159/**
11160 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11161 */
11162HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11163{
11164 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11165 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11166 {
11167 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11168 HMVMX_RETURN_UNEXPECTED_EXIT();
11169 }
11170
11171 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11172
11173 /*
11174 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11175 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11176 */
11177 uint32_t fIntrState = 0;
11178 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11179 AssertRCReturn(rc, rc);
11180
11181 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11182 if ( fBlockSti
11183 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11184 {
11185 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11186 }
11187
11188 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11189 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11190
11191 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11192 return VINF_SUCCESS;
11193}
11194
11195
11196/**
11197 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11198 */
11199HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11200{
11201 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11202 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11203}
11204
11205
11206/**
11207 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11208 */
11209HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11210{
11211 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11212 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11213}
11214
11215
11216/**
11217 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11218 */
11219HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11220{
11221 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11222 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
11223
11224 /*
11225 * Get the state we need and update the exit history entry.
11226 */
11227 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11228 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
11229 | CPUMCTX_EXTRN_CS);
11230 AssertRCReturn(rc, rc);
11231
11232 VBOXSTRICTRC rcStrict;
11233 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11234 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
11235 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11236 if (!pExitRec)
11237 {
11238 /*
11239 * Regular CPUID instruction execution.
11240 */
11241 PVM pVM = pVCpu->CTX_SUFF(pVM);
11242 rcStrict = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11243 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11244 {
11245 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11246 Assert(pVmxTransient->cbInstr == 2);
11247 }
11248 else
11249 {
11250 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11251 rcStrict = VERR_EM_INTERPRETER;
11252 }
11253 }
11254 else
11255 {
11256 /*
11257 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
11258 */
11259 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
11260 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11261 AssertRCReturn(rc2, rc2);
11262
11263 Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11264 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11265
11266 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11267 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
11268
11269 Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11270 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11271 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11272 }
11273 return VBOXSTRICTRC_TODO(rcStrict);
11274}
11275
11276
11277/**
11278 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11279 */
11280HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11281{
11282 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11283 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4);
11284 AssertRCReturn(rc, rc);
11285
11286 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11287 return VINF_EM_RAW_EMULATE_INSTR;
11288
11289 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11290 HMVMX_RETURN_UNEXPECTED_EXIT();
11291}
11292
11293
11294/**
11295 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11296 */
11297HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11298{
11299 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11300 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11301 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11302 AssertRCReturn(rc, rc);
11303
11304 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11305 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11306 {
11307 /* If we get a spurious VM-exit when offsetting is enabled,
11308 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11309 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11310 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11311 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
11312 | HM_CHANGED_GUEST_RFLAGS);
11313 }
11314 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11315 {
11316 rcStrict = VINF_SUCCESS;
11317 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11318 }
11319 return rcStrict;
11320}
11321
11322
11323/**
11324 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11325 */
11326HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11327{
11328 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11329 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
11330 | CPUMCTX_EXTRN_TSC_AUX);
11331 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11332 AssertRCReturn(rc, rc);
11333
11334 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11335 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11336 {
11337 /* If we get a spurious VM-exit when offsetting is enabled,
11338 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11339 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11340 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11341 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
11342 | HM_CHANGED_GUEST_RFLAGS);
11343 }
11344 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11345 {
11346 rcStrict = VINF_SUCCESS;
11347 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11348 }
11349 return rcStrict;
11350}
11351
11352
11353/**
11354 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11355 */
11356HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11357{
11358 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11359 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4
11360 | CPUMCTX_EXTRN_CR0
11361 | CPUMCTX_EXTRN_RFLAGS
11362 | CPUMCTX_EXTRN_SS);
11363 AssertRCReturn(rc, rc);
11364
11365 PVM pVM = pVCpu->CTX_SUFF(pVM);
11366 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11367 if (RT_LIKELY(rc == VINF_SUCCESS))
11368 {
11369 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11370 Assert(pVmxTransient->cbInstr == 2);
11371 }
11372 else
11373 {
11374 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11375 rc = VERR_EM_INTERPRETER;
11376 }
11377 return rc;
11378}
11379
11380
11381/**
11382 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11383 */
11384HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11385{
11386 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11387
11388 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11389 if (EMAreHypercallInstructionsEnabled(pVCpu))
11390 {
11391 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
11392 | CPUMCTX_EXTRN_RFLAGS
11393 | CPUMCTX_EXTRN_CR0
11394 | CPUMCTX_EXTRN_SS
11395 | CPUMCTX_EXTRN_CS
11396 | CPUMCTX_EXTRN_EFER);
11397 AssertRCReturn(rc, rc);
11398
11399 /* Perform the hypercall. */
11400 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11401 if (rcStrict == VINF_SUCCESS)
11402 {
11403 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11404 AssertRCReturn(rc, rc);
11405 }
11406 else
11407 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11408 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11409 || RT_FAILURE(rcStrict));
11410
11411 /* If the hypercall changes anything other than guest's general-purpose registers,
11412 we would need to reload the guest changed bits here before VM-entry. */
11413 }
11414 else
11415 Log4Func(("Hypercalls not enabled\n"));
11416
11417 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11418 if (RT_FAILURE(rcStrict))
11419 {
11420 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11421 rcStrict = VINF_SUCCESS;
11422 }
11423
11424 return rcStrict;
11425}
11426
11427
11428/**
11429 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11430 */
11431HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11432{
11433 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11434 PVM pVM = pVCpu->CTX_SUFF(pVM);
11435 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11436
11437 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11438 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK);
11439 AssertRCReturn(rc, rc);
11440
11441 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11442 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11443 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11444 else
11445 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11446 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11447 return rcStrict;
11448}
11449
11450
11451/**
11452 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11453 */
11454HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11455{
11456 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11457 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11458 | CPUMCTX_EXTRN_RFLAGS
11459 | CPUMCTX_EXTRN_SS);
11460 AssertRCReturn(rc, rc);
11461
11462 PVM pVM = pVCpu->CTX_SUFF(pVM);
11463 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11464 if (RT_LIKELY(rc == VINF_SUCCESS))
11465 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11466 else
11467 {
11468 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11469 rc = VERR_EM_INTERPRETER;
11470 }
11471 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11472 return rc;
11473}
11474
11475
11476/**
11477 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11478 */
11479HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11480{
11481 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11482 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11483 | CPUMCTX_EXTRN_RFLAGS
11484 | CPUMCTX_EXTRN_SS);
11485 AssertRCReturn(rc, rc);
11486
11487 PVM pVM = pVCpu->CTX_SUFF(pVM);
11488 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11489 rc = VBOXSTRICTRC_VAL(rc2);
11490 if (RT_LIKELY( rc == VINF_SUCCESS
11491 || rc == VINF_EM_HALT))
11492 {
11493 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11494 AssertRCReturn(rc3, rc3);
11495
11496 if ( rc == VINF_EM_HALT
11497 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11498 rc = VINF_SUCCESS;
11499 }
11500 else
11501 {
11502 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11503 rc = VERR_EM_INTERPRETER;
11504 }
11505 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11506 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11507 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11508 return rc;
11509}
11510
11511
11512/**
11513 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11514 */
11515HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11516{
11517 /*
11518 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
11519 * mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
11520 * treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
11521 * VMX root operation. If we get here, something funny is going on.
11522 *
11523 * See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
11524 */
11525 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11526 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11527 HMVMX_RETURN_UNEXPECTED_EXIT();
11528}
11529
11530
11531/**
11532 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11533 */
11534HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11535{
11536 /*
11537 * This can only happen if we support dual-monitor treatment of SMI, which can be activated
11538 * by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
11539 * this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
11540 * an SMI. If we get here, something funny is going on.
11541 *
11542 * See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11543 * See Intel spec. 25.3 "Other Causes of VM-Exits"
11544 */
11545 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11546 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11547 HMVMX_RETURN_UNEXPECTED_EXIT();
11548}
11549
11550
11551/**
11552 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11553 */
11554HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11555{
11556 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11557 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11558 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11559 HMVMX_RETURN_UNEXPECTED_EXIT();
11560}
11561
11562
11563/**
11564 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11565 */
11566HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11567{
11568 /*
11569 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
11570 * We don't make use of it as our guests don't have direct access to the host LAPIC.
11571 * See Intel spec. 25.3 "Other Causes of VM-exits".
11572 */
11573 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11574 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11575 HMVMX_RETURN_UNEXPECTED_EXIT();
11576}
11577
11578
11579/**
11580 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11581 * VM-exit.
11582 */
11583HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11584{
11585 /*
11586 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11587 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11588 *
11589 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11590 * See Intel spec. "23.8 Restrictions on VMX operation".
11591 */
11592 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11593 return VINF_SUCCESS;
11594}
11595
11596
11597/**
11598 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11599 * VM-exit.
11600 */
11601HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11602{
11603 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11604 return VINF_EM_RESET;
11605}
11606
11607
11608/**
11609 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11610 */
11611HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11612{
11613 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11614 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11615
11616 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11617 AssertRCReturn(rc, rc);
11618
11619 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11620 rc = VINF_SUCCESS;
11621 else
11622 rc = VINF_EM_HALT;
11623
11624 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11625 if (rc != VINF_SUCCESS)
11626 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11627 return rc;
11628}
11629
11630
11631/**
11632 * VM-exit handler for instructions that result in a \#UD exception delivered to
11633 * the guest.
11634 */
11635HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11636{
11637 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11638 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11639 return VINF_SUCCESS;
11640}
11641
11642
11643/**
11644 * VM-exit handler for expiry of the VMX preemption timer.
11645 */
11646HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11647{
11648 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11649
11650 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11651 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11652
11653 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11654 PVM pVM = pVCpu->CTX_SUFF(pVM);
11655 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11656 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11657 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11658}
11659
11660
11661/**
11662 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11663 */
11664HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11665{
11666 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11667
11668 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11669 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
11670 | CPUMCTX_EXTRN_CR4);
11671 AssertRCReturn(rc, rc);
11672
11673 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11674 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11675 : HM_CHANGED_XCPT_RAISED_MASK);
11676
11677 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11678
11679 return rcStrict;
11680}
11681
11682
11683/**
11684 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11685 */
11686HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11687{
11688 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11689 /** @todo Use VM-exit instruction information. */
11690 return VERR_EM_INTERPRETER;
11691}
11692
11693
11694/**
11695 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11696 * Error VM-exit.
11697 */
11698HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11699{
11700 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11701 rc |= hmR0VmxCheckVmcsCtls(pVCpu);
11702 AssertRCReturn(rc, rc);
11703
11704 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pMixedCtx);
11705 NOREF(uInvalidReason);
11706
11707#ifdef VBOX_STRICT
11708 uint32_t fIntrState;
11709 RTHCUINTREG uHCReg;
11710 uint64_t u64Val;
11711 uint32_t u32Val;
11712
11713 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11714 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11715 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11716 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11717 AssertRCReturn(rc, rc);
11718
11719 Log4(("uInvalidReason %u\n", uInvalidReason));
11720 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11721 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11722 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11723 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", fIntrState));
11724
11725 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11726 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11727 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11728 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11729 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11730 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11731 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11732 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11733 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11734 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11735 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11736 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11737
11738 hmR0DumpRegs(pVCpu, pMixedCtx);
11739#else
11740 NOREF(pVmxTransient);
11741#endif
11742
11743 return VERR_VMX_INVALID_GUEST_STATE;
11744}
11745
11746
11747/**
11748 * VM-exit handler for VM-entry failure due to an MSR-load
11749 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11750 */
11751HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11752{
11753 NOREF(pVmxTransient);
11754 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11755 HMVMX_RETURN_UNEXPECTED_EXIT();
11756}
11757
11758
11759/**
11760 * VM-exit handler for VM-entry failure due to a machine-check event
11761 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11762 */
11763HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11764{
11765 NOREF(pVmxTransient);
11766 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11767 HMVMX_RETURN_UNEXPECTED_EXIT();
11768}
11769
11770
11771/**
11772 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11773 * theory.
11774 */
11775HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11776{
11777 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
11778 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
11779 return VERR_VMX_UNDEFINED_EXIT_CODE;
11780}
11781
11782
11783/**
11784 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11785 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11786 * Conditional VM-exit.
11787 */
11788HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11789{
11790 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11791
11792 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
11793 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11794 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
11795 return VERR_EM_INTERPRETER;
11796 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11797 HMVMX_RETURN_UNEXPECTED_EXIT();
11798}
11799
11800
11801/**
11802 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11803 */
11804HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11805{
11806 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11807
11808 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
11809 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
11810 return VERR_EM_INTERPRETER;
11811 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11812 HMVMX_RETURN_UNEXPECTED_EXIT();
11813}
11814
11815
11816/**
11817 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11818 */
11819HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11820{
11821 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11822
11823 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. FS, GS (base) can be accessed by MSR reads. */
11824 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11825 | CPUMCTX_EXTRN_RFLAGS
11826 | CPUMCTX_EXTRN_SS
11827 | CPUMCTX_EXTRN_FS
11828 | CPUMCTX_EXTRN_GS);
11829 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11830 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
11831 AssertRCReturn(rc, rc);
11832 Log4Func(("ecx=%#RX32\n", pMixedCtx->ecx));
11833
11834#ifdef VBOX_STRICT
11835 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
11836 {
11837 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
11838 && pMixedCtx->ecx != MSR_K6_EFER)
11839 {
11840 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11841 pMixedCtx->ecx));
11842 HMVMX_RETURN_UNEXPECTED_EXIT();
11843 }
11844 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11845 {
11846 VMXMSREXITREAD enmRead;
11847 VMXMSREXITWRITE enmWrite;
11848 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
11849 AssertRCReturn(rc2, rc2);
11850 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11851 {
11852 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11853 HMVMX_RETURN_UNEXPECTED_EXIT();
11854 }
11855 }
11856 }
11857#endif
11858
11859 PVM pVM = pVCpu->CTX_SUFF(pVM);
11860 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11861 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
11862 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
11863 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11864 if (RT_SUCCESS(rc))
11865 {
11866 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11867 Assert(pVmxTransient->cbInstr == 2);
11868 }
11869 return rc;
11870}
11871
11872
11873/**
11874 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11875 */
11876HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11877{
11878 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11879 PVM pVM = pVCpu->CTX_SUFF(pVM);
11880 int rc = VINF_SUCCESS;
11881
11882 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. FS, GS (base) can be accessed by MSR writes. */
11883 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11884 | CPUMCTX_EXTRN_RFLAGS
11885 | CPUMCTX_EXTRN_SS
11886 | CPUMCTX_EXTRN_FS
11887 | CPUMCTX_EXTRN_GS);
11888 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11889 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
11890 AssertRCReturn(rc, rc);
11891 Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
11892
11893 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11894 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
11895 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
11896
11897 if (RT_SUCCESS(rc))
11898 {
11899 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11900
11901 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
11902 if ( pMixedCtx->ecx == MSR_IA32_APICBASE
11903 || ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
11904 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
11905 {
11906 /*
11907 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
11908 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
11909 * EMInterpretWrmsr() changes it.
11910 */
11911 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11912 }
11913 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
11914 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11915 else if (pMixedCtx->ecx == MSR_K6_EFER)
11916 {
11917 /*
11918 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
11919 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
11920 * the other bits as well, SCE and NXE. See @bugref{7368}.
11921 */
11922 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
11923 | HM_CHANGED_VMX_ENTRY_CTLS
11924 | HM_CHANGED_VMX_EXIT_CTLS);
11925 }
11926
11927 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
11928 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11929 {
11930 switch (pMixedCtx->ecx)
11931 {
11932 case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
11933 case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
11934 case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
11935 case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
11936 case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
11937 case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
11938 default:
11939 {
11940 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11941 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
11942 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11943 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
11944 break;
11945 }
11946 }
11947 }
11948#ifdef VBOX_STRICT
11949 else
11950 {
11951 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
11952 switch (pMixedCtx->ecx)
11953 {
11954 case MSR_IA32_SYSENTER_CS:
11955 case MSR_IA32_SYSENTER_EIP:
11956 case MSR_IA32_SYSENTER_ESP:
11957 case MSR_K8_FS_BASE:
11958 case MSR_K8_GS_BASE:
11959 {
11960 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
11961 HMVMX_RETURN_UNEXPECTED_EXIT();
11962 }
11963
11964 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
11965 default:
11966 {
11967 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11968 {
11969 /* EFER writes are always intercepted, see hmR0VmxExportGuestMsrs(). */
11970 if (pMixedCtx->ecx != MSR_K6_EFER)
11971 {
11972 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11973 pMixedCtx->ecx));
11974 HMVMX_RETURN_UNEXPECTED_EXIT();
11975 }
11976 }
11977
11978 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11979 {
11980 VMXMSREXITREAD enmRead;
11981 VMXMSREXITWRITE enmWrite;
11982 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
11983 AssertRCReturn(rc2, rc2);
11984 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
11985 {
11986 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11987 HMVMX_RETURN_UNEXPECTED_EXIT();
11988 }
11989 }
11990 break;
11991 }
11992 }
11993 }
11994#endif /* VBOX_STRICT */
11995 }
11996 return rc;
11997}
11998
11999
12000/**
12001 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12002 */
12003HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12004{
12005 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12006 /** @todo The guest has likely hit a contended spinlock. We might want to
12007 * poke a schedule different guest VCPU. */
12008 return VINF_EM_RAW_INTERRUPT;
12009}
12010
12011
12012/**
12013 * VM-exit handler for when the TPR value is lowered below the specified
12014 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12015 */
12016HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12017{
12018 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12019 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12020
12021 /*
12022 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12023 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12024 */
12025 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12026 return VINF_SUCCESS;
12027}
12028
12029
12030/**
12031 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12032 * VM-exit.
12033 *
12034 * @retval VINF_SUCCESS when guest execution can continue.
12035 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12036 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12037 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12038 * interpreter.
12039 */
12040HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12041{
12042 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12043 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12044
12045 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12046 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12047 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12048 AssertRCReturn(rc, rc);
12049
12050 VBOXSTRICTRC rcStrict;
12051 PVM pVM = pVCpu->CTX_SUFF(pVM);
12052 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12053 uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQualification);
12054 switch (uAccessType)
12055 {
12056 case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
12057 {
12058 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12059 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
12060 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification));
12061 AssertMsg( rcStrict == VINF_SUCCESS
12062 || rcStrict == VINF_IEM_RAISED_XCPT
12063 || rcStrict == VINF_PGM_CHANGE_MODE
12064 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12065
12066 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
12067 {
12068 case 0:
12069 {
12070 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12071 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
12072 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12073 break;
12074 }
12075
12076 case 2:
12077 {
12078 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
12079 /* Nothing to do here, CR2 it's not part of the VMCS. */
12080 break;
12081 }
12082
12083 case 3:
12084 {
12085 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
12086 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
12087 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR3);
12088 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12089 break;
12090 }
12091
12092 case 4:
12093 {
12094 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
12095 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
12096 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
12097 pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12098 break;
12099 }
12100
12101 case 8:
12102 {
12103 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
12104 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12105 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
12106 break;
12107 }
12108 default:
12109 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification)));
12110 break;
12111 }
12112 break;
12113 }
12114
12115 case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
12116 {
12117 Assert( !pVM->hm.s.fNestedPaging
12118 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12119 || pVCpu->hm.s.fUsingDebugLoop
12120 || VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 3);
12121 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12122 Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 8
12123 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12124
12125 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12126 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification),
12127 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification));
12128 AssertMsg( rcStrict == VINF_SUCCESS
12129 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12130#ifdef VBOX_WITH_STATISTICS
12131 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
12132 {
12133 case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
12134 case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
12135 case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
12136 case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
12137 case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
12138 }
12139#endif
12140 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
12141 VBOXSTRICTRC_VAL(rcStrict)));
12142 if (VMX_EXIT_QUAL_CRX_GENREG(uExitQualification) == X86_GREG_xSP)
12143 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RSP);
12144 break;
12145 }
12146
12147 case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12148 {
12149 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12150 AssertMsg( rcStrict == VINF_SUCCESS
12151 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12152
12153 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12154 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12155 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12156 break;
12157 }
12158
12159 case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12160 {
12161 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12162 VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQualification));
12163 AssertMsg( rcStrict == VINF_SUCCESS
12164 || rcStrict == VINF_IEM_RAISED_XCPT
12165 || rcStrict == VINF_PGM_CHANGE_MODE,
12166 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12167
12168 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12169 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12170 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12171 break;
12172 }
12173
12174 default:
12175 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12176 VERR_VMX_UNEXPECTED_EXCEPTION);
12177 }
12178
12179 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
12180 : HM_CHANGED_XCPT_RAISED_MASK);
12181 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12182 NOREF(pVM);
12183 return rcStrict;
12184}
12185
12186
12187/**
12188 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12189 * VM-exit.
12190 */
12191HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12192{
12193 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12194 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12195 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
12196
12197 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12198 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12199 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
12200 | CPUMCTX_EXTRN_SREG_MASK
12201 | CPUMCTX_EXTRN_EFER);
12202 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12203 AssertRCReturn(rc, rc);
12204
12205 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12206 uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQualification);
12207 uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQualification);
12208 bool fIOWrite = ( VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQualification)
12209 == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12210 bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQualification);
12211 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12212 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12213 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12214
12215 /*
12216 * Update exit history to see if this exit can be optimized.
12217 */
12218 VBOXSTRICTRC rcStrict;
12219 PCEMEXITREC pExitRec = NULL;
12220 if ( !fGstStepping
12221 && !fDbgStepping)
12222 pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12223 !fIOString
12224 ? !fIOWrite
12225 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12226 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12227 : !fIOWrite
12228 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12229 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12230 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12231 if (!pExitRec)
12232 {
12233 /* I/O operation lookup arrays. */
12234 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12235 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12236 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12237 uint32_t const cbInstr = pVmxTransient->cbInstr;
12238 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12239 PVM pVM = pVCpu->CTX_SUFF(pVM);
12240 if (fIOString)
12241 {
12242 /*
12243 * INS/OUTS - I/O String instruction.
12244 *
12245 * Use instruction-information if available, otherwise fall back on
12246 * interpreting the instruction.
12247 */
12248 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12249 fIOWrite ? 'w' : 'r'));
12250 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12251 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12252 {
12253 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12254 AssertRCReturn(rc2, rc2);
12255 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12256 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12257 IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12258 bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification);
12259 if (fIOWrite)
12260 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12261 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12262 else
12263 {
12264 /*
12265 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12266 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12267 * See Intel Instruction spec. for "INS".
12268 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12269 */
12270 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12271 }
12272 }
12273 else
12274 rcStrict = IEMExecOne(pVCpu);
12275
12276 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12277 fUpdateRipAlready = true;
12278 }
12279 else
12280 {
12281 /*
12282 * IN/OUT - I/O instruction.
12283 */
12284 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12285 fIOWrite ? 'w' : 'r'));
12286 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12287 Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification));
12288 if (fIOWrite)
12289 {
12290 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12291 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12292 }
12293 else
12294 {
12295 uint32_t u32Result = 0;
12296 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12297 if (IOM_SUCCESS(rcStrict))
12298 {
12299 /* Save result of I/O IN instr. in AL/AX/EAX. */
12300 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12301 }
12302 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12303 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12304 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12305 }
12306 }
12307
12308 if (IOM_SUCCESS(rcStrict))
12309 {
12310 if (!fUpdateRipAlready)
12311 {
12312 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12313 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12314 }
12315
12316 /*
12317 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12318 * while booting Fedora 17 64-bit guest.
12319 *
12320 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12321 */
12322 if (fIOString)
12323 {
12324 /** @todo Single-step for INS/OUTS with REP prefix? */
12325 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12326 }
12327 else if ( !fDbgStepping
12328 && fGstStepping)
12329 {
12330 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
12331 AssertRCReturn(rc, rc);
12332 }
12333
12334 /*
12335 * If any I/O breakpoints are armed, we need to check if one triggered
12336 * and take appropriate action.
12337 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12338 */
12339 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12340 AssertRCReturn(rc, rc);
12341
12342 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12343 * execution engines about whether hyper BPs and such are pending. */
12344 uint32_t const uDr7 = pMixedCtx->dr[7];
12345 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12346 && X86_DR7_ANY_RW_IO(uDr7)
12347 && (pMixedCtx->cr4 & X86_CR4_DE))
12348 || DBGFBpIsHwIoArmed(pVM)))
12349 {
12350 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12351
12352 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12353 VMMRZCallRing3Disable(pVCpu);
12354 HM_DISABLE_PREEMPT();
12355
12356 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12357
12358 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12359 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12360 {
12361 /* Raise #DB. */
12362 if (fIsGuestDbgActive)
12363 ASMSetDR6(pMixedCtx->dr[6]);
12364 if (pMixedCtx->dr[7] != uDr7)
12365 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_DR7;
12366
12367 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12368 }
12369 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12370 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12371 else if ( rcStrict2 != VINF_SUCCESS
12372 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12373 rcStrict = rcStrict2;
12374 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12375
12376 HM_RESTORE_PREEMPT();
12377 VMMRZCallRing3Enable(pVCpu);
12378 }
12379 }
12380
12381#ifdef VBOX_STRICT
12382 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12383 Assert(!fIOWrite);
12384 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12385 Assert(fIOWrite);
12386 else
12387 {
12388# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12389 * statuses, that the VMM device and some others may return. See
12390 * IOM_SUCCESS() for guidance. */
12391 AssertMsg( RT_FAILURE(rcStrict)
12392 || rcStrict == VINF_SUCCESS
12393 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12394 || rcStrict == VINF_EM_DBG_BREAKPOINT
12395 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12396 || rcStrict == VINF_EM_RAW_TO_R3
12397 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12398# endif
12399 }
12400#endif
12401 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12402 }
12403 else
12404 {
12405 /*
12406 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12407 */
12408 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12409 AssertRCReturn(rc2, rc2);
12410 STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
12411 : fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
12412 Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12413 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12414 VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification) ? "REP " : "",
12415 fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
12416
12417 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12418 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12419
12420 Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12421 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12422 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12423 }
12424 return rcStrict;
12425}
12426
12427
12428/**
12429 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12430 * VM-exit.
12431 */
12432HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12433{
12434 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12435
12436 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12437 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12438 AssertRCReturn(rc, rc);
12439 if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12440 {
12441 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12442 AssertRCReturn(rc, rc);
12443 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12444 {
12445 uint32_t uErrCode;
12446 RTGCUINTPTR GCPtrFaultAddress;
12447 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12448 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12449 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12450 if (fErrorCodeValid)
12451 {
12452 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12453 AssertRCReturn(rc, rc);
12454 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12455 }
12456 else
12457 uErrCode = 0;
12458
12459 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12460 && uVector == X86_XCPT_PF)
12461 GCPtrFaultAddress = pMixedCtx->cr2;
12462 else
12463 GCPtrFaultAddress = 0;
12464
12465 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12466 0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
12467
12468 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12469 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12470 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12471 }
12472 }
12473
12474 /* Fall back to the interpreter to emulate the task-switch. */
12475 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12476 return VERR_EM_INTERPRETER;
12477}
12478
12479
12480/**
12481 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12482 */
12483HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12484{
12485 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12486 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12487 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12488 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12489 AssertRCReturn(rc, rc);
12490 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12491 return VINF_EM_DBG_STEPPED;
12492}
12493
12494
12495/**
12496 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12497 */
12498HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12499{
12500 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12501
12502 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12503
12504 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12505 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12506 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12507 {
12508 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12509 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12510 {
12511 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12512 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12513 }
12514 }
12515 else
12516 {
12517 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12518 rcStrict1 = VINF_SUCCESS;
12519 return rcStrict1;
12520 }
12521
12522 /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12523 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12524 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12525 AssertRCReturn(rc, rc);
12526
12527 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12528 uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12529 VBOXSTRICTRC rcStrict2;
12530 switch (uAccessType)
12531 {
12532 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12533 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12534 {
12535 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12536 || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
12537 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12538
12539 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12540 GCPhys &= PAGE_BASE_GC_MASK;
12541 GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12542 PVM pVM = pVCpu->CTX_SUFF(pVM);
12543 Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12544 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12545
12546 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12547 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12548 CPUMCTX2CORE(pMixedCtx), GCPhys);
12549 Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12550 if ( rcStrict2 == VINF_SUCCESS
12551 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12552 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12553 {
12554 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12555 | HM_CHANGED_GUEST_RSP
12556 | HM_CHANGED_GUEST_RFLAGS
12557 | HM_CHANGED_GUEST_APIC_TPR);
12558 rcStrict2 = VINF_SUCCESS;
12559 }
12560 break;
12561 }
12562
12563 default:
12564 Log4Func(("uAccessType=%#x\n", uAccessType));
12565 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12566 break;
12567 }
12568
12569 if (rcStrict2 != VINF_SUCCESS)
12570 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12571 return rcStrict2;
12572}
12573
12574
12575/**
12576 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12577 * VM-exit.
12578 */
12579HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12580{
12581 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12582
12583 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12584 if (pVmxTransient->fWasGuestDebugStateActive)
12585 {
12586 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12587 HMVMX_RETURN_UNEXPECTED_EXIT();
12588 }
12589
12590 if ( !pVCpu->hm.s.fSingleInstruction
12591 && !pVmxTransient->fWasHyperDebugStateActive)
12592 {
12593 Assert(!DBGFIsStepping(pVCpu));
12594 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12595
12596 /* Don't intercept MOV DRx any more. */
12597 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12598 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12599 AssertRCReturn(rc, rc);
12600
12601 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12602 VMMRZCallRing3Disable(pVCpu);
12603 HM_DISABLE_PREEMPT();
12604
12605 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12606 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12607 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12608
12609 HM_RESTORE_PREEMPT();
12610 VMMRZCallRing3Enable(pVCpu);
12611
12612#ifdef VBOX_WITH_STATISTICS
12613 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12614 AssertRCReturn(rc, rc);
12615 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12616 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12617 else
12618 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12619#endif
12620 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12621 return VINF_SUCCESS;
12622 }
12623
12624 /*
12625 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12626 * Update the segment registers and DR7 from the CPU.
12627 */
12628 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12629 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK
12630 | CPUMCTX_EXTRN_DR7);
12631 AssertRCReturn(rc, rc);
12632 Log4Func(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12633
12634 PVM pVM = pVCpu->CTX_SUFF(pVM);
12635 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12636 {
12637 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12638 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification),
12639 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification));
12640 if (RT_SUCCESS(rc))
12641 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12642 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12643 }
12644 else
12645 {
12646 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12647 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification),
12648 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification));
12649 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12650 }
12651
12652 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12653 if (RT_SUCCESS(rc))
12654 {
12655 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12656 AssertRCReturn(rc2, rc2);
12657 return VINF_SUCCESS;
12658 }
12659 return rc;
12660}
12661
12662
12663/**
12664 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12665 * Conditional VM-exit.
12666 */
12667HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12668{
12669 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12670 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12671
12672 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12673 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12674 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12675 {
12676 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12677 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12678 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12679 {
12680 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12681 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12682 }
12683 }
12684 else
12685 {
12686 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12687 rcStrict1 = VINF_SUCCESS;
12688 return rcStrict1;
12689 }
12690
12691 /*
12692 * Get sufficent state and update the exit history entry.
12693 */
12694 RTGCPHYS GCPhys;
12695 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12696 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12697 AssertRCReturn(rc, rc);
12698
12699 VBOXSTRICTRC rcStrict;
12700 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12701 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
12702 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12703 if (!pExitRec)
12704 {
12705 /*
12706 * If we succeed, resume guest execution.
12707 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12708 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12709 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12710 * weird case. See @bugref{6043}.
12711 */
12712 PVM pVM = pVCpu->CTX_SUFF(pVM);
12713 rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12714 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12715 if ( rcStrict == VINF_SUCCESS
12716 || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12717 || rcStrict == VERR_PAGE_NOT_PRESENT)
12718 {
12719 /* Successfully handled MMIO operation. */
12720 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12721 | HM_CHANGED_GUEST_RSP
12722 | HM_CHANGED_GUEST_RFLAGS
12723 | HM_CHANGED_GUEST_APIC_TPR);
12724 rcStrict = VINF_SUCCESS;
12725 }
12726 }
12727 else
12728 {
12729 /*
12730 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12731 */
12732 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
12733 int rc2 = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12734 AssertRCReturn(rc2, rc2);
12735
12736 Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12737 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12738
12739 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12740 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12741
12742 Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12743 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12744 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12745 }
12746 return VBOXSTRICTRC_TODO(rcStrict);
12747}
12748
12749
12750/**
12751 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12752 * VM-exit.
12753 */
12754HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12755{
12756 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12757 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12758
12759 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12760 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12761 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12762 {
12763 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12764 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12765 Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12766 }
12767 else
12768 {
12769 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12770 rcStrict1 = VINF_SUCCESS;
12771 return rcStrict1;
12772 }
12773
12774 RTGCPHYS GCPhys;
12775 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12776 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12777 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12778 AssertRCReturn(rc, rc);
12779
12780 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12781 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12782
12783 RTGCUINT uErrorCode = 0;
12784 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
12785 uErrorCode |= X86_TRAP_PF_ID;
12786 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_DATA_WRITE)
12787 uErrorCode |= X86_TRAP_PF_RW;
12788 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
12789 uErrorCode |= X86_TRAP_PF_P;
12790
12791 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12792
12793 Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12794 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12795
12796 /* Handle the pagefault trap for the nested shadow table. */
12797 PVM pVM = pVCpu->CTX_SUFF(pVM);
12798 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12799 TRPMResetTrap(pVCpu);
12800
12801 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12802 if ( rcStrict2 == VINF_SUCCESS
12803 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12804 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12805 {
12806 /* Successfully synced our nested page tables. */
12807 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12808 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12809 | HM_CHANGED_GUEST_RSP
12810 | HM_CHANGED_GUEST_RFLAGS);
12811 return VINF_SUCCESS;
12812 }
12813
12814 Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12815 return rcStrict2;
12816}
12817
12818/** @} */
12819
12820/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12821/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
12822/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12823
12824/** @name VM-exit exception handlers.
12825 * @{
12826 */
12827
12828/**
12829 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12830 */
12831static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12832{
12833 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12834 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12835
12836 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
12837 AssertRCReturn(rc, rc);
12838
12839 if (!(pMixedCtx->cr0 & X86_CR0_NE))
12840 {
12841 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12842 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12843
12844 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12845 * provides VM-exit instruction length. If this causes problem later,
12846 * disassemble the instruction like it's done on AMD-V. */
12847 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12848 AssertRCReturn(rc2, rc2);
12849 return rc;
12850 }
12851
12852 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12853 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12854 return rc;
12855}
12856
12857
12858/**
12859 * VM-exit exception handler for \#BP (Breakpoint exception).
12860 */
12861static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12862{
12863 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12864 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12865
12866 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12867 AssertRCReturn(rc, rc);
12868
12869 rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx));
12870 if (rc == VINF_EM_RAW_GUEST_TRAP)
12871 {
12872 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12873 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12874 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12875 AssertRCReturn(rc, rc);
12876
12877 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12878 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12879 }
12880
12881 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
12882 return rc;
12883}
12884
12885
12886/**
12887 * VM-exit exception handler for \#AC (alignment check exception).
12888 */
12889static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12890{
12891 RT_NOREF_PV(pMixedCtx);
12892 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12893
12894 /*
12895 * Re-inject it. We'll detect any nesting before getting here.
12896 */
12897 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12898 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12899 AssertRCReturn(rc, rc);
12900 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
12901
12902 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12903 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12904 return VINF_SUCCESS;
12905}
12906
12907
12908/**
12909 * VM-exit exception handler for \#DB (Debug exception).
12910 */
12911static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12912{
12913 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12914 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
12915
12916 /*
12917 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
12918 * for processing.
12919 */
12920 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12921
12922 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
12923 uint64_t uDR6 = X86_DR6_INIT_VAL;
12924 uDR6 |= ( pVmxTransient->uExitQualification
12925 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
12926
12927 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
12928 Log6Func(("rc=%Rrc\n", rc));
12929 if (rc == VINF_EM_RAW_GUEST_TRAP)
12930 {
12931 /*
12932 * The exception was for the guest. Update DR6, DR7.GD and
12933 * IA32_DEBUGCTL.LBR before forwarding it.
12934 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
12935 */
12936 VMMRZCallRing3Disable(pVCpu);
12937 HM_DISABLE_PREEMPT();
12938
12939 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
12940 pMixedCtx->dr[6] |= uDR6;
12941 if (CPUMIsGuestDebugStateActive(pVCpu))
12942 ASMSetDR6(pMixedCtx->dr[6]);
12943
12944 HM_RESTORE_PREEMPT();
12945 VMMRZCallRing3Enable(pVCpu);
12946
12947 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12948 AssertRCReturn(rc, rc);
12949
12950 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
12951 pMixedCtx->dr[7] &= ~X86_DR7_GD;
12952
12953 /* Paranoia. */
12954 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
12955 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
12956
12957 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
12958 AssertRCReturn(rc, rc);
12959
12960 /*
12961 * Raise #DB in the guest.
12962 *
12963 * It is important to reflect exactly what the VM-exit gave us (preserving the
12964 * interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
12965 * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
12966 * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
12967 *
12968 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
12969 * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
12970 */
12971 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12972 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12973 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12974 AssertRCReturn(rc, rc);
12975 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12976 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12977 return VINF_SUCCESS;
12978 }
12979
12980 /*
12981 * Not a guest trap, must be a hypervisor related debug event then.
12982 * Update DR6 in case someone is interested in it.
12983 */
12984 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
12985 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
12986 CPUMSetHyperDR6(pVCpu, uDR6);
12987
12988 return rc;
12989}
12990
12991/**
12992 * VM-exit exception handler for \#GP (General-protection exception).
12993 *
12994 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
12995 */
12996static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12997{
12998 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12999 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13000
13001 int rc;
13002 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13003 { /* likely */ }
13004 else
13005 {
13006#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13007 Assert(pVCpu->hm.s.fUsingDebugLoop);
13008#endif
13009 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13010 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13011 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13012 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13013 rc |= hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13014 AssertRCReturn(rc, rc);
13015 Log4Func(("Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13016 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13017 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13018 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13019 return rc;
13020 }
13021
13022 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13023 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13024
13025 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13026 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13027 AssertRCReturn(rc, rc);
13028
13029 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13030 uint32_t cbOp = 0;
13031 PVM pVM = pVCpu->CTX_SUFF(pVM);
13032 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13033 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13034 if (RT_SUCCESS(rc))
13035 {
13036 rc = VINF_SUCCESS;
13037 Assert(cbOp == pDis->cbInstr);
13038 Log4Func(("Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13039 switch (pDis->pCurInstr->uOpcode)
13040 {
13041 case OP_CLI:
13042 {
13043 pMixedCtx->eflags.Bits.u1IF = 0;
13044 pMixedCtx->eflags.Bits.u1RF = 0;
13045 pMixedCtx->rip += pDis->cbInstr;
13046 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13047 if ( !fDbgStepping
13048 && pMixedCtx->eflags.Bits.u1TF)
13049 {
13050 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13051 AssertRCReturn(rc, rc);
13052 }
13053 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13054 break;
13055 }
13056
13057 case OP_STI:
13058 {
13059 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13060 pMixedCtx->eflags.Bits.u1IF = 1;
13061 pMixedCtx->eflags.Bits.u1RF = 0;
13062 pMixedCtx->rip += pDis->cbInstr;
13063 if (!fOldIF)
13064 {
13065 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13066 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13067 }
13068 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13069 if ( !fDbgStepping
13070 && pMixedCtx->eflags.Bits.u1TF)
13071 {
13072 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13073 AssertRCReturn(rc, rc);
13074 }
13075 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13076 break;
13077 }
13078
13079 case OP_HLT:
13080 {
13081 rc = VINF_EM_HALT;
13082 pMixedCtx->rip += pDis->cbInstr;
13083 pMixedCtx->eflags.Bits.u1RF = 0;
13084 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13085 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13086 break;
13087 }
13088
13089 case OP_POPF:
13090 {
13091 Log4Func(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13092 uint32_t cbParm;
13093 uint32_t uMask;
13094 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13095 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13096 {
13097 cbParm = 4;
13098 uMask = 0xffffffff;
13099 }
13100 else
13101 {
13102 cbParm = 2;
13103 uMask = 0xffff;
13104 }
13105
13106 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13107 RTGCPTR GCPtrStack = 0;
13108 X86EFLAGS Eflags;
13109 Eflags.u32 = 0;
13110 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13111 &GCPtrStack);
13112 if (RT_SUCCESS(rc))
13113 {
13114 Assert(sizeof(Eflags.u32) >= cbParm);
13115 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13116 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13117 }
13118 if (RT_FAILURE(rc))
13119 {
13120 rc = VERR_EM_INTERPRETER;
13121 break;
13122 }
13123 Log4Func(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13124 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13125 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13126 pMixedCtx->esp += cbParm;
13127 pMixedCtx->esp &= uMask;
13128 pMixedCtx->rip += pDis->cbInstr;
13129 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13130 | HM_CHANGED_GUEST_RSP
13131 | HM_CHANGED_GUEST_RFLAGS);
13132 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13133 POPF restores EFLAGS.TF. */
13134 if ( !fDbgStepping
13135 && fGstStepping)
13136 {
13137 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13138 AssertRCReturn(rc, rc);
13139 }
13140 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13141 break;
13142 }
13143
13144 case OP_PUSHF:
13145 {
13146 uint32_t cbParm;
13147 uint32_t uMask;
13148 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13149 {
13150 cbParm = 4;
13151 uMask = 0xffffffff;
13152 }
13153 else
13154 {
13155 cbParm = 2;
13156 uMask = 0xffff;
13157 }
13158
13159 /* Get the stack pointer & push the contents of eflags onto the stack. */
13160 RTGCPTR GCPtrStack = 0;
13161 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13162 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13163 if (RT_FAILURE(rc))
13164 {
13165 rc = VERR_EM_INTERPRETER;
13166 break;
13167 }
13168 X86EFLAGS Eflags = pMixedCtx->eflags;
13169 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13170 Eflags.Bits.u1RF = 0;
13171 Eflags.Bits.u1VM = 0;
13172
13173 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13174 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13175 {
13176 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13177 rc = VERR_EM_INTERPRETER;
13178 break;
13179 }
13180 Log4Func(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13181 pMixedCtx->esp -= cbParm;
13182 pMixedCtx->esp &= uMask;
13183 pMixedCtx->rip += pDis->cbInstr;
13184 pMixedCtx->eflags.Bits.u1RF = 0;
13185 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13186 | HM_CHANGED_GUEST_RSP
13187 | HM_CHANGED_GUEST_RFLAGS);
13188 if ( !fDbgStepping
13189 && pMixedCtx->eflags.Bits.u1TF)
13190 {
13191 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13192 AssertRCReturn(rc, rc);
13193 }
13194 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13195 break;
13196 }
13197
13198 case OP_IRET:
13199 {
13200 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13201 * instruction reference. */
13202 RTGCPTR GCPtrStack = 0;
13203 uint32_t uMask = 0xffff;
13204 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13205 uint16_t aIretFrame[3];
13206 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13207 {
13208 rc = VERR_EM_INTERPRETER;
13209 break;
13210 }
13211 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13212 &GCPtrStack);
13213 if (RT_SUCCESS(rc))
13214 {
13215 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13216 PGMACCESSORIGIN_HM));
13217 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13218 }
13219 if (RT_FAILURE(rc))
13220 {
13221 rc = VERR_EM_INTERPRETER;
13222 break;
13223 }
13224 pMixedCtx->eip = 0;
13225 pMixedCtx->ip = aIretFrame[0];
13226 pMixedCtx->cs.Sel = aIretFrame[1];
13227 pMixedCtx->cs.ValidSel = aIretFrame[1];
13228 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13229 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13230 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13231 pMixedCtx->sp += sizeof(aIretFrame);
13232 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13233 | HM_CHANGED_GUEST_CS
13234 | HM_CHANGED_GUEST_RSP
13235 | HM_CHANGED_GUEST_RFLAGS);
13236 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13237 if ( !fDbgStepping
13238 && fGstStepping)
13239 {
13240 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13241 AssertRCReturn(rc, rc);
13242 }
13243 Log4Func(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13244 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13245 break;
13246 }
13247
13248 case OP_INT:
13249 {
13250 uint16_t uVector = pDis->Param1.uValue & 0xff;
13251 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13252 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13253 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13254 break;
13255 }
13256
13257 case OP_INTO:
13258 {
13259 if (pMixedCtx->eflags.Bits.u1OF)
13260 {
13261 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13262 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13263 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13264 }
13265 else
13266 {
13267 pMixedCtx->eflags.Bits.u1RF = 0;
13268 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
13269 }
13270 break;
13271 }
13272
13273 default:
13274 {
13275 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13276 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13277 EMCODETYPE_SUPERVISOR);
13278 rc = VBOXSTRICTRC_VAL(rc2);
13279 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13280 /** @todo We have to set pending-debug exceptions here when the guest is
13281 * single-stepping depending on the instruction that was interpreted. */
13282 Log4Func(("#GP rc=%Rrc\n", rc));
13283 break;
13284 }
13285 }
13286 }
13287 else
13288 rc = VERR_EM_INTERPRETER;
13289
13290 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13291 ("#GP Unexpected rc=%Rrc\n", rc));
13292 return rc;
13293}
13294
13295
13296/**
13297 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13298 * the exception reported in the VMX transient structure back into the VM.
13299 *
13300 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13301 * up-to-date.
13302 */
13303static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13304{
13305 RT_NOREF_PV(pMixedCtx);
13306 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13307#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13308 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13309 ("uVector=%#x u32XcptBitmap=%#X32\n",
13310 VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13311#endif
13312
13313 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13314 hmR0VmxCheckExitDueToEventDelivery(). */
13315 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13316 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13317 AssertRCReturn(rc, rc);
13318 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13319
13320#ifdef DEBUG_ramshankar
13321 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS
13322 | CPUMCTX_EXTRN_RIP);
13323 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13324 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13325#endif
13326
13327 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13328 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13329 return VINF_SUCCESS;
13330}
13331
13332
13333/**
13334 * VM-exit exception handler for \#PF (Page-fault exception).
13335 */
13336static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13337{
13338 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13339 PVM pVM = pVCpu->CTX_SUFF(pVM);
13340 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13341 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13342 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13343 AssertRCReturn(rc, rc);
13344
13345 if (!pVM->hm.s.fNestedPaging)
13346 { /* likely */ }
13347 else
13348 {
13349#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13350 Assert(pVCpu->hm.s.fUsingDebugLoop);
13351#endif
13352 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13353 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13354 {
13355 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13356 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13357 }
13358 else
13359 {
13360 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13361 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13362 Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13363 }
13364 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13365 return rc;
13366 }
13367
13368 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13369 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13370 if (pVmxTransient->fVectoringPF)
13371 {
13372 Assert(pVCpu->hm.s.Event.fPending);
13373 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13374 }
13375
13376 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13377 AssertRCReturn(rc, rc);
13378
13379 Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13380 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13381
13382 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13383 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13384 (RTGCPTR)pVmxTransient->uExitQualification);
13385
13386 Log4Func(("#PF: rc=%Rrc\n", rc));
13387 if (rc == VINF_SUCCESS)
13388 {
13389 /*
13390 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13391 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13392 */
13393 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13394 TRPMResetTrap(pVCpu);
13395 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13396 return rc;
13397 }
13398
13399 if (rc == VINF_EM_RAW_GUEST_TRAP)
13400 {
13401 if (!pVmxTransient->fVectoringDoublePF)
13402 {
13403 /* It's a guest page fault and needs to be reflected to the guest. */
13404 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13405 TRPMResetTrap(pVCpu);
13406 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13407 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13408 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13409 }
13410 else
13411 {
13412 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13413 TRPMResetTrap(pVCpu);
13414 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13415 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13416 Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13417 }
13418
13419 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13420 return VINF_SUCCESS;
13421 }
13422
13423 TRPMResetTrap(pVCpu);
13424 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13425 return rc;
13426}
13427
13428/** @} */
13429
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette