VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 72798

Last change on this file since 72798 was 72798, checked in by vboxsync, 6 years ago

VMM/HMVMXR0: Don't leave hmR0VmxImportGuestState with interrupts enabled on error.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 573.2 KB
Line 
1/* $Id: HMVMXR0.cpp 72798 2018-07-02 16:06:17Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/x86.h>
25#include <iprt/asm-amd64-x86.h>
26#include <iprt/thread.h>
27
28#include <VBox/vmm/pdmapi.h>
29#include <VBox/vmm/dbgf.h>
30#include <VBox/vmm/iem.h>
31#include <VBox/vmm/iom.h>
32#include <VBox/vmm/selm.h>
33#include <VBox/vmm/tm.h>
34#include <VBox/vmm/gim.h>
35#include <VBox/vmm/apic.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#include "HMInternal.h"
40#include <VBox/vmm/vm.h>
41#include "HMVMXR0.h"
42#include "dtrace/VBoxVMM.h"
43
44#define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
45
46#ifdef DEBUG_ramshankar
47# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
48# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
49# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
50# define HMVMX_ALWAYS_CHECK_GUEST_STATE
51# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
52# define HMVMX_ALWAYS_TRAP_PF
53# define HMVMX_ALWAYS_FLUSH_TLB
54# define HMVMX_ALWAYS_SWAP_EFER
55#endif
56
57
58/*********************************************************************************************************************************
59* Defined Constants And Macros *
60*********************************************************************************************************************************/
61/** Use the function table. */
62#define HMVMX_USE_FUNCTION_TABLE
63
64/** Determine which tagged-TLB flush handler to use. */
65#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70/** @name HMVMX_READ_XXX
71 * Flags to skip redundant reads of some common VMCS fields that are not part of
72 * the guest-CPU or VCPU state but are needed while handling VM-exits.
73 */
74#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
75#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
76#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
77#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
78#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
79#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
80#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
81/** @} */
82
83/**
84 * States of the VMCS.
85 *
86 * This does not reflect all possible VMCS states but currently only those
87 * needed for maintaining the VMCS consistently even when thread-context hooks
88 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
89 */
90#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
91#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
92#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
93
94/**
95 * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
96 * guest using hardware-assisted VMX.
97 *
98 * This excludes state like GPRs (other than RSP) which are always are
99 * swapped and restored across the world-switch and also registers like EFER,
100 * MSR which cannot be modified by the guest without causing a VM-exit.
101 */
102#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
103 | CPUMCTX_EXTRN_RFLAGS \
104 | CPUMCTX_EXTRN_RSP \
105 | CPUMCTX_EXTRN_SREG_MASK \
106 | CPUMCTX_EXTRN_TABLE_MASK \
107 | CPUMCTX_EXTRN_KERNEL_GS_BASE \
108 | CPUMCTX_EXTRN_SYSCALL_MSRS \
109 | CPUMCTX_EXTRN_SYSENTER_MSRS \
110 | CPUMCTX_EXTRN_TSC_AUX \
111 | CPUMCTX_EXTRN_OTHER_MSRS \
112 | CPUMCTX_EXTRN_CR0 \
113 | CPUMCTX_EXTRN_CR3 \
114 | CPUMCTX_EXTRN_CR4 \
115 | CPUMCTX_EXTRN_DR7 \
116 | CPUMCTX_EXTRN_HM_VMX_MASK)
117
118/**
119 * Exception bitmap mask for real-mode guests (real-on-v86).
120 *
121 * We need to intercept all exceptions manually except:
122 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
123 * due to bugs in Intel CPUs.
124 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
125 * support.
126 */
127#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
128 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
129 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
130 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
131 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
132 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
133 | RT_BIT(X86_XCPT_XF))
134
135/**
136 * Exception bitmap mask for all contributory exceptions.
137 *
138 * Page fault is deliberately excluded here as it's conditional as to whether
139 * it's contributory or benign. Page faults are handled separately.
140 */
141#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
142 | RT_BIT(X86_XCPT_DE))
143
144/** Maximum VM-instruction error number. */
145#define HMVMX_INSTR_ERROR_MAX 28
146
147/** Profiling macro. */
148#ifdef HM_PROFILE_EXIT_DISPATCH
149# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
150# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
151#else
152# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
153# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
154#endif
155
156/** Assert that preemption is disabled or covered by thread-context hooks. */
157#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
158 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
159
160/** Assert that we haven't migrated CPUs when thread-context hooks are not
161 * used. */
162#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
163 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
164 ("Illegal migration! Entered on CPU %u Current %u\n", \
165 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
166
167/** Helper macro for VM-exit handlers called unexpectedly. */
168#define HMVMX_RETURN_UNEXPECTED_EXIT() \
169 do { \
170 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
171 return VERR_VMX_UNEXPECTED_EXIT; \
172 } while (0)
173
174/** Macro for saving segment registers from VMCS into the guest-CPU
175 * context. */
176#ifdef VMX_USE_CACHED_VMCS_ACCESSES
177# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
178 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
179 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
180#else
181# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
182 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
183 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
184#endif
185
186
187/*********************************************************************************************************************************
188* Structures and Typedefs *
189*********************************************************************************************************************************/
190/**
191 * VMX transient state.
192 *
193 * A state structure for holding miscellaneous information across
194 * VMX non-root operation and restored after the transition.
195 */
196typedef struct VMXTRANSIENT
197{
198 /** The host's rflags/eflags. */
199 RTCCUINTREG fEFlags;
200#if HC_ARCH_BITS == 32
201 uint32_t u32Alignment0;
202#endif
203 /** The guest's TPR value used for TPR shadowing. */
204 uint8_t u8GuestTpr;
205 /** Alignment. */
206 uint8_t abAlignment0[7];
207
208 /** The basic VM-exit reason. */
209 uint16_t uExitReason;
210 /** Alignment. */
211 uint16_t u16Alignment0;
212 /** The VM-exit interruption error code. */
213 uint32_t uExitIntErrorCode;
214 /** The VM-exit exit code qualification. */
215 uint64_t uExitQualification;
216
217 /** The VM-exit interruption-information field. */
218 uint32_t uExitIntInfo;
219 /** The VM-exit instruction-length field. */
220 uint32_t cbInstr;
221 /** The VM-exit instruction-information field. */
222 union
223 {
224 /** Plain unsigned int representation. */
225 uint32_t u;
226 /** INS and OUTS information. */
227 struct
228 {
229 uint32_t u7Reserved0 : 7;
230 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
231 uint32_t u3AddrSize : 3;
232 uint32_t u5Reserved1 : 5;
233 /** The segment register (X86_SREG_XXX). */
234 uint32_t iSegReg : 3;
235 uint32_t uReserved2 : 14;
236 } StrIo;
237 /** INVEPT, INVVPID, INVPCID information. */
238 struct
239 {
240 /** Scaling; 0=no scaling, 1=scale-by-2, 2=scale-by-4, 3=scale-by-8. */
241 uint32_t u2Scaling : 2;
242 uint32_t u5Reserved0 : 5;
243 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
244 uint32_t u3AddrSize : 3;
245 uint32_t u1Reserved0 : 1;
246 uint32_t u4Reserved0 : 4;
247 /** The segment register (X86_SREG_XXX). */
248 uint32_t iSegReg : 3;
249 /** The index register (X86_GREG_XXX). */
250 uint32_t iIdxReg : 4;
251 /** Set if index register is invalid. */
252 uint32_t fIdxRegValid : 1;
253 /** The base register (X86_GREG_XXX). */
254 uint32_t iBaseReg : 4;
255 /** Set if base register is invalid. */
256 uint32_t fBaseRegValid : 1;
257 /** Register 2 (X86_GREG_XXX). */
258 uint32_t iReg2 : 4;
259 } Inv;
260 } ExitInstrInfo;
261 /** Whether the VM-entry failed or not. */
262 bool fVMEntryFailed;
263 /** Alignment. */
264 uint8_t abAlignment1[3];
265
266 /** The VM-entry interruption-information field. */
267 uint32_t uEntryIntInfo;
268 /** The VM-entry exception error code field. */
269 uint32_t uEntryXcptErrorCode;
270 /** The VM-entry instruction length field. */
271 uint32_t cbEntryInstr;
272
273 /** IDT-vectoring information field. */
274 uint32_t uIdtVectoringInfo;
275 /** IDT-vectoring error code. */
276 uint32_t uIdtVectoringErrorCode;
277
278 /** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
279 uint32_t fVmcsFieldsRead;
280
281 /** Whether the guest debug state was active at the time of VM-exit. */
282 bool fWasGuestDebugStateActive;
283 /** Whether the hyper debug state was active at the time of VM-exit. */
284 bool fWasHyperDebugStateActive;
285 /** Whether TSC-offsetting should be setup before VM-entry. */
286 bool fUpdateTscOffsettingAndPreemptTimer;
287 /** Whether the VM-exit was caused by a page-fault during delivery of a
288 * contributory exception or a page-fault. */
289 bool fVectoringDoublePF;
290 /** Whether the VM-exit was caused by a page-fault during delivery of an
291 * external interrupt or NMI. */
292 bool fVectoringPF;
293} VMXTRANSIENT;
294AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
295AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
296AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
297AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
298AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
299/** Pointer to VMX transient state. */
300typedef VMXTRANSIENT *PVMXTRANSIENT;
301
302
303/**
304 * MSR-bitmap read permissions.
305 */
306typedef enum VMXMSREXITREAD
307{
308 /** Reading this MSR causes a VM-exit. */
309 VMXMSREXIT_INTERCEPT_READ = 0xb,
310 /** Reading this MSR does not cause a VM-exit. */
311 VMXMSREXIT_PASSTHRU_READ
312} VMXMSREXITREAD;
313/** Pointer to MSR-bitmap read permissions. */
314typedef VMXMSREXITREAD* PVMXMSREXITREAD;
315
316/**
317 * MSR-bitmap write permissions.
318 */
319typedef enum VMXMSREXITWRITE
320{
321 /** Writing to this MSR causes a VM-exit. */
322 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
323 /** Writing to this MSR does not cause a VM-exit. */
324 VMXMSREXIT_PASSTHRU_WRITE
325} VMXMSREXITWRITE;
326/** Pointer to MSR-bitmap write permissions. */
327typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
328
329
330/**
331 * VMX VM-exit handler.
332 *
333 * @returns Strict VBox status code (i.e. informational status codes too).
334 * @param pVCpu The cross context virtual CPU structure.
335 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
336 * out-of-sync. Make sure to update the required
337 * fields before using them.
338 * @param pVmxTransient Pointer to the VMX-transient structure.
339 */
340#ifndef HMVMX_USE_FUNCTION_TABLE
341typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
342#else
343typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
344/** Pointer to VM-exit handler. */
345typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
346#endif
347
348/**
349 * VMX VM-exit handler, non-strict status code.
350 *
351 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
352 *
353 * @returns VBox status code, no informational status code returned.
354 * @param pVCpu The cross context virtual CPU structure.
355 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
356 * out-of-sync. Make sure to update the required
357 * fields before using them.
358 * @param pVmxTransient Pointer to the VMX-transient structure.
359 *
360 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
361 * use of that status code will be replaced with VINF_EM_SOMETHING
362 * later when switching over to IEM.
363 */
364#ifndef HMVMX_USE_FUNCTION_TABLE
365typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
366#else
367typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
368#endif
369
370
371/*********************************************************************************************************************************
372* Internal Functions *
373*********************************************************************************************************************************/
374static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
375static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
376static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
377static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat);
378static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
379 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState);
380#if HC_ARCH_BITS == 32
381static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
382#endif
383#ifndef HMVMX_USE_FUNCTION_TABLE
384DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
385# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
386# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
387#else
388# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
389# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
390#endif
391
392
393/** @name VM-exit handlers.
394 * @{
395 */
396static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
397static FNVMXEXITHANDLER hmR0VmxExitExtInt;
398static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
399static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
400static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
401static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
402static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
403static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
404static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
405static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
406static FNVMXEXITHANDLER hmR0VmxExitCpuid;
407static FNVMXEXITHANDLER hmR0VmxExitGetsec;
408static FNVMXEXITHANDLER hmR0VmxExitHlt;
409static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
410static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
411static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
412static FNVMXEXITHANDLER hmR0VmxExitVmcall;
413static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
414static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
415static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
416static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
417static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
418static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
419static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
420static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
421static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
422static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
423static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
424static FNVMXEXITHANDLER hmR0VmxExitMwait;
425static FNVMXEXITHANDLER hmR0VmxExitMtf;
426static FNVMXEXITHANDLER hmR0VmxExitMonitor;
427static FNVMXEXITHANDLER hmR0VmxExitPause;
428static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
429static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
430static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
431static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
432static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
433static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
434static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
435static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
436static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
437static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
438static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
439static FNVMXEXITHANDLER hmR0VmxExitRdrand;
440static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
441/** @} */
442
443static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
444static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
445static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
446static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
447static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
448static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
449static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
450static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
451
452
453/*********************************************************************************************************************************
454* Global Variables *
455*********************************************************************************************************************************/
456#ifdef HMVMX_USE_FUNCTION_TABLE
457
458/**
459 * VMX_EXIT dispatch table.
460 */
461static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
462{
463 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
464 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
465 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
466 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
467 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
468 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
469 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
470 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
471 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
472 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
473 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
474 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
475 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
476 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
477 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
478 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
479 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
480 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
481 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
482 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
483 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
484 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
485 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
486 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
487 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
488 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
489 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
490 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
491 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
492 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
493 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
494 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
495 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
496 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
497 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
498 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
499 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
500 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
501 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
502 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
503 /* 40 UNDEFINED */ hmR0VmxExitPause,
504 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
505 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
506 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
507 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
508 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
509 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
510 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
511 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
512 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
513 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
514 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
515 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
516 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
517 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
518 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
519 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
520 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
521 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
522 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
523 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
524 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
525 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
526 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
527 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
528};
529#endif /* HMVMX_USE_FUNCTION_TABLE */
530
531#ifdef VBOX_STRICT
532static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
533{
534 /* 0 */ "(Not Used)",
535 /* 1 */ "VMCALL executed in VMX root operation.",
536 /* 2 */ "VMCLEAR with invalid physical address.",
537 /* 3 */ "VMCLEAR with VMXON pointer.",
538 /* 4 */ "VMLAUNCH with non-clear VMCS.",
539 /* 5 */ "VMRESUME with non-launched VMCS.",
540 /* 6 */ "VMRESUME after VMXOFF",
541 /* 7 */ "VM-entry with invalid control fields.",
542 /* 8 */ "VM-entry with invalid host state fields.",
543 /* 9 */ "VMPTRLD with invalid physical address.",
544 /* 10 */ "VMPTRLD with VMXON pointer.",
545 /* 11 */ "VMPTRLD with incorrect revision identifier.",
546 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
547 /* 13 */ "VMWRITE to read-only VMCS component.",
548 /* 14 */ "(Not Used)",
549 /* 15 */ "VMXON executed in VMX root operation.",
550 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
551 /* 17 */ "VM-entry with non-launched executing VMCS.",
552 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
553 /* 19 */ "VMCALL with non-clear VMCS.",
554 /* 20 */ "VMCALL with invalid VM-exit control fields.",
555 /* 21 */ "(Not Used)",
556 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
557 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
558 /* 24 */ "VMCALL with invalid SMM-monitor features.",
559 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
560 /* 26 */ "VM-entry with events blocked by MOV SS.",
561 /* 27 */ "(Not Used)",
562 /* 28 */ "Invalid operand to INVEPT/INVVPID."
563};
564#endif /* VBOX_STRICT */
565
566
567
568/**
569 * Updates the VM's last error record.
570 *
571 * If there was a VMX instruction error, reads the error data from the VMCS and
572 * updates VCPU's last error record as well.
573 *
574 * @param pVM The cross context VM structure.
575 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
576 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
577 * VERR_VMX_INVALID_VMCS_FIELD.
578 * @param rc The error code.
579 */
580static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
581{
582 AssertPtr(pVM);
583 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
584 || rc == VERR_VMX_UNABLE_TO_START_VM)
585 {
586 AssertPtrReturnVoid(pVCpu);
587 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
588 }
589 pVM->hm.s.lLastError = rc;
590}
591
592
593/**
594 * Reads the VM-entry interruption-information field from the VMCS into the VMX
595 * transient structure.
596 *
597 * @returns VBox status code.
598 * @param pVmxTransient Pointer to the VMX transient structure.
599 *
600 * @remarks No-long-jump zone!!!
601 */
602DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
603{
604 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
605 AssertRCReturn(rc, rc);
606 return VINF_SUCCESS;
607}
608
609#ifdef VBOX_STRICT
610/**
611 * Reads the VM-entry exception error code field from the VMCS into
612 * the VMX transient structure.
613 *
614 * @returns VBox status code.
615 * @param pVmxTransient Pointer to the VMX transient structure.
616 *
617 * @remarks No-long-jump zone!!!
618 */
619DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
620{
621 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
622 AssertRCReturn(rc, rc);
623 return VINF_SUCCESS;
624}
625
626
627/**
628 * Reads the VM-entry exception error code field from the VMCS into
629 * the VMX transient structure.
630 *
631 * @returns VBox status code.
632 * @param pVmxTransient Pointer to the VMX transient structure.
633 *
634 * @remarks No-long-jump zone!!!
635 */
636DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
637{
638 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
639 AssertRCReturn(rc, rc);
640 return VINF_SUCCESS;
641}
642#endif /* VBOX_STRICT */
643
644
645/**
646 * Reads the VM-exit interruption-information field from the VMCS into the VMX
647 * transient structure.
648 *
649 * @returns VBox status code.
650 * @param pVmxTransient Pointer to the VMX transient structure.
651 */
652DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
653{
654 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
655 {
656 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
657 AssertRCReturn(rc,rc);
658 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_INFO;
659 }
660 return VINF_SUCCESS;
661}
662
663
664/**
665 * Reads the VM-exit interruption error code from the VMCS into the VMX
666 * transient structure.
667 *
668 * @returns VBox status code.
669 * @param pVmxTransient Pointer to the VMX transient structure.
670 */
671DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
672{
673 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
674 {
675 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
676 AssertRCReturn(rc, rc);
677 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
678 }
679 return VINF_SUCCESS;
680}
681
682
683/**
684 * Reads the VM-exit instruction length field from the VMCS into the VMX
685 * transient structure.
686 *
687 * @returns VBox status code.
688 * @param pVmxTransient Pointer to the VMX transient structure.
689 */
690DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
691{
692 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
693 {
694 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
695 AssertRCReturn(rc, rc);
696 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_LEN;
697 }
698 return VINF_SUCCESS;
699}
700
701
702/**
703 * Reads the VM-exit instruction-information field from the VMCS into
704 * the VMX transient structure.
705 *
706 * @returns VBox status code.
707 * @param pVmxTransient Pointer to the VMX transient structure.
708 */
709DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
710{
711 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
712 {
713 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
714 AssertRCReturn(rc, rc);
715 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_INFO;
716 }
717 return VINF_SUCCESS;
718}
719
720
721/**
722 * Reads the exit code qualification from the VMCS into the VMX transient
723 * structure.
724 *
725 * @returns VBox status code.
726 * @param pVCpu The cross context virtual CPU structure of the
727 * calling EMT. (Required for the VMCS cache case.)
728 * @param pVmxTransient Pointer to the VMX transient structure.
729 */
730DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
731{
732 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
733 {
734 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
735 AssertRCReturn(rc, rc);
736 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION;
737 }
738 return VINF_SUCCESS;
739}
740
741
742/**
743 * Reads the IDT-vectoring information field from the VMCS into the VMX
744 * transient structure.
745 *
746 * @returns VBox status code.
747 * @param pVmxTransient Pointer to the VMX transient structure.
748 *
749 * @remarks No-long-jump zone!!!
750 */
751DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
752{
753 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
754 {
755 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
756 AssertRCReturn(rc, rc);
757 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_INFO;
758 }
759 return VINF_SUCCESS;
760}
761
762
763/**
764 * Reads the IDT-vectoring error code from the VMCS into the VMX
765 * transient structure.
766 *
767 * @returns VBox status code.
768 * @param pVmxTransient Pointer to the VMX transient structure.
769 */
770DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
771{
772 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
773 {
774 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
775 AssertRCReturn(rc, rc);
776 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
777 }
778 return VINF_SUCCESS;
779}
780
781
782/**
783 * Enters VMX root mode operation on the current CPU.
784 *
785 * @returns VBox status code.
786 * @param pVM The cross context VM structure. Can be
787 * NULL, after a resume.
788 * @param HCPhysCpuPage Physical address of the VMXON region.
789 * @param pvCpuPage Pointer to the VMXON region.
790 */
791static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
792{
793 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
794 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
795 Assert(pvCpuPage);
796 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
797
798 if (pVM)
799 {
800 /* Write the VMCS revision dword to the VMXON region. */
801 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
802 }
803
804 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
805 RTCCUINTREG fEFlags = ASMIntDisableFlags();
806
807 /* Enable the VMX bit in CR4 if necessary. */
808 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
809
810 /* Enter VMX root mode. */
811 int rc = VMXEnable(HCPhysCpuPage);
812 if (RT_FAILURE(rc))
813 {
814 if (!(uOldCr4 & X86_CR4_VMXE))
815 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
816
817 if (pVM)
818 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
819 }
820
821 /* Restore interrupts. */
822 ASMSetFlags(fEFlags);
823 return rc;
824}
825
826
827/**
828 * Exits VMX root mode operation on the current CPU.
829 *
830 * @returns VBox status code.
831 */
832static int hmR0VmxLeaveRootMode(void)
833{
834 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
835
836 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
837 RTCCUINTREG fEFlags = ASMIntDisableFlags();
838
839 /* If we're for some reason not in VMX root mode, then don't leave it. */
840 RTCCUINTREG uHostCR4 = ASMGetCR4();
841
842 int rc;
843 if (uHostCR4 & X86_CR4_VMXE)
844 {
845 /* Exit VMX root mode and clear the VMX bit in CR4. */
846 VMXDisable();
847 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
848 rc = VINF_SUCCESS;
849 }
850 else
851 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
852
853 /* Restore interrupts. */
854 ASMSetFlags(fEFlags);
855 return rc;
856}
857
858
859/**
860 * Allocates and maps one physically contiguous page. The allocated page is
861 * zero'd out. (Used by various VT-x structures).
862 *
863 * @returns IPRT status code.
864 * @param pMemObj Pointer to the ring-0 memory object.
865 * @param ppVirt Where to store the virtual address of the
866 * allocation.
867 * @param pHCPhys Where to store the physical address of the
868 * allocation.
869 */
870static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
871{
872 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
873 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
874 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
875
876 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
877 if (RT_FAILURE(rc))
878 return rc;
879 *ppVirt = RTR0MemObjAddress(*pMemObj);
880 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
881 ASMMemZero32(*ppVirt, PAGE_SIZE);
882 return VINF_SUCCESS;
883}
884
885
886/**
887 * Frees and unmaps an allocated physical page.
888 *
889 * @param pMemObj Pointer to the ring-0 memory object.
890 * @param ppVirt Where to re-initialize the virtual address of
891 * allocation as 0.
892 * @param pHCPhys Where to re-initialize the physical address of the
893 * allocation as 0.
894 */
895static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
896{
897 AssertPtr(pMemObj);
898 AssertPtr(ppVirt);
899 AssertPtr(pHCPhys);
900 if (*pMemObj != NIL_RTR0MEMOBJ)
901 {
902 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
903 AssertRC(rc);
904 *pMemObj = NIL_RTR0MEMOBJ;
905 *ppVirt = 0;
906 *pHCPhys = 0;
907 }
908}
909
910
911/**
912 * Worker function to free VT-x related structures.
913 *
914 * @returns IPRT status code.
915 * @param pVM The cross context VM structure.
916 */
917static void hmR0VmxStructsFree(PVM pVM)
918{
919 for (VMCPUID i = 0; i < pVM->cCpus; i++)
920 {
921 PVMCPU pVCpu = &pVM->aCpus[i];
922 AssertPtr(pVCpu);
923
924 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
925 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
926
927 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
928 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
929
930 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
931 }
932
933 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
934#ifdef VBOX_WITH_CRASHDUMP_MAGIC
935 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
936#endif
937}
938
939
940/**
941 * Worker function to allocate VT-x related VM structures.
942 *
943 * @returns IPRT status code.
944 * @param pVM The cross context VM structure.
945 */
946static int hmR0VmxStructsAlloc(PVM pVM)
947{
948 /*
949 * Initialize members up-front so we can cleanup properly on allocation failure.
950 */
951#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
952 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
953 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
954 pVM->hm.s.vmx.HCPhys##a_Name = 0;
955
956#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
957 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
958 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
959 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
960
961#ifdef VBOX_WITH_CRASHDUMP_MAGIC
962 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
963#endif
964 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
965
966 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
967 for (VMCPUID i = 0; i < pVM->cCpus; i++)
968 {
969 PVMCPU pVCpu = &pVM->aCpus[i];
970 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
971 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
972 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
973 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
974 }
975#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
976#undef VMXLOCAL_INIT_VM_MEMOBJ
977
978 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
979 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
980 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
981 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
982
983 /*
984 * Allocate all the VT-x structures.
985 */
986 int rc = VINF_SUCCESS;
987#ifdef VBOX_WITH_CRASHDUMP_MAGIC
988 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
989 if (RT_FAILURE(rc))
990 goto cleanup;
991 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
992 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
993#endif
994
995 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
996 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
997 {
998 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
999 &pVM->hm.s.vmx.HCPhysApicAccess);
1000 if (RT_FAILURE(rc))
1001 goto cleanup;
1002 }
1003
1004 /*
1005 * Initialize per-VCPU VT-x structures.
1006 */
1007 for (VMCPUID i = 0; i < pVM->cCpus; i++)
1008 {
1009 PVMCPU pVCpu = &pVM->aCpus[i];
1010 AssertPtr(pVCpu);
1011
1012 /* Allocate the VM control structure (VMCS). */
1013 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1014 if (RT_FAILURE(rc))
1015 goto cleanup;
1016
1017 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1018 if ( PDMHasApic(pVM)
1019 && (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW))
1020 {
1021 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1022 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1023 if (RT_FAILURE(rc))
1024 goto cleanup;
1025 }
1026
1027 /*
1028 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1029 * transparent accesses of specific MSRs.
1030 *
1031 * If the condition for enabling MSR bitmaps changes here, don't forget to
1032 * update HMAreMsrBitmapsAvailable().
1033 */
1034 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1035 {
1036 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1037 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1038 if (RT_FAILURE(rc))
1039 goto cleanup;
1040 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1041 }
1042
1043 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1044 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1045 if (RT_FAILURE(rc))
1046 goto cleanup;
1047
1048 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1049 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1050 if (RT_FAILURE(rc))
1051 goto cleanup;
1052 }
1053
1054 return VINF_SUCCESS;
1055
1056cleanup:
1057 hmR0VmxStructsFree(pVM);
1058 return rc;
1059}
1060
1061
1062/**
1063 * Does global VT-x initialization (called during module initialization).
1064 *
1065 * @returns VBox status code.
1066 */
1067VMMR0DECL(int) VMXR0GlobalInit(void)
1068{
1069#ifdef HMVMX_USE_FUNCTION_TABLE
1070 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1071# ifdef VBOX_STRICT
1072 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1073 Assert(g_apfnVMExitHandlers[i]);
1074# endif
1075#endif
1076 return VINF_SUCCESS;
1077}
1078
1079
1080/**
1081 * Does global VT-x termination (called during module termination).
1082 */
1083VMMR0DECL(void) VMXR0GlobalTerm()
1084{
1085 /* Nothing to do currently. */
1086}
1087
1088
1089/**
1090 * Sets up and activates VT-x on the current CPU.
1091 *
1092 * @returns VBox status code.
1093 * @param pCpu Pointer to the global CPU info struct.
1094 * @param pVM The cross context VM structure. Can be
1095 * NULL after a host resume operation.
1096 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1097 * fEnabledByHost is @c true).
1098 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1099 * @a fEnabledByHost is @c true).
1100 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1101 * enable VT-x on the host.
1102 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1103 */
1104VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1105 void *pvMsrs)
1106{
1107 Assert(pCpu);
1108 Assert(pvMsrs);
1109 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1110
1111 /* Enable VT-x if it's not already enabled by the host. */
1112 if (!fEnabledByHost)
1113 {
1114 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1115 if (RT_FAILURE(rc))
1116 return rc;
1117 }
1118
1119 /*
1120 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
1121 * using EPTPs) so we don't retain any stale guest-physical mappings which won't get
1122 * invalidated when flushing by VPID.
1123 */
1124 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1125 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1126 {
1127 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1128 pCpu->fFlushAsidBeforeUse = false;
1129 }
1130 else
1131 pCpu->fFlushAsidBeforeUse = true;
1132
1133 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1134 ++pCpu->cTlbFlushes;
1135
1136 return VINF_SUCCESS;
1137}
1138
1139
1140/**
1141 * Deactivates VT-x on the current CPU.
1142 *
1143 * @returns VBox status code.
1144 * @param pCpu Pointer to the global CPU info struct.
1145 * @param pvCpuPage Pointer to the VMXON region.
1146 * @param HCPhysCpuPage Physical address of the VMXON region.
1147 *
1148 * @remarks This function should never be called when SUPR0EnableVTx() or
1149 * similar was used to enable VT-x on the host.
1150 */
1151VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1152{
1153 NOREF(pCpu);
1154 NOREF(pvCpuPage);
1155 NOREF(HCPhysCpuPage);
1156
1157 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1158 return hmR0VmxLeaveRootMode();
1159}
1160
1161
1162/**
1163 * Sets the permission bits for the specified MSR in the MSR bitmap.
1164 *
1165 * @param pVCpu The cross context virtual CPU structure.
1166 * @param uMsr The MSR value.
1167 * @param enmRead Whether reading this MSR causes a VM-exit.
1168 * @param enmWrite Whether writing this MSR causes a VM-exit.
1169 */
1170static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1171{
1172 int32_t iBit;
1173 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1174
1175 /*
1176 * Layout:
1177 * 0x000 - 0x3ff - Low MSR read bits
1178 * 0x400 - 0x7ff - High MSR read bits
1179 * 0x800 - 0xbff - Low MSR write bits
1180 * 0xc00 - 0xfff - High MSR write bits
1181 */
1182 if (uMsr <= 0x00001FFF)
1183 iBit = uMsr;
1184 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1185 {
1186 iBit = uMsr - UINT32_C(0xC0000000);
1187 pbMsrBitmap += 0x400;
1188 }
1189 else
1190 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1191
1192 Assert(iBit <= 0x1fff);
1193 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1194 ASMBitSet(pbMsrBitmap, iBit);
1195 else
1196 ASMBitClear(pbMsrBitmap, iBit);
1197
1198 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1199 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1200 else
1201 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1202}
1203
1204
1205#ifdef VBOX_STRICT
1206/**
1207 * Gets the permission bits for the specified MSR in the MSR bitmap.
1208 *
1209 * @returns VBox status code.
1210 * @retval VINF_SUCCESS if the specified MSR is found.
1211 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1212 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1213 *
1214 * @param pVCpu The cross context virtual CPU structure.
1215 * @param uMsr The MSR.
1216 * @param penmRead Where to store the read permissions.
1217 * @param penmWrite Where to store the write permissions.
1218 */
1219static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1220{
1221 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1222 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1223 int32_t iBit;
1224 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1225
1226 /* See hmR0VmxSetMsrPermission() for the layout. */
1227 if (uMsr <= 0x00001FFF)
1228 iBit = uMsr;
1229 else if ( uMsr >= 0xC0000000
1230 && uMsr <= 0xC0001FFF)
1231 {
1232 iBit = (uMsr - 0xC0000000);
1233 pbMsrBitmap += 0x400;
1234 }
1235 else
1236 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1237
1238 Assert(iBit <= 0x1fff);
1239 if (ASMBitTest(pbMsrBitmap, iBit))
1240 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1241 else
1242 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1243
1244 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1245 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1246 else
1247 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1248 return VINF_SUCCESS;
1249}
1250#endif /* VBOX_STRICT */
1251
1252
1253/**
1254 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1255 * area.
1256 *
1257 * @returns VBox status code.
1258 * @param pVCpu The cross context virtual CPU structure.
1259 * @param cMsrs The number of MSRs.
1260 */
1261static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1262{
1263 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1264 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1265 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1266 {
1267 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1268 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1269 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1270 }
1271
1272 /* Update number of guest MSRs to load/store across the world-switch. */
1273 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1274 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1275
1276 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1277 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1278 AssertRCReturn(rc, rc);
1279
1280 /* Update the VCPU's copy of the MSR count. */
1281 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1282
1283 return VINF_SUCCESS;
1284}
1285
1286
1287/**
1288 * Adds a new (or updates the value of an existing) guest/host MSR
1289 * pair to be swapped during the world-switch as part of the
1290 * auto-load/store MSR area in the VMCS.
1291 *
1292 * @returns VBox status code.
1293 * @param pVCpu The cross context virtual CPU structure.
1294 * @param uMsr The MSR.
1295 * @param uGuestMsrValue Value of the guest MSR.
1296 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1297 * necessary.
1298 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1299 * its value was updated. Optional, can be NULL.
1300 */
1301static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1302 bool *pfAddedAndUpdated)
1303{
1304 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1305 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1306 uint32_t i;
1307 for (i = 0; i < cMsrs; i++)
1308 {
1309 if (pGuestMsr->u32Msr == uMsr)
1310 break;
1311 pGuestMsr++;
1312 }
1313
1314 bool fAdded = false;
1315 if (i == cMsrs)
1316 {
1317 ++cMsrs;
1318 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1319 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1320
1321 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1322 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1323 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1324
1325 fAdded = true;
1326 }
1327
1328 /* Update the MSR values in the auto-load/store MSR area. */
1329 pGuestMsr->u32Msr = uMsr;
1330 pGuestMsr->u64Value = uGuestMsrValue;
1331
1332 /* Create/update the MSR slot in the host MSR area. */
1333 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1334 pHostMsr += i;
1335 pHostMsr->u32Msr = uMsr;
1336
1337 /*
1338 * Update the host MSR only when requested by the caller AND when we're
1339 * adding it to the auto-load/store area. Otherwise, it would have been
1340 * updated by hmR0VmxExportHostMsrs(). We do this for performance reasons.
1341 */
1342 bool fUpdatedMsrValue = false;
1343 if ( fAdded
1344 && fUpdateHostMsr)
1345 {
1346 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1347 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1348 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1349 fUpdatedMsrValue = true;
1350 }
1351
1352 if (pfAddedAndUpdated)
1353 *pfAddedAndUpdated = fUpdatedMsrValue;
1354 return VINF_SUCCESS;
1355}
1356
1357
1358/**
1359 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1360 * auto-load/store MSR area in the VMCS.
1361 *
1362 * @returns VBox status code.
1363 * @param pVCpu The cross context virtual CPU structure.
1364 * @param uMsr The MSR.
1365 */
1366static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1367{
1368 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1369 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1370 for (uint32_t i = 0; i < cMsrs; i++)
1371 {
1372 /* Find the MSR. */
1373 if (pGuestMsr->u32Msr == uMsr)
1374 {
1375 /* If it's the last MSR, simply reduce the count. */
1376 if (i == cMsrs - 1)
1377 {
1378 --cMsrs;
1379 break;
1380 }
1381
1382 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1383 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1384 pLastGuestMsr += cMsrs - 1;
1385 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1386 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1387
1388 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1389 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1390 pLastHostMsr += cMsrs - 1;
1391 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1392 pHostMsr->u64Value = pLastHostMsr->u64Value;
1393 --cMsrs;
1394 break;
1395 }
1396 pGuestMsr++;
1397 }
1398
1399 /* Update the VMCS if the count changed (meaning the MSR was found). */
1400 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1401 {
1402 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1403 AssertRCReturn(rc, rc);
1404
1405 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1406 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1407 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1408
1409 Log4Func(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1410 return VINF_SUCCESS;
1411 }
1412
1413 return VERR_NOT_FOUND;
1414}
1415
1416
1417/**
1418 * Checks if the specified guest MSR is part of the auto-load/store area in
1419 * the VMCS.
1420 *
1421 * @returns true if found, false otherwise.
1422 * @param pVCpu The cross context virtual CPU structure.
1423 * @param uMsr The MSR to find.
1424 */
1425static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1426{
1427 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1428 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1429
1430 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1431 {
1432 if (pGuestMsr->u32Msr == uMsr)
1433 return true;
1434 }
1435 return false;
1436}
1437
1438
1439/**
1440 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1441 *
1442 * @param pVCpu The cross context virtual CPU structure.
1443 *
1444 * @remarks No-long-jump zone!!!
1445 */
1446static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1447{
1448 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1449 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1450 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1451 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1452
1453 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1454 {
1455 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1456
1457 /*
1458 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1459 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1460 */
1461 if (pHostMsr->u32Msr == MSR_K6_EFER)
1462 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1463 else
1464 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1465 }
1466
1467 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1468}
1469
1470
1471/**
1472 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1473 * perform lazy restoration of the host MSRs while leaving VT-x.
1474 *
1475 * @param pVCpu The cross context virtual CPU structure.
1476 *
1477 * @remarks No-long-jump zone!!!
1478 */
1479static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1480{
1481 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1482
1483 /*
1484 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1485 */
1486 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1487 {
1488 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1489#if HC_ARCH_BITS == 64
1490 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1491 {
1492 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1493 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1494 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1495 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1496 }
1497#endif
1498 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1499 }
1500}
1501
1502
1503/**
1504 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1505 * lazily while leaving VT-x.
1506 *
1507 * @returns true if it does, false otherwise.
1508 * @param pVCpu The cross context virtual CPU structure.
1509 * @param uMsr The MSR to check.
1510 */
1511static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1512{
1513 NOREF(pVCpu);
1514#if HC_ARCH_BITS == 64
1515 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1516 {
1517 switch (uMsr)
1518 {
1519 case MSR_K8_LSTAR:
1520 case MSR_K6_STAR:
1521 case MSR_K8_SF_MASK:
1522 case MSR_K8_KERNEL_GS_BASE:
1523 return true;
1524 }
1525 }
1526#else
1527 RT_NOREF(pVCpu, uMsr);
1528#endif
1529 return false;
1530}
1531
1532
1533/**
1534 * Loads a set of guests MSRs to allow read/passthru to the guest.
1535 *
1536 * The name of this function is slightly confusing. This function does NOT
1537 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1538 * common prefix for functions dealing with "lazy restoration" of the shared
1539 * MSRs.
1540 *
1541 * @param pVCpu The cross context virtual CPU structure.
1542 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1543 * out-of-sync. Make sure to update the required fields
1544 * before using them.
1545 *
1546 * @remarks No-long-jump zone!!!
1547 */
1548static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1549{
1550 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1551 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1552
1553 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1554#if HC_ARCH_BITS == 64
1555 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1556 {
1557 /*
1558 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1559 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1560 * we can skip a few MSR writes.
1561 *
1562 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1563 * guest MSR values in the guest-CPU context might be different to what's currently
1564 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1565 * CPU, see @bugref{8728}.
1566 */
1567 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1568 && pMixedCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1569 && pMixedCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1570 && pMixedCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1571 && pMixedCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1572 {
1573#ifdef VBOX_STRICT
1574 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pMixedCtx->msrKERNELGSBASE);
1575 Assert(ASMRdMsr(MSR_K8_LSTAR) == pMixedCtx->msrLSTAR);
1576 Assert(ASMRdMsr(MSR_K6_STAR) == pMixedCtx->msrSTAR);
1577 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pMixedCtx->msrSFMASK);
1578#endif
1579 }
1580 else
1581 {
1582 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1583 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1584 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1585 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1586 }
1587 }
1588#else
1589 RT_NOREF(pMixedCtx);
1590#endif
1591 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1592}
1593
1594
1595/**
1596 * Performs lazy restoration of the set of host MSRs if they were previously
1597 * loaded with guest MSR values.
1598 *
1599 * @param pVCpu The cross context virtual CPU structure.
1600 *
1601 * @remarks No-long-jump zone!!!
1602 * @remarks The guest MSRs should have been saved back into the guest-CPU
1603 * context by hmR0VmxImportGuestState()!!!
1604 */
1605static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1606{
1607 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1608 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1609
1610 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1611 {
1612 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1613#if HC_ARCH_BITS == 64
1614 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1615 {
1616 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1617 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1618 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1619 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1620 }
1621#endif
1622 }
1623 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1624}
1625
1626
1627/**
1628 * Verifies that our cached values of the VMCS controls are all
1629 * consistent with what's actually present in the VMCS.
1630 *
1631 * @returns VBox status code.
1632 * @param pVCpu The cross context virtual CPU structure.
1633 */
1634static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1635{
1636 uint32_t u32Val;
1637 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1638 AssertRCReturn(rc, rc);
1639 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1640 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1641
1642 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1643 AssertRCReturn(rc, rc);
1644 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1645 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1646
1647 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1648 AssertRCReturn(rc, rc);
1649 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1650 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1651
1652 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1653 AssertRCReturn(rc, rc);
1654 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1655 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1656
1657 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1658 {
1659 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1660 AssertRCReturn(rc, rc);
1661 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1662 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1663 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1664 }
1665
1666 return VINF_SUCCESS;
1667}
1668
1669
1670#ifdef VBOX_STRICT
1671/**
1672 * Verifies that our cached host EFER value has not changed
1673 * since we cached it.
1674 *
1675 * @param pVCpu The cross context virtual CPU structure.
1676 */
1677static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1678{
1679 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1680
1681 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1682 {
1683 uint64_t u64Val;
1684 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1685 AssertRC(rc);
1686
1687 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1688 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1689 }
1690}
1691
1692
1693/**
1694 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1695 * VMCS are correct.
1696 *
1697 * @param pVCpu The cross context virtual CPU structure.
1698 */
1699static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1700{
1701 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1702
1703 /* Verify MSR counts in the VMCS are what we think it should be. */
1704 uint32_t cMsrs;
1705 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1706 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1707
1708 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1709 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1710
1711 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1712 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1713
1714 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1715 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1716 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1717 {
1718 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1719 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1720 pGuestMsr->u32Msr, cMsrs));
1721
1722 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1723 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1724 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1725
1726 /* Verify that the permissions are as expected in the MSR bitmap. */
1727 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1728 {
1729 VMXMSREXITREAD enmRead;
1730 VMXMSREXITWRITE enmWrite;
1731 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1732 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1733 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1734 {
1735 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1736 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1737 }
1738 else
1739 {
1740 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1741 pGuestMsr->u32Msr, cMsrs));
1742 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1743 pGuestMsr->u32Msr, cMsrs));
1744 }
1745 }
1746 }
1747}
1748#endif /* VBOX_STRICT */
1749
1750
1751/**
1752 * Flushes the TLB using EPT.
1753 *
1754 * @returns VBox status code.
1755 * @param pVCpu The cross context virtual CPU structure of the calling
1756 * EMT. Can be NULL depending on @a enmFlush.
1757 * @param enmFlush Type of flush.
1758 *
1759 * @remarks Caller is responsible for making sure this function is called only
1760 * when NestedPaging is supported and providing @a enmFlush that is
1761 * supported by the CPU.
1762 * @remarks Can be called with interrupts disabled.
1763 */
1764static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1765{
1766 uint64_t au64Descriptor[2];
1767 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1768 au64Descriptor[0] = 0;
1769 else
1770 {
1771 Assert(pVCpu);
1772 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1773 }
1774 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1775
1776 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1777 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1778 rc));
1779 if ( RT_SUCCESS(rc)
1780 && pVCpu)
1781 {
1782 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1783 }
1784}
1785
1786
1787/**
1788 * Flushes the TLB using VPID.
1789 *
1790 * @returns VBox status code.
1791 * @param pVM The cross context VM structure.
1792 * @param pVCpu The cross context virtual CPU structure of the calling
1793 * EMT. Can be NULL depending on @a enmFlush.
1794 * @param enmFlush Type of flush.
1795 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1796 * on @a enmFlush).
1797 *
1798 * @remarks Can be called with interrupts disabled.
1799 */
1800static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1801{
1802 NOREF(pVM);
1803 AssertPtr(pVM);
1804 Assert(pVM->hm.s.vmx.fVpid);
1805
1806 uint64_t au64Descriptor[2];
1807 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1808 {
1809 au64Descriptor[0] = 0;
1810 au64Descriptor[1] = 0;
1811 }
1812 else
1813 {
1814 AssertPtr(pVCpu);
1815 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1816 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1817 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1818 au64Descriptor[1] = GCPtr;
1819 }
1820
1821 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1822 AssertMsg(rc == VINF_SUCCESS,
1823 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1824 if ( RT_SUCCESS(rc)
1825 && pVCpu)
1826 {
1827 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1828 }
1829}
1830
1831
1832/**
1833 * Invalidates a guest page by guest virtual address. Only relevant for
1834 * EPT/VPID, otherwise there is nothing really to invalidate.
1835 *
1836 * @returns VBox status code.
1837 * @param pVM The cross context VM structure.
1838 * @param pVCpu The cross context virtual CPU structure.
1839 * @param GCVirt Guest virtual address of the page to invalidate.
1840 */
1841VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1842{
1843 AssertPtr(pVM);
1844 AssertPtr(pVCpu);
1845 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1846
1847 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1848 if (!fFlushPending)
1849 {
1850 /*
1851 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1852 * the EPT case. See @bugref{6043} and @bugref{6177}.
1853 *
1854 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1855 * as this function maybe called in a loop with individual addresses.
1856 */
1857 if (pVM->hm.s.vmx.fVpid)
1858 {
1859 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1860
1861#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1862 /*
1863 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1864 * where executing INVVPID outside 64-bit mode does not flush translations of
1865 * 64-bit linear addresses, see @bugref{6208#c72}.
1866 */
1867 if (RT_HI_U32(GCVirt))
1868 fVpidFlush = false;
1869#endif
1870
1871 if (fVpidFlush)
1872 {
1873 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1874 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1875 }
1876 else
1877 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1878 }
1879 else if (pVM->hm.s.fNestedPaging)
1880 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1881 }
1882
1883 return VINF_SUCCESS;
1884}
1885
1886
1887/**
1888 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1889 * case where neither EPT nor VPID is supported by the CPU.
1890 *
1891 * @param pVM The cross context VM structure.
1892 * @param pVCpu The cross context virtual CPU structure.
1893 * @param pCpu Pointer to the global HM struct.
1894 *
1895 * @remarks Called with interrupts disabled.
1896 */
1897static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1898{
1899 AssertPtr(pVCpu);
1900 AssertPtr(pCpu);
1901 NOREF(pVM);
1902
1903 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1904
1905 Assert(pCpu->idCpu != NIL_RTCPUID);
1906 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1907 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1908 pVCpu->hm.s.fForceTLBFlush = false;
1909 return;
1910}
1911
1912
1913/**
1914 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1915 *
1916 * @param pVM The cross context VM structure.
1917 * @param pVCpu The cross context virtual CPU structure.
1918 * @param pCpu Pointer to the global HM CPU struct.
1919 * @remarks All references to "ASID" in this function pertains to "VPID" in
1920 * Intel's nomenclature. The reason is, to avoid confusion in compare
1921 * statements since the host-CPU copies are named "ASID".
1922 *
1923 * @remarks Called with interrupts disabled.
1924 */
1925static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1926{
1927#ifdef VBOX_WITH_STATISTICS
1928 bool fTlbFlushed = false;
1929# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1930# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1931 if (!fTlbFlushed) \
1932 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1933 } while (0)
1934#else
1935# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1936# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1937#endif
1938
1939 AssertPtr(pVM);
1940 AssertPtr(pCpu);
1941 AssertPtr(pVCpu);
1942 Assert(pCpu->idCpu != NIL_RTCPUID);
1943
1944 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1945 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1946 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1947
1948 /*
1949 * Force a TLB flush for the first world-switch if the current CPU differs from the one we
1950 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1951 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1952 * cannot reuse the current ASID anymore.
1953 */
1954 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1955 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1956 {
1957 ++pCpu->uCurrentAsid;
1958 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1959 {
1960 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1961 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1962 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1963 }
1964
1965 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1966 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1967 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1968
1969 /*
1970 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1971 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1972 */
1973 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1974 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1975 HMVMX_SET_TAGGED_TLB_FLUSHED();
1976 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1977 }
1978
1979 /* Check for explicit TLB flushes. */
1980 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1981 {
1982 /*
1983 * Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
1984 * creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
1985 * tables when EPT is in use. Flushing-by-VPID will only flush linear (only
1986 * VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
1987 * mappings, see @bugref{6568}.
1988 *
1989 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
1990 */
1991 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1992 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1993 HMVMX_SET_TAGGED_TLB_FLUSHED();
1994 }
1995
1996 pVCpu->hm.s.fForceTLBFlush = false;
1997 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1998
1999 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2000 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2001 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2002 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2003 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2004 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2005 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2006 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2007 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2008
2009 /* Update VMCS with the VPID. */
2010 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2011 AssertRC(rc);
2012
2013#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2014}
2015
2016
2017/**
2018 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2019 *
2020 * @returns VBox status code.
2021 * @param pVM The cross context VM structure.
2022 * @param pVCpu The cross context virtual CPU structure.
2023 * @param pCpu Pointer to the global HM CPU struct.
2024 *
2025 * @remarks Called with interrupts disabled.
2026 */
2027static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2028{
2029 AssertPtr(pVM);
2030 AssertPtr(pVCpu);
2031 AssertPtr(pCpu);
2032 Assert(pCpu->idCpu != NIL_RTCPUID);
2033 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2034 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2035
2036 /*
2037 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2038 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2039 */
2040 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2041 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2042 {
2043 pVCpu->hm.s.fForceTLBFlush = true;
2044 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2045 }
2046
2047 /* Check for explicit TLB flushes. */
2048 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2049 {
2050 pVCpu->hm.s.fForceTLBFlush = true;
2051 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2052 }
2053
2054 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2055 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2056
2057 if (pVCpu->hm.s.fForceTLBFlush)
2058 {
2059 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2060 pVCpu->hm.s.fForceTLBFlush = false;
2061 }
2062}
2063
2064
2065/**
2066 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2067 *
2068 * @returns VBox status code.
2069 * @param pVM The cross context VM structure.
2070 * @param pVCpu The cross context virtual CPU structure.
2071 * @param pCpu Pointer to the global HM CPU struct.
2072 *
2073 * @remarks Called with interrupts disabled.
2074 */
2075static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2076{
2077 AssertPtr(pVM);
2078 AssertPtr(pVCpu);
2079 AssertPtr(pCpu);
2080 Assert(pCpu->idCpu != NIL_RTCPUID);
2081 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2082 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2083
2084 /*
2085 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2086 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2087 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2088 */
2089 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2090 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2091 {
2092 pVCpu->hm.s.fForceTLBFlush = true;
2093 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2094 }
2095
2096 /* Check for explicit TLB flushes. */
2097 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2098 {
2099 /*
2100 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2101 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2102 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2103 */
2104 pVCpu->hm.s.fForceTLBFlush = true;
2105 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2106 }
2107
2108 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2109 if (pVCpu->hm.s.fForceTLBFlush)
2110 {
2111 ++pCpu->uCurrentAsid;
2112 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2113 {
2114 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2115 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2116 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2117 }
2118
2119 pVCpu->hm.s.fForceTLBFlush = false;
2120 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2121 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2122 if (pCpu->fFlushAsidBeforeUse)
2123 {
2124 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2125 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2126 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2127 {
2128 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2129 pCpu->fFlushAsidBeforeUse = false;
2130 }
2131 else
2132 {
2133 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2134 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2135 }
2136 }
2137 }
2138
2139 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2140 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2141 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2142 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2143 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2144 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2145 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2146
2147 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2148 AssertRC(rc);
2149}
2150
2151
2152/**
2153 * Flushes the guest TLB entry based on CPU capabilities.
2154 *
2155 * @param pVCpu The cross context virtual CPU structure.
2156 * @param pCpu Pointer to the global HM CPU struct.
2157 */
2158DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2159{
2160#ifdef HMVMX_ALWAYS_FLUSH_TLB
2161 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2162#endif
2163 PVM pVM = pVCpu->CTX_SUFF(pVM);
2164 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2165 {
2166 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2167 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2168 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2169 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2170 default:
2171 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2172 break;
2173 }
2174 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2175}
2176
2177
2178/**
2179 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2180 * TLB entries from the host TLB before VM-entry.
2181 *
2182 * @returns VBox status code.
2183 * @param pVM The cross context VM structure.
2184 */
2185static int hmR0VmxSetupTaggedTlb(PVM pVM)
2186{
2187 /*
2188 * Determine optimal flush type for Nested Paging.
2189 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2190 * guest execution (see hmR3InitFinalizeR0()).
2191 */
2192 if (pVM->hm.s.fNestedPaging)
2193 {
2194 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2195 {
2196 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2197 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2198 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2199 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2200 else
2201 {
2202 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2203 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2204 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2205 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2206 }
2207
2208 /* Make sure the write-back cacheable memory type for EPT is supported. */
2209 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2210 {
2211 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2212 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2213 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2214 }
2215
2216 /* EPT requires a page-walk length of 4. */
2217 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2218 {
2219 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2220 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2221 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2222 }
2223 }
2224 else
2225 {
2226 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2227 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2228 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2229 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2230 }
2231 }
2232
2233 /*
2234 * Determine optimal flush type for VPID.
2235 */
2236 if (pVM->hm.s.vmx.fVpid)
2237 {
2238 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2239 {
2240 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2241 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2242 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2243 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2244 else
2245 {
2246 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2247 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2248 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2249 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2250 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2251 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2252 pVM->hm.s.vmx.fVpid = false;
2253 }
2254 }
2255 else
2256 {
2257 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2258 Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2259 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2260 pVM->hm.s.vmx.fVpid = false;
2261 }
2262 }
2263
2264 /*
2265 * Setup the handler for flushing tagged-TLBs.
2266 */
2267 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2268 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2269 else if (pVM->hm.s.fNestedPaging)
2270 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2271 else if (pVM->hm.s.vmx.fVpid)
2272 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2273 else
2274 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2275 return VINF_SUCCESS;
2276}
2277
2278
2279/**
2280 * Sets up pin-based VM-execution controls in the VMCS.
2281 *
2282 * @returns VBox status code.
2283 * @param pVM The cross context VM structure.
2284 * @param pVCpu The cross context virtual CPU structure.
2285 */
2286static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2287{
2288 AssertPtr(pVM);
2289 AssertPtr(pVCpu);
2290
2291 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2292 uint32_t fZap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2293
2294 fVal |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2295 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2296
2297 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2298 fVal |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2299
2300 /* Enable the VMX preemption timer. */
2301 if (pVM->hm.s.vmx.fUsePreemptTimer)
2302 {
2303 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2304 fVal |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2305 }
2306
2307#if 0
2308 /* Enable posted-interrupt processing. */
2309 if (pVM->hm.s.fPostedIntrs)
2310 {
2311 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2312 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2313 fVal |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2314 }
2315#endif
2316
2317 if ((fVal & fZap) != fVal)
2318 {
2319 LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! Cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2320 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, fVal, fZap));
2321 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2322 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2323 }
2324
2325 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2326 AssertRCReturn(rc, rc);
2327
2328 pVCpu->hm.s.vmx.u32PinCtls = fVal;
2329 return rc;
2330}
2331
2332
2333/**
2334 * Sets up processor-based VM-execution controls in the VMCS.
2335 *
2336 * @returns VBox status code.
2337 * @param pVM The cross context VM structure.
2338 * @param pVCpu The cross context virtual CPU structure.
2339 */
2340static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2341{
2342 AssertPtr(pVM);
2343 AssertPtr(pVCpu);
2344
2345 int rc = VERR_INTERNAL_ERROR_5;
2346 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2347 uint32_t fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2348
2349 fVal |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2350 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2351 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2352 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2353 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2354 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2355 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2356
2357 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2358 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2359 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2360 {
2361 LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2362 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2363 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2364 }
2365
2366 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2367 if (!pVM->hm.s.fNestedPaging)
2368 {
2369 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2370 fVal |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2371 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2372 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2373 }
2374
2375 /* Use TPR shadowing if supported by the CPU. */
2376 if ( PDMHasApic(pVM)
2377 && pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2378 {
2379 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2380 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2381 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2382 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2383 AssertRCReturn(rc, rc);
2384
2385 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2386 /* CR8 writes cause a VM-exit based on TPR threshold. */
2387 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2388 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2389 }
2390 else
2391 {
2392 /*
2393 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2394 * Set this control only for 64-bit guests.
2395 */
2396 if (pVM->hm.s.fAllow64BitGuests)
2397 {
2398 fVal |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2399 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2400 }
2401 }
2402
2403 /* Use MSR-bitmaps if supported by the CPU. */
2404 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2405 {
2406 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2407
2408 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2409 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2410 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2411 AssertRCReturn(rc, rc);
2412
2413 /*
2414 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2415 * automatically using dedicated fields in the VMCS.
2416 */
2417 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2418 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2419 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2420 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2421 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2422
2423#if HC_ARCH_BITS == 64
2424 /*
2425 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2426 */
2427 if (pVM->hm.s.fAllow64BitGuests)
2428 {
2429 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2430 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2431 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2432 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2433 }
2434#endif
2435 /*
2436 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2437 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2438 */
2439 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2440 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2441
2442 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2443 }
2444
2445 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2446 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2447 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2448
2449 if ((fVal & fZap) != fVal)
2450 {
2451 LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2452 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, fVal, fZap));
2453 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2454 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2455 }
2456
2457 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2458 AssertRCReturn(rc, rc);
2459
2460 pVCpu->hm.s.vmx.u32ProcCtls = fVal;
2461
2462 /*
2463 * Secondary processor-based VM-execution controls.
2464 */
2465 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2466 {
2467 fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2468 fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2469
2470 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2471 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2472
2473 if (pVM->hm.s.fNestedPaging)
2474 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2475
2476 /*
2477 * Enable the INVPCID instruction if supported by the hardware and we expose
2478 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2479 */
2480 if ( (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2481 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2482 {
2483 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2484 }
2485
2486 if (pVM->hm.s.vmx.fVpid)
2487 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2488
2489 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2490 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2491
2492#if 0
2493 if (pVM->hm.s.fVirtApicRegs)
2494 {
2495 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2496 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2497
2498 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2499 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2500 }
2501#endif
2502
2503 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2504 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2505 * done dynamically. */
2506 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2507 {
2508 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2509 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2510 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2511 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2512 AssertRCReturn(rc, rc);
2513 }
2514
2515 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2516 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2517
2518 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2519 && pVM->hm.s.vmx.cPleGapTicks
2520 && pVM->hm.s.vmx.cPleWindowTicks)
2521 {
2522 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2523
2524 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2525 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2526 AssertRCReturn(rc, rc);
2527 }
2528
2529 if ((fVal & fZap) != fVal)
2530 {
2531 LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2532 "cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, fVal, fZap));
2533 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2534 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2535 }
2536
2537 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2538 AssertRCReturn(rc, rc);
2539
2540 pVCpu->hm.s.vmx.u32ProcCtls2 = fVal;
2541 }
2542 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2543 {
2544 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2545 "available\n"));
2546 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2547 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2548 }
2549
2550 return VINF_SUCCESS;
2551}
2552
2553
2554/**
2555 * Sets up miscellaneous (everything other than Pin & Processor-based
2556 * VM-execution) control fields in the VMCS.
2557 *
2558 * @returns VBox status code.
2559 * @param pVM The cross context VM structure.
2560 * @param pVCpu The cross context virtual CPU structure.
2561 */
2562static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2563{
2564 NOREF(pVM);
2565 AssertPtr(pVM);
2566 AssertPtr(pVCpu);
2567
2568 int rc = VERR_GENERAL_FAILURE;
2569
2570 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2571#if 0
2572 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxExportGuestCR3AndCR4())*/
2573 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2574 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2575
2576 /*
2577 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2578 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2579 * We thus use the exception bitmap to control it rather than use both.
2580 */
2581 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2582 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2583
2584 /* All IO & IOIO instructions cause VM-exits. */
2585 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2586 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2587
2588 /* Initialize the MSR-bitmap area. */
2589 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2590 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2591 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2592 AssertRCReturn(rc, rc);
2593#endif
2594
2595 /* Setup MSR auto-load/store area. */
2596 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2597 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2598 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2599 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2600 AssertRCReturn(rc, rc);
2601
2602 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2603 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2604 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2605 AssertRCReturn(rc, rc);
2606
2607 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2608 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2609 AssertRCReturn(rc, rc);
2610
2611 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2612#if 0
2613 /* Setup debug controls */
2614 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2615 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2616 AssertRCReturn(rc, rc);
2617#endif
2618
2619 return rc;
2620}
2621
2622
2623/**
2624 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2625 *
2626 * We shall setup those exception intercepts that don't change during the
2627 * lifetime of the VM here. The rest are done dynamically while loading the
2628 * guest state.
2629 *
2630 * @returns VBox status code.
2631 * @param pVM The cross context VM structure.
2632 * @param pVCpu The cross context virtual CPU structure.
2633 */
2634static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2635{
2636 AssertPtr(pVM);
2637 AssertPtr(pVCpu);
2638
2639 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2640
2641 uint32_t u32XcptBitmap = 0;
2642
2643 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2644 u32XcptBitmap |= RT_BIT_32(X86_XCPT_AC);
2645
2646 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2647 and writes, and because recursive #DBs can cause the CPU hang, we must always
2648 intercept #DB. */
2649 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2650
2651 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2652 if (!pVM->hm.s.fNestedPaging)
2653 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2654
2655 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2656 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2657 AssertRCReturn(rc, rc);
2658 return rc;
2659}
2660
2661
2662/**
2663 * Does per-VM VT-x initialization.
2664 *
2665 * @returns VBox status code.
2666 * @param pVM The cross context VM structure.
2667 */
2668VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2669{
2670 LogFlowFunc(("pVM=%p\n", pVM));
2671
2672 int rc = hmR0VmxStructsAlloc(pVM);
2673 if (RT_FAILURE(rc))
2674 {
2675 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2676 return rc;
2677 }
2678
2679 return VINF_SUCCESS;
2680}
2681
2682
2683/**
2684 * Does per-VM VT-x termination.
2685 *
2686 * @returns VBox status code.
2687 * @param pVM The cross context VM structure.
2688 */
2689VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2690{
2691 LogFlowFunc(("pVM=%p\n", pVM));
2692
2693#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2694 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2695 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2696#endif
2697 hmR0VmxStructsFree(pVM);
2698 return VINF_SUCCESS;
2699}
2700
2701
2702/**
2703 * Sets up the VM for execution under VT-x.
2704 * This function is only called once per-VM during initialization.
2705 *
2706 * @returns VBox status code.
2707 * @param pVM The cross context VM structure.
2708 */
2709VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2710{
2711 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2712 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2713
2714 LogFlowFunc(("pVM=%p\n", pVM));
2715
2716 /*
2717 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be
2718 * allocated. We no longer support the highly unlikely case of UnrestrictedGuest without
2719 * pRealModeTSS, see hmR3InitFinalizeR0Intel().
2720 */
2721 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2722 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2723 || !pVM->hm.s.vmx.pRealModeTSS))
2724 {
2725 LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2726 return VERR_INTERNAL_ERROR;
2727 }
2728
2729 /* Initialize these always, see hmR3InitFinalizeR0().*/
2730 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2731 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2732
2733 /* Setup the tagged-TLB flush handlers. */
2734 int rc = hmR0VmxSetupTaggedTlb(pVM);
2735 if (RT_FAILURE(rc))
2736 {
2737 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2738 return rc;
2739 }
2740
2741 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2742 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2743#if HC_ARCH_BITS == 64
2744 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2745 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2746 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2747 {
2748 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2749 }
2750#endif
2751
2752 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2753 RTCCUINTREG uHostCR4 = ASMGetCR4();
2754 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2755 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2756
2757 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2758 {
2759 PVMCPU pVCpu = &pVM->aCpus[i];
2760 AssertPtr(pVCpu);
2761 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2762
2763 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2764 Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2765
2766 /* Set revision dword at the beginning of the VMCS structure. */
2767 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2768
2769 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2770 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2771 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2772 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2773
2774 /* Load this VMCS as the current VMCS. */
2775 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2776 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2777 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2778
2779 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2780 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2781 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2782
2783 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2784 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2785 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2786
2787 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2788 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2789 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2790
2791 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2792 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2793 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2794
2795#if HC_ARCH_BITS == 32
2796 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2797 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2798 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2799#endif
2800
2801 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2802 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2803 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2804 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2805
2806 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2807
2808 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2809 }
2810
2811 return VINF_SUCCESS;
2812}
2813
2814
2815/**
2816 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2817 * the VMCS.
2818 *
2819 * @returns VBox status code.
2820 */
2821static int hmR0VmxExportHostControlRegs(void)
2822{
2823 RTCCUINTREG uReg = ASMGetCR0();
2824 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2825 AssertRCReturn(rc, rc);
2826
2827 uReg = ASMGetCR3();
2828 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2829 AssertRCReturn(rc, rc);
2830
2831 uReg = ASMGetCR4();
2832 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2833 AssertRCReturn(rc, rc);
2834 return rc;
2835}
2836
2837
2838/**
2839 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2840 * the host-state area in the VMCS.
2841 *
2842 * @returns VBox status code.
2843 * @param pVCpu The cross context virtual CPU structure.
2844 */
2845static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
2846{
2847#if HC_ARCH_BITS == 64
2848/**
2849 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2850 * requirements. See hmR0VmxExportHostSegmentRegs().
2851 */
2852# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2853 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2854 { \
2855 bool fValidSelector = true; \
2856 if ((selValue) & X86_SEL_LDT) \
2857 { \
2858 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2859 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2860 } \
2861 if (fValidSelector) \
2862 { \
2863 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2864 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2865 } \
2866 (selValue) = 0; \
2867 }
2868
2869 /*
2870 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2871 * should -not- save the messed up state without restoring the original host-state,
2872 * see @bugref{7240}.
2873 *
2874 * This apparently can happen (most likely the FPU changes), deal with it rather than
2875 * asserting. Was observed booting Solaris 10u10 32-bit guest.
2876 */
2877 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2878 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2879 {
2880 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2881 pVCpu->idCpu));
2882 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2883 }
2884 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2885#else
2886 RT_NOREF(pVCpu);
2887#endif
2888
2889 /*
2890 * Host DS, ES, FS and GS segment registers.
2891 */
2892#if HC_ARCH_BITS == 64
2893 RTSEL uSelDS = ASMGetDS();
2894 RTSEL uSelES = ASMGetES();
2895 RTSEL uSelFS = ASMGetFS();
2896 RTSEL uSelGS = ASMGetGS();
2897#else
2898 RTSEL uSelDS = 0;
2899 RTSEL uSelES = 0;
2900 RTSEL uSelFS = 0;
2901 RTSEL uSelGS = 0;
2902#endif
2903
2904 /*
2905 * Host CS and SS segment registers.
2906 */
2907 RTSEL uSelCS = ASMGetCS();
2908 RTSEL uSelSS = ASMGetSS();
2909
2910 /*
2911 * Host TR segment register.
2912 */
2913 RTSEL uSelTR = ASMGetTR();
2914
2915#if HC_ARCH_BITS == 64
2916 /*
2917 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
2918 * gain VM-entry and restore them before we get preempted.
2919 *
2920 * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2921 */
2922 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2923 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2924 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2925 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2926# undef VMXLOCAL_ADJUST_HOST_SEG
2927#endif
2928
2929 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2930 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2931 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2932 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2933 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2934 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2935 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2936 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2937 Assert(uSelCS);
2938 Assert(uSelTR);
2939
2940 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2941#if 0
2942 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2943 Assert(uSelSS != 0);
2944#endif
2945
2946 /* Write these host selector fields into the host-state area in the VMCS. */
2947 int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2948 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2949#if HC_ARCH_BITS == 64
2950 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2951 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2952 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2953 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2954#else
2955 NOREF(uSelDS);
2956 NOREF(uSelES);
2957 NOREF(uSelFS);
2958 NOREF(uSelGS);
2959#endif
2960 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2961 AssertRCReturn(rc, rc);
2962
2963 /*
2964 * Host GDTR and IDTR.
2965 */
2966 RTGDTR Gdtr;
2967 RTIDTR Idtr;
2968 RT_ZERO(Gdtr);
2969 RT_ZERO(Idtr);
2970 ASMGetGDTR(&Gdtr);
2971 ASMGetIDTR(&Idtr);
2972 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2973 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2974 AssertRCReturn(rc, rc);
2975
2976#if HC_ARCH_BITS == 64
2977 /*
2978 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
2979 * them to the maximum limit (0xffff) on every VM-exit.
2980 */
2981 if (Gdtr.cbGdt != 0xffff)
2982 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2983
2984 /*
2985 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
2986 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the
2987 * limit as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU
2988 * behavior. However, several hosts either insists on 0xfff being the limit (Windows
2989 * Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
2990 * but botches sidt alignment in at least one consumer). So, we're only allowing the
2991 * IDTR.LIMIT to be left at 0xffff on hosts where we are sure it won't cause trouble.
2992 */
2993# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
2994 if (Idtr.cbIdt < 0x0fff)
2995# else
2996 if (Idtr.cbIdt != 0xffff)
2997# endif
2998 {
2999 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3000 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3001 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3002 }
3003#endif
3004
3005 /*
3006 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3007 * and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3008 * RPL should be too in most cases.
3009 */
3010 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3011 ("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3012
3013 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3014#if HC_ARCH_BITS == 64
3015 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3016
3017 /*
3018 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3019 * all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3020 * restoration if the host has something else. Task switching is not supported in 64-bit
3021 * mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3022 * limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3023 *
3024 * [1] See Intel spec. 3.5 "System Descriptor Types".
3025 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3026 */
3027 PVM pVM = pVCpu->CTX_SUFF(pVM);
3028 Assert(pDesc->System.u4Type == 11);
3029 if ( pDesc->System.u16LimitLow != 0x67
3030 || pDesc->System.u4LimitHigh)
3031 {
3032 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3033 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3034 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3035 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3036 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3037 }
3038
3039 /*
3040 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3041 */
3042 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3043 {
3044 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3045 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3046 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3047 {
3048 /* The GDT is read-only but the writable GDT is available. */
3049 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3050 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3051 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3052 AssertRCReturn(rc, rc);
3053 }
3054 }
3055#else
3056 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3057#endif
3058 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3059 AssertRCReturn(rc, rc);
3060
3061 /*
3062 * Host FS base and GS base.
3063 */
3064#if HC_ARCH_BITS == 64
3065 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3066 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3067 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3068 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3069 AssertRCReturn(rc, rc);
3070
3071 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3072 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3073 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3074 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3075 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3076#endif
3077 return VINF_SUCCESS;
3078}
3079
3080
3081/**
3082 * Exports certain host MSRs in the VM-exit MSR-load area and some in the
3083 * host-state area of the VMCS.
3084 *
3085 * Theses MSRs will be automatically restored on the host after every successful
3086 * VM-exit.
3087 *
3088 * @returns VBox status code.
3089 * @param pVCpu The cross context virtual CPU structure.
3090 *
3091 * @remarks No-long-jump zone!!!
3092 */
3093static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
3094{
3095 AssertPtr(pVCpu);
3096 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3097
3098 /*
3099 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3100 * rather than swapping them on every VM-entry.
3101 */
3102 hmR0VmxLazySaveHostMsrs(pVCpu);
3103
3104 /*
3105 * Host Sysenter MSRs.
3106 */
3107 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3108#if HC_ARCH_BITS == 32
3109 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3110 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3111#else
3112 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3113 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3114#endif
3115 AssertRCReturn(rc, rc);
3116
3117 /*
3118 * Host EFER MSR.
3119 *
3120 * If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3121 * done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3122 */
3123 PVM pVM = pVCpu->CTX_SUFF(pVM);
3124 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3125 {
3126 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3127 AssertRCReturn(rc, rc);
3128 }
3129
3130 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see hmR0VmxExportGuestExitCtls(). */
3131
3132 return VINF_SUCCESS;
3133}
3134
3135
3136/**
3137 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3138 *
3139 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3140 * these two bits are handled by VM-entry, see hmR0VmxExportGuestExitCtls() and
3141 * hmR0VMxExportGuestEntryCtls().
3142 *
3143 * @returns true if we need to load guest EFER, false otherwise.
3144 * @param pVCpu The cross context virtual CPU structure.
3145 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3146 * out-of-sync. Make sure to update the required fields
3147 * before using them.
3148 *
3149 * @remarks Requires EFER, CR4.
3150 * @remarks No-long-jump zone!!!
3151 */
3152static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3153{
3154#ifdef HMVMX_ALWAYS_SWAP_EFER
3155 return true;
3156#endif
3157
3158#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3159 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3160 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3161 return false;
3162#endif
3163
3164 PVM pVM = pVCpu->CTX_SUFF(pVM);
3165 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3166 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3167
3168 /*
3169 * For 64-bit guests, if EFER.SCE bit differs, we need to swap EFER to ensure that the
3170 * guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3171 */
3172 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3173 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3174 {
3175 return true;
3176 }
3177
3178 /*
3179 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3180 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3181 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3182 */
3183 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3184 && (pMixedCtx->cr0 & X86_CR0_PG)
3185 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3186 {
3187 /* Assert that host is PAE capable. */
3188 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3189 return true;
3190 }
3191
3192 return false;
3193}
3194
3195
3196/**
3197 * Exports the guest state with appropriate VM-entry controls in the VMCS.
3198 *
3199 * These controls can affect things done on VM-exit; e.g. "load debug controls",
3200 * see Intel spec. 24.8.1 "VM-entry controls".
3201 *
3202 * @returns VBox status code.
3203 * @param pVCpu The cross context virtual CPU structure.
3204 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3205 * out-of-sync. Make sure to update the required fields
3206 * before using them.
3207 *
3208 * @remarks Requires EFER.
3209 * @remarks No-long-jump zone!!!
3210 */
3211static int hmR0VmxExportGuestEntryCtls(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3212{
3213 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_CTLS)
3214 {
3215 PVM pVM = pVCpu->CTX_SUFF(pVM);
3216 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3217 uint32_t fZap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3218
3219 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3220 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3221
3222 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3223 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3224 {
3225 fVal |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3226 Log4Func(("VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n"));
3227 }
3228 else
3229 Assert(!(fVal & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3230
3231 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3232 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3233 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3234 {
3235 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3236 Log4Func(("VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n"));
3237 }
3238
3239 /*
3240 * The following should -not- be set (since we're not in SMM mode):
3241 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3242 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3243 */
3244
3245 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3246 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3247
3248 if ((fVal & fZap) != fVal)
3249 {
3250 Log4Func(("Invalid VM-entry controls combo! Cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3251 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, fVal, fZap));
3252 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3253 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3254 }
3255
3256 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
3257 AssertRCReturn(rc, rc);
3258
3259 pVCpu->hm.s.vmx.u32EntryCtls = fVal;
3260 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_CTLS);
3261 }
3262 return VINF_SUCCESS;
3263}
3264
3265
3266/**
3267 * Exports the guest state with appropriate VM-exit controls in the VMCS.
3268 *
3269 * @returns VBox status code.
3270 * @param pVCpu The cross context virtual CPU structure.
3271 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3272 * out-of-sync. Make sure to update the required fields
3273 * before using them.
3274 *
3275 * @remarks Requires EFER.
3276 */
3277static int hmR0VmxExportGuestExitCtls(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3278{
3279 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_EXIT_CTLS)
3280 {
3281 PVM pVM = pVCpu->CTX_SUFF(pVM);
3282 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3283 uint32_t fZap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3284
3285 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3286 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3287
3288 /*
3289 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3290 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in
3291 * hmR0VmxExportHostMsrs().
3292 */
3293#if HC_ARCH_BITS == 64
3294 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3295 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3296#else
3297 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3298 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3299 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3300 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3301 {
3302 /* The switcher returns to long mode, EFER is managed by the switcher. */
3303 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3304 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3305 }
3306 else
3307 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3308#endif
3309
3310 /* If the newer VMCS fields for managing EFER exists, use it. */
3311 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3312 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3313 {
3314 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3315 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3316 Log4Func(("VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR and VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n"));
3317 }
3318
3319 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3320 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3321
3322 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3323 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3324 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3325
3326 if ( pVM->hm.s.vmx.fUsePreemptTimer
3327 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3328 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3329
3330 if ((fVal & fZap) != fVal)
3331 {
3332 LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3333 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, fVal, fZap));
3334 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3335 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3336 }
3337
3338 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
3339 AssertRCReturn(rc, rc);
3340
3341 pVCpu->hm.s.vmx.u32ExitCtls = fVal;
3342 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_EXIT_CTLS);
3343 }
3344 return VINF_SUCCESS;
3345}
3346
3347
3348/**
3349 * Sets the TPR threshold in the VMCS.
3350 *
3351 * @returns VBox status code.
3352 * @param pVCpu The cross context virtual CPU structure.
3353 * @param u32TprThreshold The TPR threshold (task-priority class only).
3354 */
3355DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3356{
3357 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3358 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3359 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3360}
3361
3362
3363/**
3364 * Exports the guest APIC TPR state into the VMCS.
3365 *
3366 * @returns VBox status code.
3367 * @param pVCpu The cross context virtual CPU structure.
3368 *
3369 * @remarks No-long-jump zone!!!
3370 */
3371static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu)
3372{
3373 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
3374 {
3375 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3376 && APICIsEnabled(pVCpu))
3377 {
3378 /*
3379 * Setup TPR shadowing.
3380 */
3381 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3382 {
3383 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3384
3385 bool fPendingIntr = false;
3386 uint8_t u8Tpr = 0;
3387 uint8_t u8PendingIntr = 0;
3388 int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3389 AssertRCReturn(rc, rc);
3390
3391 /*
3392 * If there are interrupts pending but masked by the TPR, instruct VT-x to
3393 * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
3394 * priority of the pending interrupt so we can deliver the interrupt. If there
3395 * are no interrupts pending, set threshold to 0 to not cause any
3396 * TPR-below-threshold VM-exits.
3397 */
3398 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3399 uint32_t u32TprThreshold = 0;
3400 if (fPendingIntr)
3401 {
3402 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3403 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3404 const uint8_t u8TprPriority = u8Tpr >> 4;
3405 if (u8PendingPriority <= u8TprPriority)
3406 u32TprThreshold = u8PendingPriority;
3407 }
3408
3409 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3410 AssertRCReturn(rc, rc);
3411 }
3412 }
3413 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
3414 }
3415 return VINF_SUCCESS;
3416}
3417
3418
3419/**
3420 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3421 *
3422 * @returns Guest's interruptibility-state.
3423 * @param pVCpu The cross context virtual CPU structure.
3424 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3425 * out-of-sync. Make sure to update the required fields
3426 * before using them.
3427 *
3428 * @remarks No-long-jump zone!!!
3429 */
3430static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3431{
3432 /*
3433 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3434 */
3435 uint32_t fIntrState = 0;
3436 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3437 {
3438 /* If inhibition is active, RIP & RFLAGS should've been accessed
3439 (i.e. read previously from the VMCS or from ring-3). */
3440#ifdef VBOX_STRICT
3441 uint64_t const fExtrn = ASMAtomicUoReadU64(&pMixedCtx->fExtrn);
3442 AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
3443#endif
3444 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3445 {
3446 if (pMixedCtx->eflags.Bits.u1IF)
3447 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3448 else
3449 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3450 }
3451 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3452 {
3453 /*
3454 * We can clear the inhibit force flag as even if we go back to the recompiler
3455 * without executing guest code in VT-x, the flag's condition to be cleared is
3456 * met and thus the cleared state is correct.
3457 */
3458 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3459 }
3460 }
3461
3462 /*
3463 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3464 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3465 * setting this would block host-NMIs and IRET will not clear the blocking.
3466 *
3467 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3468 */
3469 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3470 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3471 {
3472 fIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3473 }
3474
3475 return fIntrState;
3476}
3477
3478
3479/**
3480 * Exports the guest's interruptibility-state into the guest-state area in the
3481 * VMCS.
3482 *
3483 * @returns VBox status code.
3484 * @param pVCpu The cross context virtual CPU structure.
3485 * @param fIntrState The interruptibility-state to set.
3486 */
3487static int hmR0VmxExportGuestIntrState(PVMCPU pVCpu, uint32_t fIntrState)
3488{
3489 NOREF(pVCpu);
3490 AssertMsg(!(fIntrState & 0xfffffff0), ("%#x\n", fIntrState)); /* Bits 31:4 MBZ. */
3491 Assert((fIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3492 return VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, fIntrState);
3493}
3494
3495
3496/**
3497 * Exports the exception intercepts required for guest execution in the VMCS.
3498 *
3499 * @returns VBox status code.
3500 * @param pVCpu The cross context virtual CPU structure.
3501 *
3502 * @remarks No-long-jump zone!!!
3503 */
3504static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu)
3505{
3506 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
3507 {
3508 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxExportSharedCR0(). */
3509 if (pVCpu->hm.s.fGIMTrapXcptUD)
3510 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3511#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3512 else
3513 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3514#endif
3515
3516 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3517 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3518
3519 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3520 AssertRCReturn(rc, rc);
3521
3522 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3523 Log4Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64\n", pVCpu->hm.s.vmx.u32XcptBitmap));
3524 }
3525 return VINF_SUCCESS;
3526}
3527
3528
3529/**
3530 * Exports the guest's RIP into the guest-state area in the VMCS.
3531 *
3532 * @returns VBox status code.
3533 * @param pVCpu The cross context virtual CPU structure.
3534 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3535 * out-of-sync. Make sure to update the required fields
3536 * before using them.
3537 *
3538 * @remarks No-long-jump zone!!!
3539 */
3540static int hmR0VmxExportGuestRip(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3541{
3542 int rc = VINF_SUCCESS;
3543 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
3544 {
3545 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3546 AssertRCReturn(rc, rc);
3547
3548 /* Update the exit history entry with the correct CS.BASE + RIP or just RIP. */
3549 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
3550 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->cs.u64Base + pMixedCtx->rip, true);
3551 else
3552 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->rip, false);
3553
3554 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3555 Log4Func(("RIP=%#RX64\n", pMixedCtx->rip));
3556 }
3557 return rc;
3558}
3559
3560
3561/**
3562 * Exports the guest's RSP into the guest-state area in the VMCS.
3563 *
3564 * @returns VBox status code.
3565 * @param pVCpu The cross context virtual CPU structure.
3566 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3567 * out-of-sync. Make sure to update the required fields
3568 * before using them.
3569 *
3570 * @remarks No-long-jump zone!!!
3571 */
3572static int hmR0VmxExportGuestRsp(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3573{
3574 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3575 {
3576 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3577 AssertRCReturn(rc, rc);
3578
3579 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3580 }
3581 return VINF_SUCCESS;
3582}
3583
3584
3585/**
3586 * Exports the guest's RFLAGS into the guest-state area in the VMCS.
3587 *
3588 * @returns VBox status code.
3589 * @param pVCpu The cross context virtual CPU structure.
3590 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3591 * out-of-sync. Make sure to update the required fields
3592 * before using them.
3593 *
3594 * @remarks No-long-jump zone!!!
3595 */
3596static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3597{
3598 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3599 {
3600 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3601 Let us assert it as such and use 32-bit VMWRITE. */
3602 Assert(!(pMixedCtx->rflags.u64 >> 32));
3603 X86EFLAGS fEFlags = pMixedCtx->eflags;
3604 Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3605 Assert(!(fEFlags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3606
3607 /*
3608 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3609 * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3610 * can run the real-mode guest code under Virtual 8086 mode.
3611 */
3612 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3613 {
3614 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3615 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3616 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3617 fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3618 fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3619 }
3620
3621 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3622 AssertRCReturn(rc, rc);
3623
3624 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3625 Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
3626 }
3627 return VINF_SUCCESS;
3628}
3629
3630
3631/**
3632 * Exports the guest CR0 control register into the guest-state area in the VMCS.
3633 *
3634 * The guest FPU state is always pre-loaded hence we don't need to bother about
3635 * sharing FPU related CR0 bits between the guest and host.
3636 *
3637 * @returns VBox status code.
3638 * @param pVCpu The cross context virtual CPU structure.
3639 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3640 * out-of-sync. Make sure to update the required fields
3641 * before using them.
3642 *
3643 * @remarks No-long-jump zone!!!
3644 */
3645static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3646{
3647 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
3648 {
3649 PVM pVM = pVCpu->CTX_SUFF(pVM);
3650 Assert(!RT_HI_U32(pMixedCtx->cr0));
3651 uint32_t const uShadowCR0 = pMixedCtx->cr0;
3652 uint32_t uGuestCR0 = pMixedCtx->cr0;
3653
3654 /*
3655 * Setup VT-x's view of the guest CR0.
3656 * Minimize VM-exits due to CR3 changes when we have NestedPaging.
3657 */
3658 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
3659 if (pVM->hm.s.fNestedPaging)
3660 {
3661 if (CPUMIsGuestPagingEnabled(pVCpu))
3662 {
3663 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3664 uProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3665 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3666 }
3667 else
3668 {
3669 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3670 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3671 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3672 }
3673
3674 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3675 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3676 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3677 }
3678 else
3679 {
3680 /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3681 uGuestCR0 |= X86_CR0_WP;
3682 }
3683
3684 /*
3685 * Guest FPU bits.
3686 *
3687 * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3688 * using CR0.TS.
3689 *
3690 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3691 * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3692 */
3693 uGuestCR0 |= X86_CR0_NE;
3694
3695 /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3696 bool const fInterceptMF = !(uShadowCR0 & X86_CR0_NE);
3697
3698 /*
3699 * Update exception intercepts.
3700 */
3701 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3702 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3703 {
3704 Assert(PDMVmmDevHeapIsEnabled(pVM));
3705 Assert(pVM->hm.s.vmx.pRealModeTSS);
3706 uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3707 }
3708 else
3709 {
3710 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3711 uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3712 if (fInterceptMF)
3713 uXcptBitmap |= RT_BIT(X86_XCPT_MF);
3714 }
3715
3716 /* Additional intercepts for debugging, define these yourself explicitly. */
3717#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3718 uXcptBitmap |= 0
3719 | RT_BIT(X86_XCPT_BP)
3720 | RT_BIT(X86_XCPT_DE)
3721 | RT_BIT(X86_XCPT_NM)
3722 | RT_BIT(X86_XCPT_TS)
3723 | RT_BIT(X86_XCPT_UD)
3724 | RT_BIT(X86_XCPT_NP)
3725 | RT_BIT(X86_XCPT_SS)
3726 | RT_BIT(X86_XCPT_GP)
3727 | RT_BIT(X86_XCPT_PF)
3728 | RT_BIT(X86_XCPT_MF)
3729 ;
3730#elif defined(HMVMX_ALWAYS_TRAP_PF)
3731 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
3732#endif
3733 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3734 {
3735 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3736 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3737 }
3738 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3739
3740 /*
3741 * Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW).
3742 */
3743 uint32_t fSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3744 uint32_t fZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3745 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3746 fSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3747 else
3748 Assert((fSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3749
3750 uGuestCR0 |= fSetCR0;
3751 uGuestCR0 &= fZapCR0;
3752 uGuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3753
3754 /*
3755 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3756 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3757 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3758 */
3759 uint32_t uCR0Mask = X86_CR0_PE
3760 | X86_CR0_NE
3761 | (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
3762 | X86_CR0_PG
3763 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3764 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3765 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3766
3767 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3768 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3769 * and @bugref{6944}. */
3770#if 0
3771 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3772 uCr0Mask &= ~X86_CR0_PE;
3773#endif
3774 /* Update the HMCPU's copy of the CR0 mask. */
3775 pVCpu->hm.s.vmx.u32CR0Mask = uCR0Mask;
3776
3777 /*
3778 * Finally, update VMCS fields with the CR0 values.
3779 */
3780 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, uGuestCR0);
3781 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, uShadowCR0);
3782 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, uCR0Mask);
3783 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
3784 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3785 AssertRCReturn(rc, rc);
3786 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
3787
3788 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3789
3790 Log4Func(("uCr0Mask=%#RX32 uShadowCR0=%#RX32 uGuestCR0=%#RX32 (fSetCR0=%#RX32 fZapCR0=%#RX32\n", uCR0Mask, uShadowCR0,
3791 uGuestCR0, fSetCR0, fZapCR0));
3792 }
3793
3794 return VINF_SUCCESS;
3795}
3796
3797
3798/**
3799 * Exports the guest control registers (CR3, CR4) into the guest-state area
3800 * in the VMCS.
3801 *
3802 * @returns VBox strict status code.
3803 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3804 * without unrestricted guest access and the VMMDev is not presently
3805 * mapped (e.g. EFI32).
3806 *
3807 * @param pVCpu The cross context virtual CPU structure.
3808 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3809 * out-of-sync. Make sure to update the required fields
3810 * before using them.
3811 *
3812 * @remarks No-long-jump zone!!!
3813 */
3814static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3815{
3816 int rc = VINF_SUCCESS;
3817 PVM pVM = pVCpu->CTX_SUFF(pVM);
3818
3819 /*
3820 * Guest CR2.
3821 * It's always loaded in the assembler code. Nothing to do here.
3822 */
3823
3824 /*
3825 * Guest CR3.
3826 */
3827 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
3828 {
3829 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3830 if (pVM->hm.s.fNestedPaging)
3831 {
3832 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3833
3834 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3835 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3836 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3837 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3838
3839 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3840 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3841 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3842
3843 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3844 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3845 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3846 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3847 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3848 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3849 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3850
3851 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3852 AssertRCReturn(rc, rc);
3853
3854 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3855 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3856 {
3857 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3858 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3859 {
3860 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3861 AssertRCReturn(rc, rc);
3862 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3863 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3864 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3865 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3866 AssertRCReturn(rc, rc);
3867 }
3868
3869 /*
3870 * The guest's view of its CR3 is unblemished with Nested Paging when the
3871 * guest is using paging or we have unrestricted guest execution to handle
3872 * the guest when it's not using paging.
3873 */
3874 GCPhysGuestCR3 = pMixedCtx->cr3;
3875 }
3876 else
3877 {
3878 /*
3879 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
3880 * thinks it accesses physical memory directly, we use our identity-mapped
3881 * page table to map guest-linear to guest-physical addresses. EPT takes care
3882 * of translating it to host-physical addresses.
3883 */
3884 RTGCPHYS GCPhys;
3885 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3886
3887 /* We obtain it here every time as the guest could have relocated this PCI region. */
3888 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3889 if (RT_SUCCESS(rc))
3890 { /* likely */ }
3891 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3892 {
3893 Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3894 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3895 }
3896 else
3897 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3898
3899 GCPhysGuestCR3 = GCPhys;
3900 }
3901
3902 Log4Func(("uGuestCR3=%#RGp (GstN)\n", GCPhysGuestCR3));
3903 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3904 AssertRCReturn(rc, rc);
3905 }
3906 else
3907 {
3908 /* Non-nested paging case, just use the hypervisor's CR3. */
3909 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3910
3911 Log4Func(("uGuestCR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3912 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3913 AssertRCReturn(rc, rc);
3914 }
3915
3916 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3917 }
3918
3919 /*
3920 * Guest CR4.
3921 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3922 */
3923 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
3924 {
3925 Assert(!RT_HI_U32(pMixedCtx->cr4));
3926 uint32_t uGuestCR4 = pMixedCtx->cr4;
3927
3928 /* The guest's view of its CR4 is unblemished. */
3929 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, uGuestCR4);
3930 AssertRCReturn(rc, rc);
3931 Log4Func(("uShadowCR4=%#RX32\n", uGuestCR4));
3932
3933 /*
3934 * Setup VT-x's view of the guest CR4.
3935 *
3936 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3937 * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3938 * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3939 *
3940 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3941 */
3942 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3943 {
3944 Assert(pVM->hm.s.vmx.pRealModeTSS);
3945 Assert(PDMVmmDevHeapIsEnabled(pVM));
3946 uGuestCR4 &= ~X86_CR4_VME;
3947 }
3948
3949 if (pVM->hm.s.fNestedPaging)
3950 {
3951 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3952 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3953 {
3954 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3955 uGuestCR4 |= X86_CR4_PSE;
3956 /* Our identity mapping is a 32-bit page directory. */
3957 uGuestCR4 &= ~X86_CR4_PAE;
3958 }
3959 /* else use guest CR4.*/
3960 }
3961 else
3962 {
3963 /*
3964 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3965 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3966 */
3967 switch (pVCpu->hm.s.enmShadowMode)
3968 {
3969 case PGMMODE_REAL: /* Real-mode. */
3970 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3971 case PGMMODE_32_BIT: /* 32-bit paging. */
3972 {
3973 uGuestCR4 &= ~X86_CR4_PAE;
3974 break;
3975 }
3976
3977 case PGMMODE_PAE: /* PAE paging. */
3978 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3979 {
3980 uGuestCR4 |= X86_CR4_PAE;
3981 break;
3982 }
3983
3984 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3985 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3986#ifdef VBOX_ENABLE_64_BITS_GUESTS
3987 break;
3988#endif
3989 default:
3990 AssertFailed();
3991 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3992 }
3993 }
3994
3995 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3996 uint64_t fSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3997 uint64_t fZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3998 uGuestCR4 |= fSetCR4;
3999 uGuestCR4 &= fZapCR4;
4000
4001 /* Write VT-x's view of the guest CR4 into the VMCS. */
4002 Log4Func(("uGuestCR4=%#RX32 (fSetCR4=%#RX32 fZapCR4=%#RX32)\n", uGuestCR4, fSetCR4, fZapCR4));
4003 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, uGuestCR4);
4004 AssertRCReturn(rc, rc);
4005
4006 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4007 uint32_t u32CR4Mask = X86_CR4_VME
4008 | X86_CR4_PAE
4009 | X86_CR4_PGE
4010 | X86_CR4_PSE
4011 | X86_CR4_VMXE;
4012 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4013 u32CR4Mask |= X86_CR4_OSXSAVE;
4014 if (pVM->cpum.ro.GuestFeatures.fPcid)
4015 u32CR4Mask |= X86_CR4_PCIDE;
4016 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4017 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4018 AssertRCReturn(rc, rc);
4019
4020 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4021 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4022
4023 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
4024 }
4025 return rc;
4026}
4027
4028
4029/**
4030 * Exports the guest debug registers into the guest-state area in the VMCS.
4031 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4032 *
4033 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4034 *
4035 * @returns VBox status code.
4036 * @param pVCpu The cross context virtual CPU structure.
4037 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4038 * out-of-sync. Make sure to update the required fields
4039 * before using them.
4040 *
4041 * @remarks No-long-jump zone!!!
4042 */
4043static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4044{
4045 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4046
4047#ifdef VBOX_STRICT
4048 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4049 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4050 {
4051 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4052 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4053 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4054 }
4055#endif
4056
4057 bool fSteppingDB = false;
4058 bool fInterceptMovDRx = false;
4059 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
4060 if (pVCpu->hm.s.fSingleInstruction)
4061 {
4062 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4063 PVM pVM = pVCpu->CTX_SUFF(pVM);
4064 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4065 {
4066 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4067 Assert(fSteppingDB == false);
4068 }
4069 else
4070 {
4071 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4072 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_RFLAGS;
4073 pVCpu->hm.s.fClearTrapFlag = true;
4074 fSteppingDB = true;
4075 }
4076 }
4077
4078 uint32_t uGuestDR7;
4079 if ( fSteppingDB
4080 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4081 {
4082 /*
4083 * Use the combined guest and host DRx values found in the hypervisor register set
4084 * because the debugger has breakpoints active or someone is single stepping on the
4085 * host side without a monitor trap flag.
4086 *
4087 * Note! DBGF expects a clean DR6 state before executing guest code.
4088 */
4089#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4090 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4091 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4092 {
4093 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4094 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4095 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4096 }
4097 else
4098#endif
4099 if (!CPUMIsHyperDebugStateActive(pVCpu))
4100 {
4101 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4102 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4103 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4104 }
4105
4106 /* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
4107 uGuestDR7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
4108 pVCpu->hm.s.fUsingHyperDR7 = true;
4109 fInterceptMovDRx = true;
4110 }
4111 else
4112 {
4113 /*
4114 * If the guest has enabled debug registers, we need to load them prior to
4115 * executing guest code so they'll trigger at the right time.
4116 */
4117 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD))
4118 {
4119#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4120 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4121 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4122 {
4123 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4124 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4125 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4126 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4127 }
4128 else
4129#endif
4130 if (!CPUMIsGuestDebugStateActive(pVCpu))
4131 {
4132 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4133 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4134 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4135 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4136 }
4137 Assert(!fInterceptMovDRx);
4138 }
4139 /*
4140 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4141 * must intercept #DB in order to maintain a correct DR6 guest value, and
4142 * because we need to intercept it to prevent nested #DBs from hanging the
4143 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4144 */
4145#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4146 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4147 && !CPUMIsGuestDebugStateActive(pVCpu))
4148#else
4149 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4150#endif
4151 {
4152 fInterceptMovDRx = true;
4153 }
4154
4155 /* Update DR7 with the actual guest value. */
4156 uGuestDR7 = pMixedCtx->dr[7];
4157 pVCpu->hm.s.fUsingHyperDR7 = false;
4158 }
4159
4160 if (fInterceptMovDRx)
4161 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4162 else
4163 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4164
4165 /*
4166 * Update the processor-based VM-execution controls for MOV-DRx intercepts and the monitor-trap flag.
4167 */
4168 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
4169 {
4170 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4171 AssertRCReturn(rc2, rc2);
4172 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
4173 }
4174
4175 /*
4176 * Update guest DR7.
4177 */
4178 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, uGuestDR7);
4179 AssertRCReturn(rc, rc);
4180
4181 return VINF_SUCCESS;
4182}
4183
4184
4185#ifdef VBOX_STRICT
4186/**
4187 * Strict function to validate segment registers.
4188 *
4189 * @remarks Will import guest CR0 on strict builds during validation of
4190 * segments.
4191 */
4192static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCCPUMCTX pCtx)
4193{
4194 /*
4195 * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
4196 *
4197 * The reason we check for attribute value 0 in this function and not just the unusable bit is
4198 * because hmR0VmxWriteSegmentReg() only updates the VMCS' copy of the value with the unusable bit
4199 * and doesn't change the guest-context value.
4200 */
4201 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
4202 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4203 && ( !CPUMIsGuestInRealModeEx(pCtx)
4204 && !CPUMIsGuestInV86ModeEx(pCtx)))
4205 {
4206 /* Protected mode checks */
4207 /* CS */
4208 Assert(pCtx->cs.Attr.n.u1Present);
4209 Assert(!(pCtx->cs.Attr.u & 0xf00));
4210 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4211 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4212 || !(pCtx->cs.Attr.n.u1Granularity));
4213 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4214 || (pCtx->cs.Attr.n.u1Granularity));
4215 /* CS cannot be loaded with NULL in protected mode. */
4216 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4217 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4218 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4219 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4220 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4221 else
4222 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4223 /* SS */
4224 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4225 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4226 if ( !(pCtx->cr0 & X86_CR0_PE)
4227 || pCtx->cs.Attr.n.u4Type == 3)
4228 {
4229 Assert(!pCtx->ss.Attr.n.u2Dpl);
4230 }
4231 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4232 {
4233 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4234 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4235 Assert(pCtx->ss.Attr.n.u1Present);
4236 Assert(!(pCtx->ss.Attr.u & 0xf00));
4237 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4238 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4239 || !(pCtx->ss.Attr.n.u1Granularity));
4240 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4241 || (pCtx->ss.Attr.n.u1Granularity));
4242 }
4243 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4244 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4245 {
4246 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4247 Assert(pCtx->ds.Attr.n.u1Present);
4248 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4249 Assert(!(pCtx->ds.Attr.u & 0xf00));
4250 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4251 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4252 || !(pCtx->ds.Attr.n.u1Granularity));
4253 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4254 || (pCtx->ds.Attr.n.u1Granularity));
4255 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4256 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4257 }
4258 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4259 {
4260 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4261 Assert(pCtx->es.Attr.n.u1Present);
4262 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4263 Assert(!(pCtx->es.Attr.u & 0xf00));
4264 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4265 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4266 || !(pCtx->es.Attr.n.u1Granularity));
4267 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4268 || (pCtx->es.Attr.n.u1Granularity));
4269 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4270 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4271 }
4272 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4273 {
4274 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4275 Assert(pCtx->fs.Attr.n.u1Present);
4276 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4277 Assert(!(pCtx->fs.Attr.u & 0xf00));
4278 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4279 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4280 || !(pCtx->fs.Attr.n.u1Granularity));
4281 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4282 || (pCtx->fs.Attr.n.u1Granularity));
4283 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4284 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4285 }
4286 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4287 {
4288 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4289 Assert(pCtx->gs.Attr.n.u1Present);
4290 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4291 Assert(!(pCtx->gs.Attr.u & 0xf00));
4292 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4293 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4294 || !(pCtx->gs.Attr.n.u1Granularity));
4295 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4296 || (pCtx->gs.Attr.n.u1Granularity));
4297 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4298 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4299 }
4300 /* 64-bit capable CPUs. */
4301# if HC_ARCH_BITS == 64
4302 Assert(!(pCtx->cs.u64Base >> 32));
4303 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4304 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4305 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4306# endif
4307 }
4308 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4309 || ( CPUMIsGuestInRealModeEx(pCtx)
4310 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4311 {
4312 /* Real and v86 mode checks. */
4313 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4314 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4315 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4316 {
4317 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4318 }
4319 else
4320 {
4321 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4322 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4323 }
4324
4325 /* CS */
4326 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4327 Assert(pCtx->cs.u32Limit == 0xffff);
4328 Assert(u32CSAttr == 0xf3);
4329 /* SS */
4330 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4331 Assert(pCtx->ss.u32Limit == 0xffff);
4332 Assert(u32SSAttr == 0xf3);
4333 /* DS */
4334 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4335 Assert(pCtx->ds.u32Limit == 0xffff);
4336 Assert(u32DSAttr == 0xf3);
4337 /* ES */
4338 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4339 Assert(pCtx->es.u32Limit == 0xffff);
4340 Assert(u32ESAttr == 0xf3);
4341 /* FS */
4342 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4343 Assert(pCtx->fs.u32Limit == 0xffff);
4344 Assert(u32FSAttr == 0xf3);
4345 /* GS */
4346 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4347 Assert(pCtx->gs.u32Limit == 0xffff);
4348 Assert(u32GSAttr == 0xf3);
4349 /* 64-bit capable CPUs. */
4350# if HC_ARCH_BITS == 64
4351 Assert(!(pCtx->cs.u64Base >> 32));
4352 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4353 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4354 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4355# endif
4356 }
4357}
4358#endif /* VBOX_STRICT */
4359
4360
4361/**
4362 * Writes a guest segment register into the guest-state area in the VMCS.
4363 *
4364 * @returns VBox status code.
4365 * @param pVCpu The cross context virtual CPU structure.
4366 * @param idxSel Index of the selector in the VMCS.
4367 * @param idxLimit Index of the segment limit in the VMCS.
4368 * @param idxBase Index of the segment base in the VMCS.
4369 * @param idxAccess Index of the access rights of the segment in the VMCS.
4370 * @param pSelReg Pointer to the segment selector.
4371 *
4372 * @remarks No-long-jump zone!!!
4373 */
4374static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4375 uint32_t idxAccess, PCCPUMSELREG pSelReg)
4376{
4377 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4378 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4379 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4380 AssertRCReturn(rc, rc);
4381
4382 uint32_t u32Access = pSelReg->Attr.u;
4383 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4384 {
4385 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4386 u32Access = 0xf3;
4387 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4388 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4389 }
4390 else
4391 {
4392 /*
4393 * The way to differentiate between whether this is really a null selector or was just
4394 * a selector loaded with 0 in real-mode is using the segment attributes. A selector
4395 * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4396 * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4397 * NULL selectors loaded in protected-mode have their attribute as 0.
4398 */
4399 if (!u32Access)
4400 u32Access = X86DESCATTR_UNUSABLE;
4401 }
4402
4403 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4404 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4405 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4406
4407 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4408 AssertRCReturn(rc, rc);
4409 return rc;
4410}
4411
4412
4413/**
4414 * Exports the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4415 * into the guest-state area in the VMCS.
4416 *
4417 * @returns VBox status code.
4418 * @param pVCpu The cross context virtual CPU structure.
4419 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4420 * out-of-sync. Make sure to update the required fields
4421 * before using them.
4422 *
4423 * @remarks Will import guest CR0 on strict builds during validation of
4424 * segments.
4425 * @remarks No-long-jump zone!!!
4426 */
4427static int hmR0VmxExportGuestSegmentRegs(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4428{
4429 int rc = VERR_INTERNAL_ERROR_5;
4430 PVM pVM = pVCpu->CTX_SUFF(pVM);
4431
4432 /*
4433 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4434 */
4435 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4436 {
4437 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4438 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4439 {
4440 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4441 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4442 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4443 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4444 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4445 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4446 }
4447
4448#ifdef VBOX_WITH_REM
4449 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4450 {
4451 Assert(pVM->hm.s.vmx.pRealModeTSS);
4452 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4453 if ( pVCpu->hm.s.vmx.fWasInRealMode
4454 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4455 {
4456 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4457 in real-mode (e.g. OpenBSD 4.0) */
4458 REMFlushTBs(pVM);
4459 Log4Func(("Switch to protected mode detected!\n"));
4460 pVCpu->hm.s.vmx.fWasInRealMode = false;
4461 }
4462 }
4463#endif
4464 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4465 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4466 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4467 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4468 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4469 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4470 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4471 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4472 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4473 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4474 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4475 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4476 AssertRCReturn(rc, rc);
4477
4478#ifdef VBOX_STRICT
4479 /* Validate. */
4480 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4481#endif
4482
4483 /* Update the exit history entry with the correct CS.BASE + RIP. */
4484 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4485 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->cs.u64Base + pMixedCtx->rip, true);
4486
4487 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SREG_MASK);
4488 Log4Func(("CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4489 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4490 }
4491
4492 /*
4493 * Guest TR.
4494 */
4495 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
4496 {
4497 /*
4498 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4499 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4500 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4501 */
4502 uint16_t u16Sel = 0;
4503 uint32_t u32Limit = 0;
4504 uint64_t u64Base = 0;
4505 uint32_t u32AccessRights = 0;
4506
4507 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4508 {
4509 u16Sel = pMixedCtx->tr.Sel;
4510 u32Limit = pMixedCtx->tr.u32Limit;
4511 u64Base = pMixedCtx->tr.u64Base;
4512 u32AccessRights = pMixedCtx->tr.Attr.u;
4513 }
4514 else
4515 {
4516 Assert(pVM->hm.s.vmx.pRealModeTSS);
4517 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4518
4519 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4520 RTGCPHYS GCPhys;
4521 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4522 AssertRCReturn(rc, rc);
4523
4524 X86DESCATTR DescAttr;
4525 DescAttr.u = 0;
4526 DescAttr.n.u1Present = 1;
4527 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4528
4529 u16Sel = 0;
4530 u32Limit = HM_VTX_TSS_SIZE;
4531 u64Base = GCPhys; /* in real-mode phys = virt. */
4532 u32AccessRights = DescAttr.u;
4533 }
4534
4535 /* Validate. */
4536 Assert(!(u16Sel & RT_BIT(2)));
4537 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4538 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4539 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4540 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4541 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4542 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4543 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4544 Assert( (u32Limit & 0xfff) == 0xfff
4545 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4546 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4547 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4548
4549 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4550 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4551 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4552 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4553 AssertRCReturn(rc, rc);
4554
4555 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
4556 Log4Func(("TR base=%#RX64\n", pMixedCtx->tr.u64Base));
4557 }
4558
4559 /*
4560 * Guest GDTR.
4561 */
4562 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4563 {
4564 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4565 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4566 AssertRCReturn(rc, rc);
4567
4568 /* Validate. */
4569 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4570
4571 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4572 Log4Func(("GDTR base=%#RX64\n", pMixedCtx->gdtr.pGdt));
4573 }
4574
4575 /*
4576 * Guest LDTR.
4577 */
4578 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4579 {
4580 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4581 uint32_t u32Access = 0;
4582 if (!pMixedCtx->ldtr.Attr.u)
4583 u32Access = X86DESCATTR_UNUSABLE;
4584 else
4585 u32Access = pMixedCtx->ldtr.Attr.u;
4586
4587 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4588 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4589 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4590 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4591 AssertRCReturn(rc, rc);
4592
4593 /* Validate. */
4594 if (!(u32Access & X86DESCATTR_UNUSABLE))
4595 {
4596 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4597 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4598 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4599 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4600 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4601 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4602 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4603 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4604 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4605 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4606 }
4607
4608 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4609 Log4Func(("LDTR base=%#RX64\n", pMixedCtx->ldtr.u64Base));
4610 }
4611
4612 /*
4613 * Guest IDTR.
4614 */
4615 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4616 {
4617 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4618 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4619 AssertRCReturn(rc, rc);
4620
4621 /* Validate. */
4622 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4623
4624 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4625 Log4Func(("IDTR base=%#RX64\n", pMixedCtx->idtr.pIdt));
4626 }
4627
4628 return VINF_SUCCESS;
4629}
4630
4631
4632/**
4633 * Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4634 * areas.
4635 *
4636 * These MSRs will automatically be loaded to the host CPU on every successful
4637 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4638 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4639 * -not- updated here for performance reasons. See hmR0VmxExportHostMsrs().
4640 *
4641 * Also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4642 *
4643 * @returns VBox status code.
4644 * @param pVCpu The cross context virtual CPU structure.
4645 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4646 * out-of-sync. Make sure to update the required fields
4647 * before using them.
4648 *
4649 * @remarks No-long-jump zone!!!
4650 */
4651static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4652{
4653 AssertPtr(pVCpu);
4654 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4655
4656 /*
4657 * MSRs that we use the auto-load/store MSR area in the VMCS.
4658 * For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
4659 */
4660 PVM pVM = pVCpu->CTX_SUFF(pVM);
4661 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4662 {
4663 if (pVM->hm.s.fAllow64BitGuests)
4664 {
4665#if HC_ARCH_BITS == 32
4666 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4667 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4668 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4669 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4670 AssertRCReturn(rc, rc);
4671# ifdef LOG_ENABLED
4672 PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4673 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4674 Log4Func(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4675# endif
4676#endif
4677 }
4678 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4679 }
4680
4681 /*
4682 * Guest Sysenter MSRs.
4683 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4684 * VM-exits on WRMSRs for these MSRs.
4685 */
4686 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4687 {
4688 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4689 {
4690 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs);
4691 AssertRCReturn(rc, rc);
4692 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4693 }
4694
4695 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4696 {
4697 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip);
4698 AssertRCReturn(rc, rc);
4699 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4700 }
4701
4702 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4703 {
4704 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp);
4705 AssertRCReturn(rc, rc);
4706 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4707 }
4708 }
4709
4710 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4711 {
4712 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4713 {
4714 /*
4715 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4716 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4717 */
4718 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4719 {
4720 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4721 AssertRCReturn(rc,rc);
4722 Log4Func(("EFER=%#RX64\n", pMixedCtx->msrEFER));
4723 }
4724 else
4725 {
4726 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4727 NULL /* pfAddedAndUpdated */);
4728 AssertRCReturn(rc, rc);
4729
4730 /* We need to intercept reads too, see @bugref{7386#c16}. */
4731 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4732 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4733 Log4Func(("MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pMixedCtx->msrEFER,
4734 pVCpu->hm.s.vmx.cMsrs));
4735 }
4736 }
4737 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4738 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4739 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4740 }
4741
4742 return VINF_SUCCESS;
4743}
4744
4745
4746#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4747/**
4748 * Check if guest state allows safe use of 32-bit switcher again.
4749 *
4750 * Segment bases and protected mode structures must be 32-bit addressable
4751 * because the 32-bit switcher will ignore high dword when writing these VMCS
4752 * fields. See @bugref{8432} for details.
4753 *
4754 * @returns true if safe, false if must continue to use the 64-bit switcher.
4755 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4756 * out-of-sync. Make sure to update the required fields
4757 * before using them.
4758 *
4759 * @remarks No-long-jump zone!!!
4760 */
4761static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pMixedCtx)
4762{
4763 if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4764 if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4765 if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4766 if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4767 if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4768 if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4769 if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4770 if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4771 if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4772 if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4773
4774 /* All good, bases are 32-bit. */
4775 return true;
4776}
4777#endif
4778
4779
4780/**
4781 * Selects up the appropriate function to run guest code.
4782 *
4783 * @returns VBox status code.
4784 * @param pVCpu The cross context virtual CPU structure.
4785 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4786 * out-of-sync. Make sure to update the required fields
4787 * before using them.
4788 *
4789 * @remarks No-long-jump zone!!!
4790 */
4791static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4792{
4793 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4794 {
4795#ifndef VBOX_ENABLE_64_BITS_GUESTS
4796 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4797#endif
4798 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4799#if HC_ARCH_BITS == 32
4800 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4801 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4802 {
4803#ifdef VBOX_STRICT
4804 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4805 {
4806 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4807 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4808 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4809 | HM_CHANGED_VMX_ENTRY_CTLS
4810 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4811 }
4812#endif
4813 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4814
4815 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4816 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4817 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4818 Log4Func(("Selected 64-bit switcher\n"));
4819 }
4820#else
4821 /* 64-bit host. */
4822 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4823#endif
4824 }
4825 else
4826 {
4827 /* Guest is not in long mode, use the 32-bit handler. */
4828#if HC_ARCH_BITS == 32
4829 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4830 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4831 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4832 {
4833# ifdef VBOX_STRICT
4834 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4835 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4836 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4837 | HM_CHANGED_VMX_ENTRY_CTLS
4838 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4839# endif
4840 }
4841# ifdef VBOX_ENABLE_64_BITS_GUESTS
4842 /*
4843 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
4844 * design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
4845 * switcher flag because now we know the guest is in a sane state where it's safe
4846 * to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4847 * the much faster 32-bit switcher again.
4848 */
4849 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4850 {
4851 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4852 Log4Func(("Selected 32-bit switcher\n"));
4853 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4854 }
4855 else
4856 {
4857 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4858 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4859 || hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
4860 {
4861 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4862 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4863 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
4864 | HM_CHANGED_VMX_ENTRY_CTLS
4865 | HM_CHANGED_VMX_EXIT_CTLS
4866 | HM_CHANGED_HOST_CONTEXT);
4867 Log4Func(("Selected 32-bit switcher (safe)\n"));
4868 }
4869 }
4870# else
4871 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4872# endif
4873#else
4874 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4875#endif
4876 }
4877 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4878 return VINF_SUCCESS;
4879}
4880
4881
4882/**
4883 * Wrapper for running the guest code in VT-x.
4884 *
4885 * @returns VBox status code, no informational status codes.
4886 * @param pVM The cross context VM structure.
4887 * @param pVCpu The cross context virtual CPU structure.
4888 * @param pCtx Pointer to the guest-CPU context.
4889 *
4890 * @remarks No-long-jump zone!!!
4891 */
4892DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4893{
4894 /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4895 pCtx->fExtrn |= HMVMX_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
4896
4897 /*
4898 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4899 * floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
4900 * callee-saved and thus the need for this XMM wrapper.
4901 *
4902 * See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
4903 */
4904 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4905 /** @todo Add stats for resume vs launch. */
4906#ifdef VBOX_WITH_KERNEL_USING_XMM
4907 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4908#else
4909 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4910#endif
4911 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4912 return rc;
4913}
4914
4915
4916/**
4917 * Reports world-switch error and dumps some useful debug info.
4918 *
4919 * @param pVM The cross context VM structure.
4920 * @param pVCpu The cross context virtual CPU structure.
4921 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4922 * @param pCtx Pointer to the guest-CPU context.
4923 * @param pVmxTransient Pointer to the VMX transient structure (only
4924 * exitReason updated).
4925 */
4926static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4927{
4928 Assert(pVM);
4929 Assert(pVCpu);
4930 Assert(pCtx);
4931 Assert(pVmxTransient);
4932 HMVMX_ASSERT_PREEMPT_SAFE();
4933
4934 Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
4935 switch (rcVMRun)
4936 {
4937 case VERR_VMX_INVALID_VMXON_PTR:
4938 AssertFailed();
4939 break;
4940 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4941 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4942 {
4943 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4944 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4945 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4946 AssertRC(rc);
4947
4948 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4949 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4950 Cannot do it here as we may have been long preempted. */
4951
4952#ifdef VBOX_STRICT
4953 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4954 pVmxTransient->uExitReason));
4955 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4956 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4957 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4958 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4959 else
4960 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4961 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4962 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4963
4964 /* VMX control bits. */
4965 uint32_t u32Val;
4966 uint64_t u64Val;
4967 RTHCUINTREG uHCReg;
4968 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4969 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4970 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4971 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4972 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
4973 {
4974 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4975 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4976 }
4977 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4978 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4979 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4980 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4981 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4982 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4983 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4984 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4985 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4986 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4987 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4988 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4989 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4990 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4991 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4992 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4993 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4994 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4995 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4996 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4997 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4998 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4999 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5000 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5001 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5002 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5003 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5004 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5005 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5006 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5007 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5008 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5009 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5010 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5011 if (pVM->hm.s.fNestedPaging)
5012 {
5013 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5014 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5015 }
5016
5017 /* Guest bits. */
5018 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5019 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5020 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5021 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5022 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5023 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5024 if (pVM->hm.s.vmx.fVpid)
5025 {
5026 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5027 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5028 }
5029
5030 /* Host bits. */
5031 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5032 Log4(("Host CR0 %#RHr\n", uHCReg));
5033 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5034 Log4(("Host CR3 %#RHr\n", uHCReg));
5035 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5036 Log4(("Host CR4 %#RHr\n", uHCReg));
5037
5038 RTGDTR HostGdtr;
5039 PCX86DESCHC pDesc;
5040 ASMGetGDTR(&HostGdtr);
5041 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5042 Log4(("Host CS %#08x\n", u32Val));
5043 if (u32Val < HostGdtr.cbGdt)
5044 {
5045 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5046 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5047 }
5048
5049 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5050 Log4(("Host DS %#08x\n", u32Val));
5051 if (u32Val < HostGdtr.cbGdt)
5052 {
5053 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5054 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5055 }
5056
5057 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5058 Log4(("Host ES %#08x\n", u32Val));
5059 if (u32Val < HostGdtr.cbGdt)
5060 {
5061 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5062 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5063 }
5064
5065 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5066 Log4(("Host FS %#08x\n", u32Val));
5067 if (u32Val < HostGdtr.cbGdt)
5068 {
5069 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5070 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5071 }
5072
5073 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5074 Log4(("Host GS %#08x\n", u32Val));
5075 if (u32Val < HostGdtr.cbGdt)
5076 {
5077 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5078 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5079 }
5080
5081 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5082 Log4(("Host SS %#08x\n", u32Val));
5083 if (u32Val < HostGdtr.cbGdt)
5084 {
5085 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5086 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5087 }
5088
5089 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5090 Log4(("Host TR %#08x\n", u32Val));
5091 if (u32Val < HostGdtr.cbGdt)
5092 {
5093 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5094 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5095 }
5096
5097 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5098 Log4(("Host TR Base %#RHv\n", uHCReg));
5099 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5100 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5101 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5102 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5103 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5104 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5105 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5106 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5107 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5108 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5109 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5110 Log4(("Host RSP %#RHv\n", uHCReg));
5111 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5112 Log4(("Host RIP %#RHv\n", uHCReg));
5113# if HC_ARCH_BITS == 64
5114 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5115 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5116 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5117 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5118 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5119 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5120# endif
5121#endif /* VBOX_STRICT */
5122 break;
5123 }
5124
5125 default:
5126 /* Impossible */
5127 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5128 break;
5129 }
5130 NOREF(pVM); NOREF(pCtx);
5131}
5132
5133
5134#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5135#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5136# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5137#endif
5138#ifdef VBOX_STRICT
5139static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5140{
5141 switch (idxField)
5142 {
5143 case VMX_VMCS_GUEST_RIP:
5144 case VMX_VMCS_GUEST_RSP:
5145 case VMX_VMCS_GUEST_SYSENTER_EIP:
5146 case VMX_VMCS_GUEST_SYSENTER_ESP:
5147 case VMX_VMCS_GUEST_GDTR_BASE:
5148 case VMX_VMCS_GUEST_IDTR_BASE:
5149 case VMX_VMCS_GUEST_CS_BASE:
5150 case VMX_VMCS_GUEST_DS_BASE:
5151 case VMX_VMCS_GUEST_ES_BASE:
5152 case VMX_VMCS_GUEST_FS_BASE:
5153 case VMX_VMCS_GUEST_GS_BASE:
5154 case VMX_VMCS_GUEST_SS_BASE:
5155 case VMX_VMCS_GUEST_LDTR_BASE:
5156 case VMX_VMCS_GUEST_TR_BASE:
5157 case VMX_VMCS_GUEST_CR3:
5158 return true;
5159 }
5160 return false;
5161}
5162
5163static bool hmR0VmxIsValidReadField(uint32_t idxField)
5164{
5165 switch (idxField)
5166 {
5167 /* Read-only fields. */
5168 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5169 return true;
5170 }
5171 /* Remaining readable fields should also be writable. */
5172 return hmR0VmxIsValidWriteField(idxField);
5173}
5174#endif /* VBOX_STRICT */
5175
5176
5177/**
5178 * Executes the specified handler in 64-bit mode.
5179 *
5180 * @returns VBox status code (no informational status codes).
5181 * @param pVCpu The cross context virtual CPU structure.
5182 * @param enmOp The operation to perform.
5183 * @param cParams Number of parameters.
5184 * @param paParam Array of 32-bit parameters.
5185 */
5186VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp,
5187 uint32_t cParams, uint32_t *paParam)
5188{
5189 PVM pVM = pVCpu->CTX_SUFF(pVM);
5190 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5191 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5192 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5193 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5194
5195#ifdef VBOX_STRICT
5196 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5197 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5198
5199 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5200 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5201#endif
5202
5203 /* Disable interrupts. */
5204 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5205
5206#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5207 RTCPUID idHostCpu = RTMpCpuId();
5208 CPUMR0SetLApic(pVCpu, idHostCpu);
5209#endif
5210
5211 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5212 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5213
5214 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5215 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5216 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5217
5218 /* Leave VMX Root Mode. */
5219 VMXDisable();
5220
5221 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5222
5223 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5224 CPUMSetHyperEIP(pVCpu, enmOp);
5225 for (int i = (int)cParams - 1; i >= 0; i--)
5226 CPUMPushHyper(pVCpu, paParam[i]);
5227
5228 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5229
5230 /* Call the switcher. */
5231 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5232 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5233
5234 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5235 /* Make sure the VMX instructions don't cause #UD faults. */
5236 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5237
5238 /* Re-enter VMX Root Mode */
5239 int rc2 = VMXEnable(HCPhysCpuPage);
5240 if (RT_FAILURE(rc2))
5241 {
5242 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5243 ASMSetFlags(fOldEFlags);
5244 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5245 return rc2;
5246 }
5247
5248 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5249 AssertRC(rc2);
5250 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5251 Assert(!(ASMGetFlags() & X86_EFL_IF));
5252 ASMSetFlags(fOldEFlags);
5253 return rc;
5254}
5255
5256
5257/**
5258 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5259 * supporting 64-bit guests.
5260 *
5261 * @returns VBox status code.
5262 * @param fResume Whether to VMLAUNCH or VMRESUME.
5263 * @param pCtx Pointer to the guest-CPU context.
5264 * @param pCache Pointer to the VMCS cache.
5265 * @param pVM The cross context VM structure.
5266 * @param pVCpu The cross context virtual CPU structure.
5267 */
5268DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5269{
5270 NOREF(fResume);
5271
5272 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5273 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5274
5275#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5276 pCache->uPos = 1;
5277 pCache->interPD = PGMGetInterPaeCR3(pVM);
5278 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5279#endif
5280
5281#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5282 pCache->TestIn.HCPhysCpuPage = 0;
5283 pCache->TestIn.HCPhysVmcs = 0;
5284 pCache->TestIn.pCache = 0;
5285 pCache->TestOut.HCPhysVmcs = 0;
5286 pCache->TestOut.pCache = 0;
5287 pCache->TestOut.pCtx = 0;
5288 pCache->TestOut.eflags = 0;
5289#else
5290 NOREF(pCache);
5291#endif
5292
5293 uint32_t aParam[10];
5294 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5295 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5296 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5297 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5298 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5299 aParam[5] = 0;
5300 aParam[6] = VM_RC_ADDR(pVM, pVM);
5301 aParam[7] = 0;
5302 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5303 aParam[9] = 0;
5304
5305#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5306 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5307 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5308#endif
5309 int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5310
5311#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5312 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5313 Assert(pCtx->dr[4] == 10);
5314 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5315#endif
5316
5317#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5318 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5319 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5320 pVCpu->hm.s.vmx.HCPhysVmcs));
5321 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5322 pCache->TestOut.HCPhysVmcs));
5323 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5324 pCache->TestOut.pCache));
5325 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5326 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5327 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5328 pCache->TestOut.pCtx));
5329 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5330#endif
5331 NOREF(pCtx);
5332 return rc;
5333}
5334
5335
5336/**
5337 * Initialize the VMCS-Read cache.
5338 *
5339 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5340 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5341 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5342 * (those that have a 32-bit FULL & HIGH part).
5343 *
5344 * @returns VBox status code.
5345 * @param pVM The cross context VM structure.
5346 * @param pVCpu The cross context virtual CPU structure.
5347 */
5348static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5349{
5350#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5351{ \
5352 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5353 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5354 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5355 ++cReadFields; \
5356}
5357
5358 AssertPtr(pVM);
5359 AssertPtr(pVCpu);
5360 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5361 uint32_t cReadFields = 0;
5362
5363 /*
5364 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5365 * and serve to indicate exceptions to the rules.
5366 */
5367
5368 /* Guest-natural selector base fields. */
5369#if 0
5370 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5371 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5372 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5373#endif
5374 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5375 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5376 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5377 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5378 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5379 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5380 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5381 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5382 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5383 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5384 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5385 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5386#if 0
5387 /* Unused natural width guest-state fields. */
5388 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5389 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5390#endif
5391 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5392 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5393
5394 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
5395 these 64-bit fields (using "FULL" and "HIGH" fields). */
5396#if 0
5397 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5398 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5399 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5400 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5401 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5402 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5403 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5404 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5405 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5406#endif
5407
5408 /* Natural width guest-state fields. */
5409 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5410#if 0
5411 /* Currently unused field. */
5412 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5413#endif
5414
5415 if (pVM->hm.s.fNestedPaging)
5416 {
5417 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5418 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5419 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5420 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5421 }
5422 else
5423 {
5424 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5425 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5426 }
5427
5428#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5429 return VINF_SUCCESS;
5430}
5431
5432
5433/**
5434 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5435 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5436 * darwin, running 64-bit guests).
5437 *
5438 * @returns VBox status code.
5439 * @param pVCpu The cross context virtual CPU structure.
5440 * @param idxField The VMCS field encoding.
5441 * @param u64Val 16, 32 or 64-bit value.
5442 */
5443VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5444{
5445 int rc;
5446 switch (idxField)
5447 {
5448 /*
5449 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5450 */
5451 /* 64-bit Control fields. */
5452 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5453 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5454 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5455 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5456 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5457 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5458 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5459 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5460 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5461 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5462 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5463 case VMX_VMCS64_CTRL_EPTP_FULL:
5464 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5465 /* 64-bit Guest-state fields. */
5466 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5467 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5468 case VMX_VMCS64_GUEST_PAT_FULL:
5469 case VMX_VMCS64_GUEST_EFER_FULL:
5470 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5471 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5472 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5473 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5474 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5475 /* 64-bit Host-state fields. */
5476 case VMX_VMCS64_HOST_PAT_FULL:
5477 case VMX_VMCS64_HOST_EFER_FULL:
5478 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5479 {
5480 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5481 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5482 break;
5483 }
5484
5485 /*
5486 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5487 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5488 */
5489 /* Natural-width Guest-state fields. */
5490 case VMX_VMCS_GUEST_CR3:
5491 case VMX_VMCS_GUEST_ES_BASE:
5492 case VMX_VMCS_GUEST_CS_BASE:
5493 case VMX_VMCS_GUEST_SS_BASE:
5494 case VMX_VMCS_GUEST_DS_BASE:
5495 case VMX_VMCS_GUEST_FS_BASE:
5496 case VMX_VMCS_GUEST_GS_BASE:
5497 case VMX_VMCS_GUEST_LDTR_BASE:
5498 case VMX_VMCS_GUEST_TR_BASE:
5499 case VMX_VMCS_GUEST_GDTR_BASE:
5500 case VMX_VMCS_GUEST_IDTR_BASE:
5501 case VMX_VMCS_GUEST_RSP:
5502 case VMX_VMCS_GUEST_RIP:
5503 case VMX_VMCS_GUEST_SYSENTER_ESP:
5504 case VMX_VMCS_GUEST_SYSENTER_EIP:
5505 {
5506 if (!(RT_HI_U32(u64Val)))
5507 {
5508 /* If this field is 64-bit, VT-x will zero out the top bits. */
5509 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5510 }
5511 else
5512 {
5513 /* Assert that only the 32->64 switcher case should ever come here. */
5514 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5515 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5516 }
5517 break;
5518 }
5519
5520 default:
5521 {
5522 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5523 rc = VERR_INVALID_PARAMETER;
5524 break;
5525 }
5526 }
5527 AssertRCReturn(rc, rc);
5528 return rc;
5529}
5530
5531
5532/**
5533 * Queue up a VMWRITE by using the VMCS write cache.
5534 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5535 *
5536 * @param pVCpu The cross context virtual CPU structure.
5537 * @param idxField The VMCS field encoding.
5538 * @param u64Val 16, 32 or 64-bit value.
5539 */
5540VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5541{
5542 AssertPtr(pVCpu);
5543 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5544
5545 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5546 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5547
5548 /* Make sure there are no duplicates. */
5549 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5550 {
5551 if (pCache->Write.aField[i] == idxField)
5552 {
5553 pCache->Write.aFieldVal[i] = u64Val;
5554 return VINF_SUCCESS;
5555 }
5556 }
5557
5558 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5559 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5560 pCache->Write.cValidEntries++;
5561 return VINF_SUCCESS;
5562}
5563#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5564
5565
5566/**
5567 * Sets up the usage of TSC-offsetting and updates the VMCS.
5568 *
5569 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5570 * VMX preemption timer.
5571 *
5572 * @returns VBox status code.
5573 * @param pVM The cross context VM structure.
5574 * @param pVCpu The cross context virtual CPU structure.
5575 *
5576 * @remarks No-long-jump zone!!!
5577 */
5578static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5579{
5580 int rc;
5581 bool fOffsettedTsc;
5582 bool fParavirtTsc;
5583 if (pVM->hm.s.vmx.fUsePreemptTimer)
5584 {
5585 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5586 &fOffsettedTsc, &fParavirtTsc);
5587
5588 /* Make sure the returned values have sane upper and lower boundaries. */
5589 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5590 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5591 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5592 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5593
5594 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5595 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5596 }
5597 else
5598 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5599
5600 /** @todo later optimize this to be done elsewhere and not before every
5601 * VM-entry. */
5602 if (fParavirtTsc)
5603 {
5604 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5605 information before every VM-entry, hence disable it for performance sake. */
5606#if 0
5607 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5608 AssertRC(rc);
5609#endif
5610 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5611 }
5612
5613 if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5614 {
5615 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5616 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5617
5618 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5619 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5620 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5621 }
5622 else
5623 {
5624 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5625 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5626 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5627 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5628 }
5629}
5630
5631
5632/**
5633 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5634 * VM-exit interruption info type.
5635 *
5636 * @returns The IEM exception flags.
5637 * @param uVector The event vector.
5638 * @param uVmxVectorType The VMX event type.
5639 *
5640 * @remarks This function currently only constructs flags required for
5641 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5642 * and CR2 aspects of an exception are not included).
5643 */
5644static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5645{
5646 uint32_t fIemXcptFlags;
5647 switch (uVmxVectorType)
5648 {
5649 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5650 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5651 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5652 break;
5653
5654 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5655 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5656 break;
5657
5658 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5659 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5660 break;
5661
5662 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5663 {
5664 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5665 if (uVector == X86_XCPT_BP)
5666 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5667 else if (uVector == X86_XCPT_OF)
5668 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5669 else
5670 {
5671 fIemXcptFlags = 0;
5672 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5673 }
5674 break;
5675 }
5676
5677 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5678 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5679 break;
5680
5681 default:
5682 fIemXcptFlags = 0;
5683 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5684 break;
5685 }
5686 return fIemXcptFlags;
5687}
5688
5689
5690/**
5691 * Sets an event as a pending event to be injected into the guest.
5692 *
5693 * @param pVCpu The cross context virtual CPU structure.
5694 * @param u32IntInfo The VM-entry interruption-information field.
5695 * @param cbInstr The VM-entry instruction length in bytes (for software
5696 * interrupts, exceptions and privileged software
5697 * exceptions).
5698 * @param u32ErrCode The VM-entry exception error code.
5699 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5700 * page-fault.
5701 *
5702 * @remarks Statistics counter assumes this is a guest event being injected or
5703 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5704 * always incremented.
5705 */
5706DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5707 RTGCUINTPTR GCPtrFaultAddress)
5708{
5709 Assert(!pVCpu->hm.s.Event.fPending);
5710 pVCpu->hm.s.Event.fPending = true;
5711 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5712 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5713 pVCpu->hm.s.Event.cbInstr = cbInstr;
5714 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5715}
5716
5717
5718/**
5719 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5720 *
5721 * @param pVCpu The cross context virtual CPU structure.
5722 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5723 * out-of-sync. Make sure to update the required fields
5724 * before using them.
5725 */
5726DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5727{
5728 NOREF(pMixedCtx);
5729 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5730 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5731 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5732 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5733}
5734
5735
5736/**
5737 * Handle a condition that occurred while delivering an event through the guest
5738 * IDT.
5739 *
5740 * @returns Strict VBox status code (i.e. informational status codes too).
5741 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5742 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5743 * to continue execution of the guest which will delivery the \#DF.
5744 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5745 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5746 *
5747 * @param pVCpu The cross context virtual CPU structure.
5748 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5749 * out-of-sync. Make sure to update the required fields
5750 * before using them.
5751 * @param pVmxTransient Pointer to the VMX transient structure.
5752 *
5753 * @remarks No-long-jump zone!!!
5754 */
5755static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5756{
5757 uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5758
5759 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5760 rc2 |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5761 AssertRCReturn(rc2, rc2);
5762
5763 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5764 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5765 {
5766 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5767 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5768
5769 /*
5770 * If the event was a software interrupt (generated with INT n) or a software exception
5771 * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
5772 * can handle the VM-exit and continue guest execution which will re-execute the
5773 * instruction rather than re-injecting the exception, as that can cause premature
5774 * trips to ring-3 before injection and involve TRPM which currently has no way of
5775 * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
5776 * the problem).
5777 */
5778 IEMXCPTRAISE enmRaise;
5779 IEMXCPTRAISEINFO fRaiseInfo;
5780 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5781 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5782 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5783 {
5784 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5785 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5786 }
5787 else if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5788 {
5789 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5790 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5791 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5792 /** @todo Make AssertMsgReturn as just AssertMsg later. */
5793 AssertMsgReturn(uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT,
5794 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5795 uExitVectorType), VERR_VMX_IPE_5);
5796
5797 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5798
5799 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5800 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
5801 {
5802 pVmxTransient->fVectoringPF = true;
5803 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5804 }
5805 }
5806 else
5807 {
5808 /*
5809 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5810 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5811 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
5812 */
5813 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5814 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5815 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5816 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5817 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5818 }
5819
5820 /*
5821 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5822 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5823 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5824 * subsequent VM-entry would fail.
5825 *
5826 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5827 */
5828 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5829 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5830 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
5831 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
5832 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5833 {
5834 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5835 }
5836
5837 switch (enmRaise)
5838 {
5839 case IEMXCPTRAISE_CURRENT_XCPT:
5840 {
5841 Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
5842 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
5843 Assert(rcStrict == VINF_SUCCESS);
5844 break;
5845 }
5846
5847 case IEMXCPTRAISE_PREV_EVENT:
5848 {
5849 uint32_t u32ErrCode;
5850 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5851 {
5852 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5853 AssertRCReturn(rc2, rc2);
5854 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5855 }
5856 else
5857 u32ErrCode = 0;
5858
5859 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
5860 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5861 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5862 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5863
5864 Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
5865 pVCpu->hm.s.Event.u32ErrCode));
5866 Assert(rcStrict == VINF_SUCCESS);
5867 break;
5868 }
5869
5870 case IEMXCPTRAISE_REEXEC_INSTR:
5871 Assert(rcStrict == VINF_SUCCESS);
5872 break;
5873
5874 case IEMXCPTRAISE_DOUBLE_FAULT:
5875 {
5876 /*
5877 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
5878 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
5879 */
5880 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
5881 {
5882 pVmxTransient->fVectoringDoublePF = true;
5883 Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
5884 pMixedCtx->cr2));
5885 rcStrict = VINF_SUCCESS;
5886 }
5887 else
5888 {
5889 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5890 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5891 Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
5892 uIdtVector, uExitVector));
5893 rcStrict = VINF_HM_DOUBLE_FAULT;
5894 }
5895 break;
5896 }
5897
5898 case IEMXCPTRAISE_TRIPLE_FAULT:
5899 {
5900 Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
5901 rcStrict = VINF_EM_RESET;
5902 break;
5903 }
5904
5905 case IEMXCPTRAISE_CPU_HANG:
5906 {
5907 Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
5908 rcStrict = VERR_EM_GUEST_CPU_HANG;
5909 break;
5910 }
5911
5912 default:
5913 {
5914 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
5915 rcStrict = VERR_VMX_IPE_2;
5916 break;
5917 }
5918 }
5919 }
5920 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5921 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5922 && uExitVector != X86_XCPT_DF
5923 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5924 {
5925 /*
5926 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5927 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5928 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5929 */
5930 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5931 {
5932 Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
5933 VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5934 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5935 }
5936 }
5937
5938 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
5939 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
5940 return rcStrict;
5941}
5942
5943
5944/**
5945 * Imports a guest segment register from the current VMCS into
5946 * the guest-CPU context.
5947 *
5948 * @returns VBox status code.
5949 * @param pVCpu The cross context virtual CPU structure.
5950 * @param idxSel Index of the selector in the VMCS.
5951 * @param idxLimit Index of the segment limit in the VMCS.
5952 * @param idxBase Index of the segment base in the VMCS.
5953 * @param idxAccess Index of the access rights of the segment in the VMCS.
5954 * @param pSelReg Pointer to the segment selector.
5955 *
5956 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
5957 * do not log!
5958 *
5959 * @remarks Never call this function directly!!! Use the
5960 * HMVMX_IMPORT_SREG() macro as that takes care
5961 * of whether to read from the VMCS cache or not.
5962 */
5963static int hmR0VmxImportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5964 PCPUMSELREG pSelReg)
5965{
5966 NOREF(pVCpu);
5967
5968 uint32_t u32Sel;
5969 uint32_t u32Limit;
5970 uint32_t u32Attr;
5971 uint64_t u64Base;
5972 int rc = VMXReadVmcs32(idxSel, &u32Sel);
5973 rc |= VMXReadVmcs32(idxLimit, &u32Limit);
5974 rc |= VMXReadVmcs32(idxAccess, &u32Attr);
5975 rc |= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
5976 AssertRCReturn(rc, rc);
5977
5978 pSelReg->Sel = (uint16_t)u32Sel;
5979 pSelReg->ValidSel = (uint16_t)u32Sel;
5980 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5981 pSelReg->u32Limit = u32Limit;
5982 pSelReg->u64Base = u64Base;
5983 pSelReg->Attr.u = u32Attr;
5984
5985 /*
5986 * If VT-x marks the segment as unusable, most other bits remain undefined:
5987 * - For CS the L, D and G bits have meaning.
5988 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
5989 * - For the remaining data segments no bits are defined.
5990 *
5991 * The present bit and the unusable bit has been observed to be set at the
5992 * same time (the selector was supposed to be invalid as we started executing
5993 * a V8086 interrupt in ring-0).
5994 *
5995 * What should be important for the rest of the VBox code, is that the P bit is
5996 * cleared. Some of the other VBox code recognizes the unusable bit, but
5997 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
5998 * safe side here, we'll strip off P and other bits we don't care about. If
5999 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6000 *
6001 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6002 */
6003 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6004 {
6005 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6006
6007 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6008 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6009 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6010#ifdef VBOX_STRICT
6011 VMMRZCallRing3Disable(pVCpu);
6012 Log4Func(("Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Sel, pSelReg->Attr.u));
6013# ifdef DEBUG_bird
6014 AssertMsg((u32Attr & ~X86DESCATTR_P) == pSelReg->Attr.u,
6015 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6016 idxSel, u32Sel, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6017# endif
6018 VMMRZCallRing3Enable(pVCpu);
6019#endif
6020 }
6021 return VINF_SUCCESS;
6022}
6023
6024
6025/**
6026 * Imports the guest RIP from the VMCS back into the guest-CPU context.
6027 *
6028 * @returns VBox status code.
6029 * @param pVCpu The cross context virtual CPU structure.
6030 *
6031 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6032 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6033 * instead!!!
6034 */
6035DECLINLINE(int) hmR0VmxImportGuestRip(PVMCPU pVCpu)
6036{
6037 uint64_t u64Val;
6038 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6039 if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
6040 {
6041 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6042 if (RT_SUCCESS(rc))
6043 {
6044 pCtx->rip = u64Val;
6045 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
6046 }
6047 return rc;
6048 }
6049 return VINF_SUCCESS;
6050}
6051
6052
6053/**
6054 * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
6055 *
6056 * @returns VBox status code.
6057 * @param pVCpu The cross context virtual CPU structure.
6058 *
6059 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6060 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6061 * instead!!!
6062 */
6063DECLINLINE(int) hmR0VmxImportGuestRFlags(PVMCPU pVCpu)
6064{
6065 uint32_t u32Val;
6066 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6067 if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
6068 {
6069 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
6070 if (RT_SUCCESS(rc))
6071 {
6072 pCtx->eflags.u32 = u32Val;
6073
6074 /* Restore eflags for real-on-v86-mode hack. */
6075 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6076 {
6077 pCtx->eflags.Bits.u1VM = 0;
6078 pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6079 }
6080 }
6081 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
6082 return rc;
6083 }
6084 return VINF_SUCCESS;
6085}
6086
6087
6088/**
6089 * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
6090 * context.
6091 *
6092 * @returns VBox status code.
6093 * @param pVCpu The cross context virtual CPU structure.
6094 *
6095 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6096 * do not log!
6097 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6098 * instead!!!
6099 */
6100DECLINLINE(int) hmR0VmxImportGuestIntrState(PVMCPU pVCpu)
6101{
6102 uint32_t u32Val;
6103 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6104 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32Val);
6105 if (RT_SUCCESS(rc))
6106 {
6107 /*
6108 * We additionally have a requirement to import RIP, RFLAGS depending on whether we
6109 * might need them in hmR0VmxEvaluatePendingEvent().
6110 */
6111 if (!u32Val)
6112 {
6113 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6114 {
6115 rc = hmR0VmxImportGuestRip(pVCpu);
6116 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6117 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6118 }
6119
6120 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6121 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6122 }
6123 else
6124 {
6125 rc = hmR0VmxImportGuestRip(pVCpu);
6126 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6127
6128 if (u32Val & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6129 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6130 {
6131 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
6132 }
6133 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6134 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6135
6136 if (u32Val & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6137 {
6138 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6139 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6140 }
6141 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6142 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6143 }
6144 }
6145 return rc;
6146}
6147
6148
6149/**
6150 * Worker for VMXR0ImportStateOnDemand.
6151 *
6152 * @returns VBox status code.
6153 * @param pVCpu The cross context virtual CPU structure.
6154 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6155 */
6156static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
6157{
6158#define VMXLOCAL_BREAK_RC(a_rc) \
6159 if (RT_FAILURE(a_rc)) \
6160 break
6161
6162 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
6163
6164 int rc = VINF_SUCCESS;
6165 PVM pVM = pVCpu->CTX_SUFF(pVM);
6166 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6167 uint64_t u64Val;
6168 uint32_t u32Val;
6169
6170 Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
6171
6172 /*
6173 * We disable interrupts to make the updating of the state and in particular
6174 * the fExtrn modification atomic wrt to preemption hooks.
6175 */
6176 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
6177
6178 fWhat &= pCtx->fExtrn;
6179 if (fWhat)
6180 {
6181 do
6182 {
6183 if (fWhat & CPUMCTX_EXTRN_RIP)
6184 {
6185 rc = hmR0VmxImportGuestRip(pVCpu);
6186 VMXLOCAL_BREAK_RC(rc);
6187 }
6188
6189 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
6190 {
6191 rc = hmR0VmxImportGuestRFlags(pVCpu);
6192 VMXLOCAL_BREAK_RC(rc);
6193 }
6194
6195 if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
6196 {
6197 rc = hmR0VmxImportGuestIntrState(pVCpu);
6198 VMXLOCAL_BREAK_RC(rc);
6199 }
6200
6201 if (fWhat & CPUMCTX_EXTRN_RSP)
6202 {
6203 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6204 VMXLOCAL_BREAK_RC(rc);
6205 pCtx->rsp = u64Val;
6206 }
6207
6208 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
6209 {
6210 if (fWhat & CPUMCTX_EXTRN_CS)
6211 {
6212 rc = HMVMX_IMPORT_SREG(CS, &pCtx->cs);
6213 VMXLOCAL_BREAK_RC(rc);
6214 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6215 pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6216 }
6217 if (fWhat & CPUMCTX_EXTRN_SS)
6218 {
6219 rc = HMVMX_IMPORT_SREG(SS, &pCtx->ss);
6220 VMXLOCAL_BREAK_RC(rc);
6221 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6222 pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6223 }
6224 if (fWhat & CPUMCTX_EXTRN_DS)
6225 {
6226 rc = HMVMX_IMPORT_SREG(DS, &pCtx->ds);
6227 VMXLOCAL_BREAK_RC(rc);
6228 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6229 pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6230 }
6231 if (fWhat & CPUMCTX_EXTRN_ES)
6232 {
6233 rc = HMVMX_IMPORT_SREG(ES, &pCtx->es);
6234 VMXLOCAL_BREAK_RC(rc);
6235 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6236 pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6237 }
6238 if (fWhat & CPUMCTX_EXTRN_FS)
6239 {
6240 rc = HMVMX_IMPORT_SREG(FS, &pCtx->fs);
6241 VMXLOCAL_BREAK_RC(rc);
6242 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6243 pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6244 }
6245 if (fWhat & CPUMCTX_EXTRN_GS)
6246 {
6247 rc = HMVMX_IMPORT_SREG(GS, &pCtx->gs);
6248 VMXLOCAL_BREAK_RC(rc);
6249 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6250 pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6251 }
6252 }
6253
6254 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
6255 {
6256 if (fWhat & CPUMCTX_EXTRN_LDTR)
6257 {
6258 rc = HMVMX_IMPORT_SREG(LDTR, &pCtx->ldtr);
6259 VMXLOCAL_BREAK_RC(rc);
6260 }
6261
6262 if (fWhat & CPUMCTX_EXTRN_GDTR)
6263 {
6264 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6265 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
6266 VMXLOCAL_BREAK_RC(rc);
6267 pCtx->gdtr.pGdt = u64Val;
6268 pCtx->gdtr.cbGdt = u32Val;
6269 }
6270
6271 /* Guest IDTR. */
6272 if (fWhat & CPUMCTX_EXTRN_IDTR)
6273 {
6274 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6275 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
6276 VMXLOCAL_BREAK_RC(rc);
6277 pCtx->idtr.pIdt = u64Val;
6278 pCtx->idtr.cbIdt = u32Val;
6279 }
6280
6281 /* Guest TR. */
6282 if (fWhat & CPUMCTX_EXTRN_TR)
6283 {
6284 /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
6285 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6286 {
6287 rc = HMVMX_IMPORT_SREG(TR, &pCtx->tr);
6288 VMXLOCAL_BREAK_RC(rc);
6289 }
6290 }
6291 }
6292
6293 if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
6294 {
6295 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
6296 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
6297 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
6298 pCtx->SysEnter.cs = u32Val;
6299 VMXLOCAL_BREAK_RC(rc);
6300 }
6301
6302#if HC_ARCH_BITS == 64
6303 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
6304 {
6305 if ( pVM->hm.s.fAllow64BitGuests
6306 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6307 pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
6308 }
6309
6310 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
6311 {
6312 if ( pVM->hm.s.fAllow64BitGuests
6313 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6314 {
6315 pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
6316 pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
6317 pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
6318 }
6319 }
6320#endif
6321
6322 if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
6323#if HC_ARCH_BITS == 32
6324 || (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS))
6325#endif
6326 )
6327 {
6328 PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6329 uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
6330 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6331 {
6332 switch (pMsr->u32Msr)
6333 {
6334#if HC_ARCH_BITS == 32
6335 case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
6336 case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
6337 case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
6338 case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6339#endif
6340 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6341 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6342 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit */ break;
6343 default:
6344 {
6345 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6346 ASMSetFlags(fEFlags);
6347 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
6348 cMsrs));
6349 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6350 }
6351 }
6352 }
6353 }
6354
6355 if (fWhat & CPUMCTX_EXTRN_DR7)
6356 {
6357 if (!pVCpu->hm.s.fUsingHyperDR7)
6358 {
6359 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6360 rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
6361 VMXLOCAL_BREAK_RC(rc);
6362 pCtx->dr[7] = u32Val;
6363 }
6364 }
6365
6366 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
6367 {
6368 uint32_t u32Shadow;
6369 /* CR0 required for saving CR3 below, see CPUMIsGuestPagingEnabledEx(). */
6370 if (fWhat & CPUMCTX_EXTRN_CR0)
6371 {
6372 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
6373 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
6374 VMXLOCAL_BREAK_RC(rc);
6375 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32CR0Mask)
6376 | (u32Shadow & pVCpu->hm.s.vmx.u32CR0Mask);
6377 VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
6378 CPUMSetGuestCR0(pVCpu, u32Val);
6379 VMMRZCallRing3Enable(pVCpu);
6380 }
6381
6382 /* CR4 required for saving CR3 below, see CPUMIsGuestInPAEModeEx(). */
6383 if (fWhat & CPUMCTX_EXTRN_CR4)
6384 {
6385 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
6386 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
6387 VMXLOCAL_BREAK_RC(rc);
6388 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32CR4Mask)
6389 | (u32Shadow & pVCpu->hm.s.vmx.u32CR4Mask);
6390 CPUMSetGuestCR4(pVCpu, u32Val);
6391 }
6392
6393 if (fWhat & CPUMCTX_EXTRN_CR3)
6394 {
6395 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6396 || ( pVM->hm.s.fNestedPaging
6397 && CPUMIsGuestPagingEnabledEx(pCtx))) /* PG bit changes are always intercepted, so it's up to date. */
6398 {
6399 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6400 if (pCtx->cr3 != u64Val)
6401 {
6402 CPUMSetGuestCR3(pVCpu, u64Val);
6403 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6404 }
6405
6406 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6407 if (CPUMIsGuestInPAEModeEx(pCtx))
6408 {
6409 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6410 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6411 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6412 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6413 VMXLOCAL_BREAK_RC(rc);
6414 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6415 }
6416 }
6417 }
6418 }
6419 } while (0);
6420
6421 if (RT_SUCCESS(rc))
6422 {
6423 /* Update fExtrn. */
6424 pCtx->fExtrn &= ~fWhat;
6425
6426 /* If everything has been imported, clear the HM keeper bit. */
6427 if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
6428 {
6429 pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
6430 Assert(!pCtx->fExtrn);
6431 }
6432 }
6433 }
6434 else
6435 AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
6436
6437 ASMSetFlags(fEFlags);
6438
6439 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
6440
6441 /*
6442 * Honor any pending CR3 updates.
6443 *
6444 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6445 * -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6446 * -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
6447 *
6448 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6449 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6450 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6451 * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
6452 *
6453 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6454 */
6455 if (VMMRZCallRing3IsEnabled(pVCpu))
6456 {
6457 VMMR0LogFlushDisable(pVCpu);
6458
6459 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6460 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6461
6462 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6463 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6464
6465 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6466 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6467
6468 VMMR0LogFlushEnable(pVCpu);
6469 }
6470
6471 return VINF_SUCCESS;
6472#undef VMXLOCAL_BREAK_RC
6473}
6474
6475
6476/**
6477 * Saves the guest state from the VMCS into the guest-CPU context.
6478 *
6479 * @returns VBox status code.
6480 * @param pVCpu The cross context virtual CPU structure.
6481 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6482 */
6483VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
6484{
6485 return hmR0VmxImportGuestState(pVCpu, fWhat);
6486}
6487
6488
6489/**
6490 * Check per-VM and per-VCPU force flag actions that require us to go back to
6491 * ring-3 for one reason or another.
6492 *
6493 * @returns Strict VBox status code (i.e. informational status codes too)
6494 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6495 * ring-3.
6496 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6497 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6498 * interrupts)
6499 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6500 * all EMTs to be in ring-3.
6501 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6502 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6503 * to the EM loop.
6504 *
6505 * @param pVM The cross context VM structure.
6506 * @param pVCpu The cross context virtual CPU structure.
6507 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6508 * out-of-sync. Make sure to update the required fields
6509 * before using them.
6510 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6511 */
6512static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6513{
6514 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6515
6516 /*
6517 * Anything pending? Should be more likely than not if we're doing a good job.
6518 */
6519 if ( !fStepping
6520 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6521 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6522 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6523 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6524 return VINF_SUCCESS;
6525
6526#if 0
6527 /* We need the control registers now, make sure the guest-CPU context is updated. */
6528 int rc3 = hmR0VmxImportGuestStatae(pVCpu, CPUMCTX_EXTRN_CR0);
6529 AssertRCReturn(rc3, rc3);
6530
6531 /** @todo r=ramshankar: VMCPU_FF_HM_UPDATE_CR3 and VMCPU_FF_HM_UPDATE_PAE_PDPES
6532 * are not part of VMCPU_FF_HP_R0_PRE_HM_MASK. Hence, the two if
6533 * statements below won't ever be entered. Consider removing it or
6534 * determine if it is necessary to add these flags to VMCPU_FF_HP_R0_PRE_HM_MASK. */
6535 /* Pending HM CR3 sync. */
6536 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6537 {
6538 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6539 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6540 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6541 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6542 }
6543
6544 /* Pending HM PAE PDPEs. */
6545 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6546 {
6547 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6548 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6549 }
6550#endif
6551
6552 /* Pending PGM C3 sync. */
6553 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6554 {
6555 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
6556 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6557 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6558 if (rcStrict2 != VINF_SUCCESS)
6559 {
6560 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6561 Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6562 return rcStrict2;
6563 }
6564 }
6565
6566 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6567 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6568 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6569 {
6570 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6571 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6572 Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6573 return rc2;
6574 }
6575
6576 /* Pending VM request packets, such as hardware interrupts. */
6577 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6578 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6579 {
6580 Log4Func(("Pending VM request forcing us back to ring-3\n"));
6581 return VINF_EM_PENDING_REQUEST;
6582 }
6583
6584 /* Pending PGM pool flushes. */
6585 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6586 {
6587 Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
6588 return VINF_PGM_POOL_FLUSH_PENDING;
6589 }
6590
6591 /* Pending DMA requests. */
6592 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6593 {
6594 Log4Func(("Pending DMA request forcing us back to ring-3\n"));
6595 return VINF_EM_RAW_TO_R3;
6596 }
6597
6598 return VINF_SUCCESS;
6599}
6600
6601
6602/**
6603 * Converts any TRPM trap into a pending HM event. This is typically used when
6604 * entering from ring-3 (not longjmp returns).
6605 *
6606 * @param pVCpu The cross context virtual CPU structure.
6607 */
6608static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6609{
6610 Assert(TRPMHasTrap(pVCpu));
6611 Assert(!pVCpu->hm.s.Event.fPending);
6612
6613 uint8_t uVector;
6614 TRPMEVENT enmTrpmEvent;
6615 RTGCUINT uErrCode;
6616 RTGCUINTPTR GCPtrFaultAddress;
6617 uint8_t cbInstr;
6618
6619 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6620 AssertRC(rc);
6621
6622 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6623 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6624 if (enmTrpmEvent == TRPM_TRAP)
6625 {
6626 switch (uVector)
6627 {
6628 case X86_XCPT_NMI:
6629 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6630 break;
6631
6632 case X86_XCPT_BP:
6633 case X86_XCPT_OF:
6634 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6635 break;
6636
6637 case X86_XCPT_PF:
6638 case X86_XCPT_DF:
6639 case X86_XCPT_TS:
6640 case X86_XCPT_NP:
6641 case X86_XCPT_SS:
6642 case X86_XCPT_GP:
6643 case X86_XCPT_AC:
6644 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6645 RT_FALL_THRU();
6646 default:
6647 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6648 break;
6649 }
6650 }
6651 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6652 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6653 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6654 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6655 else
6656 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6657
6658 rc = TRPMResetTrap(pVCpu);
6659 AssertRC(rc);
6660 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6661 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6662
6663 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6664}
6665
6666
6667/**
6668 * Converts the pending HM event into a TRPM trap.
6669 *
6670 * @param pVCpu The cross context virtual CPU structure.
6671 */
6672static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6673{
6674 Assert(pVCpu->hm.s.Event.fPending);
6675
6676 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6677 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6678 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6679 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6680
6681 /* If a trap was already pending, we did something wrong! */
6682 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6683
6684 TRPMEVENT enmTrapType;
6685 switch (uVectorType)
6686 {
6687 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6688 enmTrapType = TRPM_HARDWARE_INT;
6689 break;
6690
6691 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6692 enmTrapType = TRPM_SOFTWARE_INT;
6693 break;
6694
6695 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6696 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6697 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6698 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6699 enmTrapType = TRPM_TRAP;
6700 break;
6701
6702 default:
6703 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6704 enmTrapType = TRPM_32BIT_HACK;
6705 break;
6706 }
6707
6708 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6709
6710 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6711 AssertRC(rc);
6712
6713 if (fErrorCodeValid)
6714 TRPMSetErrorCode(pVCpu, uErrorCode);
6715
6716 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6717 && uVector == X86_XCPT_PF)
6718 {
6719 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6720 }
6721 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6722 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6723 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6724 {
6725 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6726 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6727 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6728 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6729 }
6730
6731 /* Clear any pending events from the VMCS. */
6732 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
6733 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
6734
6735 /* We're now done converting the pending event. */
6736 pVCpu->hm.s.Event.fPending = false;
6737}
6738
6739
6740/**
6741 * Does the necessary state syncing before returning to ring-3 for any reason
6742 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6743 *
6744 * @returns VBox status code.
6745 * @param pVCpu The cross context virtual CPU structure.
6746 * @param fImportState Whether to import the guest state from the VMCS back
6747 * to the guest-CPU context.
6748 *
6749 * @remarks No-long-jmp zone!!!
6750 */
6751static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
6752{
6753 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6754 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6755
6756 RTCPUID idCpu = RTMpCpuId();
6757 Log4Func(("HostCpuId=%u\n", idCpu));
6758
6759 /*
6760 * !!! IMPORTANT !!!
6761 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
6762 */
6763
6764 /* Save the guest state if necessary. */
6765 if (fImportState)
6766 {
6767 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
6768 AssertRCReturn(rc, rc);
6769 }
6770
6771 /* Restore host FPU state if necessary. We will resync on next R0 reentry. */
6772 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
6773 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6774
6775 /* Restore host debug registers if necessary. We will resync on next R0 reentry. */
6776#ifdef VBOX_STRICT
6777 if (CPUMIsHyperDebugStateActive(pVCpu))
6778 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6779#endif
6780 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6781 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6782 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6783
6784#if HC_ARCH_BITS == 64
6785 /* Restore host-state bits that VT-x only restores partially. */
6786 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6787 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6788 {
6789 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6790 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6791 }
6792 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6793#endif
6794
6795 /* Restore the lazy host MSRs as we're leaving VT-x context. */
6796 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
6797 {
6798 /* We shouldn't restore the host MSRs without saving the guest MSRs first. */
6799 if (!fImportState)
6800 {
6801 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE
6802 | CPUMCTX_EXTRN_SYSCALL_MSRS);
6803 AssertRCReturn(rc, rc);
6804 }
6805 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6806 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
6807 }
6808 else
6809 pVCpu->hm.s.vmx.fLazyMsrs = 0;
6810
6811 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6812 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6813
6814 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6815 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
6816 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
6817 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6818 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6819 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6820 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6821 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6822 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6823
6824 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6825
6826 /** @todo This partially defeats the purpose of having preemption hooks.
6827 * The problem is, deregistering the hooks should be moved to a place that
6828 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6829 * context.
6830 */
6831 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6832 {
6833 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6834 AssertRCReturn(rc, rc);
6835
6836 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6837 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6838 }
6839 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6840 NOREF(idCpu);
6841
6842 return VINF_SUCCESS;
6843}
6844
6845
6846/**
6847 * Leaves the VT-x session.
6848 *
6849 * @returns VBox status code.
6850 * @param pVCpu The cross context virtual CPU structure.
6851 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6852 * out-of-sync. Make sure to update the required fields
6853 * before using them.
6854 *
6855 * @remarks No-long-jmp zone!!!
6856 */
6857static int hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6858{
6859 HM_DISABLE_PREEMPT();
6860 HMVMX_ASSERT_CPU_SAFE();
6861 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6862 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6863
6864 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6865 and done this from the VMXR0ThreadCtxCallback(). */
6866 if (!pVCpu->hm.s.fLeaveDone)
6867 {
6868 int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
6869 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
6870 pVCpu->hm.s.fLeaveDone = true;
6871 }
6872 Assert(!pMixedCtx->fExtrn); NOREF(pMixedCtx);
6873
6874 /*
6875 * !!! IMPORTANT !!!
6876 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6877 */
6878
6879 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6880 /** @todo Deregistering here means we need to VMCLEAR always
6881 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6882 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
6883 VMMR0ThreadCtxHookDisable(pVCpu);
6884
6885 /* Leave HM context. This takes care of local init (term). */
6886 int rc = HMR0LeaveCpu(pVCpu);
6887
6888 HM_RESTORE_PREEMPT();
6889 return rc;
6890}
6891
6892
6893/**
6894 * Does the necessary state syncing before doing a longjmp to ring-3.
6895 *
6896 * @returns VBox status code.
6897 * @param pVCpu The cross context virtual CPU structure.
6898 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6899 * out-of-sync. Make sure to update the required fields
6900 * before using them.
6901 *
6902 * @remarks No-long-jmp zone!!!
6903 */
6904DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6905{
6906 return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
6907}
6908
6909
6910/**
6911 * Take necessary actions before going back to ring-3.
6912 *
6913 * An action requires us to go back to ring-3. This function does the necessary
6914 * steps before we can safely return to ring-3. This is not the same as longjmps
6915 * to ring-3, this is voluntary and prepares the guest so it may continue
6916 * executing outside HM (recompiler/IEM).
6917 *
6918 * @returns VBox status code.
6919 * @param pVM The cross context VM structure.
6920 * @param pVCpu The cross context virtual CPU structure.
6921 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6922 * out-of-sync. Make sure to update the required fields
6923 * before using them.
6924 * @param rcExit The reason for exiting to ring-3. Can be
6925 * VINF_VMM_UNKNOWN_RING3_CALL.
6926 */
6927static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
6928{
6929 Assert(pVM);
6930 Assert(pVCpu);
6931 Assert(pMixedCtx);
6932 HMVMX_ASSERT_PREEMPT_SAFE();
6933
6934 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6935 {
6936 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6937 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6938 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6939 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6940 }
6941
6942 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6943 VMMRZCallRing3Disable(pVCpu);
6944 Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
6945
6946 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6947 if (pVCpu->hm.s.Event.fPending)
6948 {
6949 hmR0VmxPendingEventToTrpmTrap(pVCpu);
6950 Assert(!pVCpu->hm.s.Event.fPending);
6951 }
6952
6953 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
6954 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
6955
6956 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
6957 and if we're injecting an event we should have a TRPM trap pending. */
6958 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6959#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
6960 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6961#endif
6962
6963 /* Save guest state and restore host state bits. */
6964 int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
6965 AssertRCReturn(rc, rc);
6966 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6967 /* Thread-context hooks are unregistered at this point!!! */
6968
6969 /* Sync recompiler state. */
6970 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6971 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6972 | CPUM_CHANGED_LDTR
6973 | CPUM_CHANGED_GDTR
6974 | CPUM_CHANGED_IDTR
6975 | CPUM_CHANGED_TR
6976 | CPUM_CHANGED_HIDDEN_SEL_REGS);
6977 if ( pVM->hm.s.fNestedPaging
6978 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
6979 {
6980 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6981 }
6982
6983 Assert(!pVCpu->hm.s.fClearTrapFlag);
6984
6985 /* Update the exit-to-ring 3 reason. */
6986 pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
6987
6988 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6989 if (rcExit != VINF_EM_RAW_INTERRUPT)
6990 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
6991
6992 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6993
6994 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6995 VMMRZCallRing3RemoveNotification(pVCpu);
6996 VMMRZCallRing3Enable(pVCpu);
6997
6998 return rc;
6999}
7000
7001
7002/**
7003 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7004 * longjump to ring-3 and possibly get preempted.
7005 *
7006 * @returns VBox status code.
7007 * @param pVCpu The cross context virtual CPU structure.
7008 * @param enmOperation The operation causing the ring-3 longjump.
7009 * @param pvUser Opaque pointer to the guest-CPU context. The data
7010 * may be out-of-sync. Make sure to update the required
7011 * fields before using them.
7012 */
7013static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7014{
7015 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7016 {
7017 /*
7018 * !!! IMPORTANT !!!
7019 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7020 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7021 */
7022 VMMRZCallRing3RemoveNotification(pVCpu);
7023 VMMRZCallRing3Disable(pVCpu);
7024 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7025 RTThreadPreemptDisable(&PreemptState);
7026
7027 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7028 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7029
7030#if HC_ARCH_BITS == 64
7031 /* Restore host-state bits that VT-x only restores partially. */
7032 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7033 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7034 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7035 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7036#endif
7037
7038 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7039 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7040 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7041
7042 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7043 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7044 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7045 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7046 {
7047 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7048 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7049 }
7050
7051 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7052 VMMR0ThreadCtxHookDisable(pVCpu);
7053 HMR0LeaveCpu(pVCpu);
7054 RTThreadPreemptRestore(&PreemptState);
7055 return VINF_SUCCESS;
7056 }
7057
7058 Assert(pVCpu);
7059 Assert(pvUser);
7060 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7061 HMVMX_ASSERT_PREEMPT_SAFE();
7062
7063 VMMRZCallRing3Disable(pVCpu);
7064 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7065
7066 Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
7067
7068 int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7069 AssertRCReturn(rc, rc);
7070
7071 VMMRZCallRing3Enable(pVCpu);
7072 return VINF_SUCCESS;
7073}
7074
7075
7076/**
7077 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7078 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7079 *
7080 * @param pVCpu The cross context virtual CPU structure.
7081 */
7082DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7083{
7084 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7085 {
7086 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7087 {
7088 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7089 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7090 AssertRC(rc);
7091 Log4Func(("Setup interrupt-window exiting\n"));
7092 }
7093 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7094}
7095
7096
7097/**
7098 * Clears the interrupt-window exiting control in the VMCS.
7099 *
7100 * @param pVCpu The cross context virtual CPU structure.
7101 */
7102DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7103{
7104 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7105 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7106 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7107 AssertRC(rc);
7108 Log4Func(("Cleared interrupt-window exiting\n"));
7109}
7110
7111
7112/**
7113 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7114 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7115 *
7116 * @param pVCpu The cross context virtual CPU structure.
7117 */
7118DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7119{
7120 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7121 {
7122 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7123 {
7124 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7125 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7126 AssertRC(rc);
7127 Log4Func(("Setup NMI-window exiting\n"));
7128 }
7129 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7130}
7131
7132
7133/**
7134 * Clears the NMI-window exiting control in the VMCS.
7135 *
7136 * @param pVCpu The cross context virtual CPU structure.
7137 */
7138DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7139{
7140 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7141 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7142 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7143 AssertRC(rc);
7144 Log4Func(("Cleared NMI-window exiting\n"));
7145}
7146
7147
7148/**
7149 * Evaluates the event to be delivered to the guest and sets it as the pending
7150 * event.
7151 *
7152 * @returns The VT-x guest-interruptibility state.
7153 * @param pVCpu The cross context virtual CPU structure.
7154 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7155 * out-of-sync. Make sure to update the required fields
7156 * before using them.
7157 */
7158static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7159{
7160 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7161 uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7162 bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7163 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7164 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7165
7166 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7167 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7168 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7169 Assert(!TRPMHasTrap(pVCpu));
7170
7171 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7172 APICUpdatePendingInterrupts(pVCpu);
7173
7174 /*
7175 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7176 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7177 */
7178 /** @todo SMI. SMIs take priority over NMIs. */
7179 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7180 {
7181 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7182 if ( !pVCpu->hm.s.Event.fPending
7183 && !fBlockNmi
7184 && !fBlockSti
7185 && !fBlockMovSS)
7186 {
7187 Log4Func(("Pending NMI\n"));
7188 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7189 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7190
7191 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7192 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7193 }
7194 else
7195 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7196 }
7197 /*
7198 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7199 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7200 */
7201 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7202 && !pVCpu->hm.s.fSingleInstruction)
7203 {
7204 Assert(!DBGFIsStepping(pVCpu));
7205 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7206 AssertRCReturn(rc, 0);
7207 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7208 if ( !pVCpu->hm.s.Event.fPending
7209 && !fBlockInt
7210 && !fBlockSti
7211 && !fBlockMovSS)
7212 {
7213 uint8_t u8Interrupt;
7214 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7215 if (RT_SUCCESS(rc))
7216 {
7217 Log4Func(("Pending external interrupt u8Interrupt=%#x\n", u8Interrupt));
7218 uint32_t u32IntInfo = u8Interrupt
7219 | VMX_EXIT_INTERRUPTION_INFO_VALID
7220 | (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7221
7222 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7223 }
7224 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7225 {
7226 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7227 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7228 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7229
7230 /*
7231 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7232 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7233 * need to re-set this force-flag here.
7234 */
7235 }
7236 else
7237 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7238 }
7239 else
7240 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7241 }
7242
7243 return fIntrState;
7244}
7245
7246
7247/**
7248 * Sets a pending-debug exception to be delivered to the guest if the guest is
7249 * single-stepping in the VMCS.
7250 *
7251 * @param pVCpu The cross context virtual CPU structure.
7252 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7253 * out-of-sync. Make sure to update the required fields
7254 * before using them.
7255 */
7256DECLINLINE(int) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7257{
7258 RT_NOREF(pVCpu);
7259 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS)); NOREF(pMixedCtx);
7260 return VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7261}
7262
7263
7264/**
7265 * Injects any pending events into the guest if the guest is in a state to
7266 * receive them.
7267 *
7268 * @returns Strict VBox status code (i.e. informational status codes too).
7269 * @param pVCpu The cross context virtual CPU structure.
7270 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7271 * out-of-sync. Make sure to update the required fields
7272 * before using them.
7273 * @param fIntrState The VT-x guest-interruptibility state.
7274 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7275 * return VINF_EM_DBG_STEPPED if the event was
7276 * dispatched directly.
7277 */
7278static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t fIntrState, bool fStepping)
7279{
7280 HMVMX_ASSERT_PREEMPT_SAFE();
7281 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7282
7283 bool fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7284 bool fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7285
7286 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7287 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7288 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7289 Assert(!TRPMHasTrap(pVCpu));
7290
7291 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7292 if (pVCpu->hm.s.Event.fPending)
7293 {
7294 /*
7295 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7296 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7297 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7298 *
7299 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7300 */
7301 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7302#ifdef VBOX_STRICT
7303 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7304 {
7305 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7306 Assert(!fBlockInt);
7307 Assert(!fBlockSti);
7308 Assert(!fBlockMovSS);
7309 }
7310 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7311 {
7312 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7313 Assert(!fBlockSti);
7314 Assert(!fBlockMovSS);
7315 Assert(!fBlockNmi);
7316 }
7317#endif
7318 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7319 (uint8_t)uIntType));
7320 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7321 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping,
7322 &fIntrState);
7323 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7324
7325 /* Update the interruptibility-state as it could have been changed by
7326 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7327 fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7328 fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7329
7330 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7331 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7332 else
7333 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7334 }
7335
7336 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7337 if ( fBlockSti
7338 || fBlockMovSS)
7339 {
7340 if (!pVCpu->hm.s.fSingleInstruction)
7341 {
7342 /*
7343 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7344 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7345 * See Intel spec. 27.3.4 "Saving Non-Register State".
7346 */
7347 Assert(!DBGFIsStepping(pVCpu));
7348 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7349 AssertRCReturn(rc, rc);
7350 if (pMixedCtx->eflags.Bits.u1TF)
7351 {
7352 int rc2 = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
7353 AssertRCReturn(rc2, rc2);
7354 }
7355 }
7356 else if (pMixedCtx->eflags.Bits.u1TF)
7357 {
7358 /*
7359 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7360 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7361 */
7362 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7363 fIntrState = 0;
7364 }
7365 }
7366
7367 /*
7368 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7369 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7370 */
7371 int rc3 = hmR0VmxExportGuestIntrState(pVCpu, fIntrState);
7372 AssertRCReturn(rc3, rc3);
7373
7374 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7375 NOREF(fBlockMovSS); NOREF(fBlockSti);
7376 return rcStrict;
7377}
7378
7379
7380/**
7381 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7382 *
7383 * @param pVCpu The cross context virtual CPU structure.
7384 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7385 * out-of-sync. Make sure to update the required fields
7386 * before using them.
7387 */
7388DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7389{
7390 NOREF(pMixedCtx);
7391 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7392 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7393}
7394
7395
7396/**
7397 * Injects a double-fault (\#DF) exception into the VM.
7398 *
7399 * @returns Strict VBox status code (i.e. informational status codes too).
7400 * @param pVCpu The cross context virtual CPU structure.
7401 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7402 * out-of-sync. Make sure to update the required fields
7403 * before using them.
7404 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7405 * and should return VINF_EM_DBG_STEPPED if the event
7406 * is injected directly (register modified by us, not
7407 * by hardware on VM-entry).
7408 * @param pfIntrState Pointer to the current guest interruptibility-state.
7409 * This interruptibility-state will be updated if
7410 * necessary. This cannot not be NULL.
7411 */
7412DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCCPUMCTX pMixedCtx, bool fStepping, uint32_t *pfIntrState)
7413{
7414 NOREF(pMixedCtx);
7415 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7416 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7417 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7418 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */, fStepping,
7419 pfIntrState);
7420}
7421
7422
7423/**
7424 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7425 *
7426 * @param pVCpu The cross context virtual CPU structure.
7427 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7428 * out-of-sync. Make sure to update the required fields
7429 * before using them.
7430 */
7431DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7432{
7433 NOREF(pMixedCtx);
7434 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7435 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7436 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7437}
7438
7439
7440/**
7441 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7442 *
7443 * @param pVCpu The cross context virtual CPU structure.
7444 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7445 * out-of-sync. Make sure to update the required fields
7446 * before using them.
7447 * @param cbInstr The value of RIP that is to be pushed on the guest
7448 * stack.
7449 */
7450DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7451{
7452 NOREF(pMixedCtx);
7453 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7454 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7455 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7456}
7457
7458
7459/**
7460 * Injects a general-protection (\#GP) fault into the VM.
7461 *
7462 * @returns Strict VBox status code (i.e. informational status codes too).
7463 * @param pVCpu The cross context virtual CPU structure.
7464 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7465 * out-of-sync. Make sure to update the required fields
7466 * before using them.
7467 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7468 * mode, i.e. in real-mode it's not valid).
7469 * @param u32ErrorCode The error code associated with the \#GP.
7470 * @param fStepping Whether we're running in
7471 * hmR0VmxRunGuestCodeStep() and should return
7472 * VINF_EM_DBG_STEPPED if the event is injected
7473 * directly (register modified by us, not by
7474 * hardware on VM-entry).
7475 * @param pfIntrState Pointer to the current guest interruptibility-state.
7476 * This interruptibility-state will be updated if
7477 * necessary. This cannot not be NULL.
7478 */
7479DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7480 bool fStepping, uint32_t *pfIntrState)
7481{
7482 NOREF(pMixedCtx);
7483 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7484 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7485 if (fErrorCodeValid)
7486 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7487 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */, fStepping,
7488 pfIntrState);
7489}
7490
7491
7492#if 0 /* unused */
7493/**
7494 * Sets a general-protection (\#GP) exception as pending-for-injection into the
7495 * VM.
7496 *
7497 * @param pVCpu The cross context virtual CPU structure.
7498 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7499 * out-of-sync. Make sure to update the required fields
7500 * before using them.
7501 * @param u32ErrorCode The error code associated with the \#GP.
7502 */
7503DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7504{
7505 NOREF(pMixedCtx);
7506 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7507 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7508 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7509 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7510}
7511#endif /* unused */
7512
7513
7514/**
7515 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7516 *
7517 * @param pVCpu The cross context virtual CPU structure.
7518 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7519 * out-of-sync. Make sure to update the required fields
7520 * before using them.
7521 * @param uVector The software interrupt vector number.
7522 * @param cbInstr The value of RIP that is to be pushed on the guest
7523 * stack.
7524 */
7525DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7526{
7527 NOREF(pMixedCtx);
7528 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7529 if ( uVector == X86_XCPT_BP
7530 || uVector == X86_XCPT_OF)
7531 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7532 else
7533 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7534 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7535}
7536
7537
7538/**
7539 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7540 * stack.
7541 *
7542 * @returns Strict VBox status code (i.e. informational status codes too).
7543 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7544 * @param pVM The cross context VM structure.
7545 * @param pMixedCtx Pointer to the guest-CPU context.
7546 * @param uValue The value to push to the guest stack.
7547 */
7548DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7549{
7550 /*
7551 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7552 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7553 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7554 */
7555 if (pMixedCtx->sp == 1)
7556 return VINF_EM_RESET;
7557 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7558 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7559 AssertRC(rc);
7560 return rc;
7561}
7562
7563
7564/**
7565 * Injects an event into the guest upon VM-entry by updating the relevant fields
7566 * in the VM-entry area in the VMCS.
7567 *
7568 * @returns Strict VBox status code (i.e. informational status codes too).
7569 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7570 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7571 *
7572 * @param pVCpu The cross context virtual CPU structure.
7573 * @param u64IntInfo The VM-entry interruption-information field.
7574 * @param cbInstr The VM-entry instruction length in bytes (for
7575 * software interrupts, exceptions and privileged
7576 * software exceptions).
7577 * @param u32ErrCode The VM-entry exception error code.
7578 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7579 * @param pfIntrState Pointer to the current guest interruptibility-state.
7580 * This interruptibility-state will be updated if
7581 * necessary. This cannot not be NULL.
7582 * @param fStepping Whether we're running in
7583 * hmR0VmxRunGuestCodeStep() and should return
7584 * VINF_EM_DBG_STEPPED if the event is injected
7585 * directly (register modified by us, not by
7586 * hardware on VM-entry).
7587 *
7588 * @remarks Requires CR0!
7589 */
7590static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7591 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState)
7592{
7593 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7594 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7595 Assert(pfIntrState);
7596
7597 PCPUMCTX pMixedCtx = &pVCpu->cpum.GstCtx;
7598 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7599 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7600 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7601
7602#ifdef VBOX_STRICT
7603 /*
7604 * Validate the error-code-valid bit for hardware exceptions.
7605 * No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7606 */
7607 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7608 && !CPUMIsGuestInRealModeEx(pMixedCtx))
7609 {
7610 switch (uVector)
7611 {
7612 case X86_XCPT_PF:
7613 case X86_XCPT_DF:
7614 case X86_XCPT_TS:
7615 case X86_XCPT_NP:
7616 case X86_XCPT_SS:
7617 case X86_XCPT_GP:
7618 case X86_XCPT_AC:
7619 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7620 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7621 RT_FALL_THRU();
7622 default:
7623 break;
7624 }
7625 }
7626#endif
7627
7628 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7629 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7630 || !(*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7631
7632 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7633
7634 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
7635 {
7636 /*
7637 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7638 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7639 */
7640 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7641 }
7642 else
7643 {
7644 /* We require CR0 to check if the guest is in real-mode. */
7645 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
7646 AssertRCReturn(rc, rc);
7647
7648 /*
7649 * Hardware interrupts & exceptions cannot be delivered through the software interrupt
7650 * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
7651 * interrupt handler in the (real-mode) guest.
7652 *
7653 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
7654 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7655 */
7656 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7657 {
7658 PVM pVM = pVCpu->CTX_SUFF(pVM);
7659 Assert(PDMVmmDevHeapIsEnabled(pVM));
7660 Assert(pVM->hm.s.vmx.pRealModeTSS);
7661
7662 /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
7663 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK
7664 | CPUMCTX_EXTRN_TABLE_MASK
7665 | CPUMCTX_EXTRN_RIP
7666 | CPUMCTX_EXTRN_RSP
7667 | CPUMCTX_EXTRN_RFLAGS);
7668 AssertRCReturn(rc, rc);
7669
7670 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7671 size_t const cbIdtEntry = sizeof(X86IDTR16);
7672 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7673 {
7674 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7675 if (uVector == X86_XCPT_DF)
7676 return VINF_EM_RESET;
7677
7678 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7679 if (uVector == X86_XCPT_GP)
7680 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, pfIntrState);
7681
7682 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7683 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7684 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
7685 fStepping, pfIntrState);
7686 }
7687
7688 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7689 uint16_t uGuestIp = pMixedCtx->ip;
7690 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7691 {
7692 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7693 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7694 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7695 }
7696 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7697 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7698
7699 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7700 X86IDTR16 IdtEntry;
7701 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7702 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7703 AssertRCReturn(rc, rc);
7704
7705 /* Construct the stack frame for the interrupt/exception handler. */
7706 VBOXSTRICTRC rcStrict;
7707 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7708 if (rcStrict == VINF_SUCCESS)
7709 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7710 if (rcStrict == VINF_SUCCESS)
7711 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7712
7713 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7714 if (rcStrict == VINF_SUCCESS)
7715 {
7716 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7717 pMixedCtx->rip = IdtEntry.offSel;
7718 pMixedCtx->cs.Sel = IdtEntry.uSel;
7719 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7720 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7721 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7722 && uVector == X86_XCPT_PF)
7723 pMixedCtx->cr2 = GCPtrFaultAddress;
7724
7725 /* If any other guest-state bits are changed here, make sure to update
7726 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7727 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS
7728 | HM_CHANGED_GUEST_CR2
7729 | HM_CHANGED_GUEST_RIP
7730 | HM_CHANGED_GUEST_RFLAGS
7731 | HM_CHANGED_GUEST_RSP);
7732
7733 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7734 if (*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7735 {
7736 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7737 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7738 Log4Func(("Clearing inhibition due to STI\n"));
7739 *pfIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7740 }
7741 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7742 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
7743
7744 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7745 it, if we are returning to ring-3 before executing guest code. */
7746 pVCpu->hm.s.Event.fPending = false;
7747
7748 /* Make hmR0VmxPreRunGuest() return if we're stepping since we've changed cs:rip. */
7749 if (fStepping)
7750 rcStrict = VINF_EM_DBG_STEPPED;
7751 }
7752 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7753 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7754 return rcStrict;
7755 }
7756 }
7757
7758 /* Validate. */
7759 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7760 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7761 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7762
7763 /* Inject. */
7764 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7765 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7766 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7767 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7768 AssertRCReturn(rc, rc);
7769
7770 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7771 && uVector == X86_XCPT_PF)
7772 pMixedCtx->cr2 = GCPtrFaultAddress;
7773
7774 Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr,
7775 pMixedCtx->cr2));
7776
7777 return VINF_SUCCESS;
7778}
7779
7780
7781/**
7782 * Clears the interrupt-window exiting control in the VMCS and if necessary
7783 * clears the current event in the VMCS as well.
7784 *
7785 * @returns VBox status code.
7786 * @param pVCpu The cross context virtual CPU structure.
7787 *
7788 * @remarks Use this function only to clear events that have not yet been
7789 * delivered to the guest but are injected in the VMCS!
7790 * @remarks No-long-jump zone!!!
7791 */
7792static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
7793{
7794 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7795 {
7796 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7797 Log4Func(("Cleared interrupt widow\n"));
7798 }
7799
7800 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
7801 {
7802 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
7803 Log4Func(("Cleared interrupt widow\n"));
7804 }
7805}
7806
7807
7808/**
7809 * Enters the VT-x session.
7810 *
7811 * @returns VBox status code.
7812 * @param pVM The cross context VM structure.
7813 * @param pVCpu The cross context virtual CPU structure.
7814 * @param pCpu Pointer to the CPU info struct.
7815 */
7816VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7817{
7818 AssertPtr(pVM);
7819 AssertPtr(pVCpu);
7820 Assert(pVM->hm.s.vmx.fSupported);
7821 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7822 NOREF(pCpu); NOREF(pVM);
7823
7824 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7825 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7826 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7827
7828#ifdef VBOX_STRICT
7829 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
7830 RTCCUINTREG uHostCR4 = ASMGetCR4();
7831 if (!(uHostCR4 & X86_CR4_VMXE))
7832 {
7833 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7834 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7835 }
7836#endif
7837
7838 /*
7839 * Load the VCPU's VMCS as the current (and active) one.
7840 */
7841 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7842 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7843 if (RT_FAILURE(rc))
7844 return rc;
7845
7846 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7847 pVCpu->hm.s.fLeaveDone = false;
7848 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7849
7850 return VINF_SUCCESS;
7851}
7852
7853
7854/**
7855 * The thread-context callback (only on platforms which support it).
7856 *
7857 * @param enmEvent The thread-context event.
7858 * @param pVCpu The cross context virtual CPU structure.
7859 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7860 * @thread EMT(pVCpu)
7861 */
7862VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7863{
7864 NOREF(fGlobalInit);
7865
7866 switch (enmEvent)
7867 {
7868 case RTTHREADCTXEVENT_OUT:
7869 {
7870 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7871 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7872 VMCPU_ASSERT_EMT(pVCpu);
7873
7874 /* No longjmps (logger flushes, locks) in this fragile context. */
7875 VMMRZCallRing3Disable(pVCpu);
7876 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7877
7878 /*
7879 * Restore host-state (FPU, debug etc.)
7880 */
7881 if (!pVCpu->hm.s.fLeaveDone)
7882 {
7883 /*
7884 * Do -not- import the guest-state here as we might already be in the middle of importing
7885 * it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
7886 */
7887 hmR0VmxLeave(pVCpu, false /* fImportState */);
7888 pVCpu->hm.s.fLeaveDone = true;
7889 }
7890
7891 /* Leave HM context, takes care of local init (term). */
7892 int rc = HMR0LeaveCpu(pVCpu);
7893 AssertRC(rc); NOREF(rc);
7894
7895 /* Restore longjmp state. */
7896 VMMRZCallRing3Enable(pVCpu);
7897 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
7898 break;
7899 }
7900
7901 case RTTHREADCTXEVENT_IN:
7902 {
7903 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7904 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7905 VMCPU_ASSERT_EMT(pVCpu);
7906
7907 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7908 VMMRZCallRing3Disable(pVCpu);
7909 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7910
7911 /* Initialize the bare minimum state required for HM. This takes care of
7912 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7913 int rc = HMR0EnterCpu(pVCpu);
7914 AssertRC(rc);
7915 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7916 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7917
7918 /* Load the active VMCS as the current one. */
7919 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7920 {
7921 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7922 AssertRC(rc); NOREF(rc);
7923 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7924 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7925 }
7926 pVCpu->hm.s.fLeaveDone = false;
7927
7928 /* Restore longjmp state. */
7929 VMMRZCallRing3Enable(pVCpu);
7930 break;
7931 }
7932
7933 default:
7934 break;
7935 }
7936}
7937
7938
7939/**
7940 * Exports the host state into the VMCS host-state area.
7941 * Sets up the VM-exit MSR-load area.
7942 *
7943 * The CPU state will be loaded from these fields on every successful VM-exit.
7944 *
7945 * @returns VBox status code.
7946 * @param pVCpu The cross context virtual CPU structure.
7947 *
7948 * @remarks No-long-jump zone!!!
7949 */
7950static int hmR0VmxExportHostState(PVMCPU pVCpu)
7951{
7952 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7953
7954 int rc = VINF_SUCCESS;
7955 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
7956 {
7957 rc = hmR0VmxExportHostControlRegs();
7958 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7959
7960 rc = hmR0VmxExportHostSegmentRegs(pVCpu);
7961 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7962
7963 rc = hmR0VmxExportHostMsrs(pVCpu);
7964 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7965
7966 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
7967 }
7968 return rc;
7969}
7970
7971
7972/**
7973 * Saves the host state in the VMCS host-state.
7974 *
7975 * @returns VBox status code.
7976 * @param pVCpu The cross context virtual CPU structure.
7977 *
7978 * @remarks No-long-jump zone!!!
7979 */
7980VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
7981{
7982 AssertPtr(pVCpu);
7983 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7984
7985 /*
7986 * Export the host state here while entering HM context.
7987 * When thread-context hooks are used, we might get preempted and have to re-save the host
7988 * state but most of the time we won't be, so do it here before we disable interrupts.
7989 */
7990 return hmR0VmxExportHostState(pVCpu);
7991}
7992
7993
7994/**
7995 * Exports the guest state into the VMCS guest-state area.
7996 *
7997 * The will typically be done before VM-entry when the guest-CPU state and the
7998 * VMCS state may potentially be out of sync.
7999 *
8000 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8001 * VM-entry controls.
8002 * Sets up the appropriate VMX non-root function to execute guest code based on
8003 * the guest CPU mode.
8004 *
8005 * @returns VBox strict status code.
8006 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8007 * without unrestricted guest access and the VMMDev is not presently
8008 * mapped (e.g. EFI32).
8009 *
8010 * @param pVM The cross context VM structure.
8011 * @param pVCpu The cross context virtual CPU structure.
8012 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8013 * out-of-sync. Make sure to update the required fields
8014 * before using them.
8015 *
8016 * @remarks No-long-jump zone!!!
8017 */
8018static VBOXSTRICTRC hmR0VmxExportGuestState(PVM pVM, PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
8019{
8020 AssertPtr(pVM);
8021 AssertPtr(pVCpu);
8022 AssertPtr(pMixedCtx);
8023 HMVMX_ASSERT_PREEMPT_SAFE();
8024
8025 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8026
8027 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
8028
8029 /* Determine real-on-v86 mode. */
8030 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8031 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8032 && CPUMIsGuestInRealModeEx(pMixedCtx))
8033 {
8034 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8035 }
8036
8037 /*
8038 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8039 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8040 */
8041 int rc = hmR0VmxSelectVMRunHandler(pVCpu, pMixedCtx);
8042 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8043
8044 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8045 rc = hmR0VmxExportGuestEntryCtls(pVCpu, pMixedCtx);
8046 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8047
8048 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8049 rc = hmR0VmxExportGuestExitCtls(pVCpu, pMixedCtx);
8050 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8051
8052 rc = hmR0VmxExportGuestCR0(pVCpu, pMixedCtx);
8053 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8054
8055 VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pMixedCtx);
8056 if (rcStrict == VINF_SUCCESS)
8057 { /* likely */ }
8058 else
8059 {
8060 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8061 return rcStrict;
8062 }
8063
8064 rc = hmR0VmxExportGuestSegmentRegs(pVCpu, pMixedCtx);
8065 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8066
8067 /* This needs to be done after hmR0VmxExportGuestEntryCtls() and hmR0VmxExportGuestExitCtls() as it
8068 may alter controls if we determine we don't have to swap EFER after all. */
8069 rc = hmR0VmxExportGuestMsrs(pVCpu, pMixedCtx);
8070 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8071
8072 rc = hmR0VmxExportGuestApicTpr(pVCpu);
8073 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8074
8075 /* This needs to be done after hmR0VmxExportGuestCR0() as it may alter intercepted exceptions. */
8076 rc = hmR0VmxExportGuestXcptIntercepts(pVCpu);
8077 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8078
8079 /* Exporting RFLAGS here is fine, even though RFLAGS.TF might depend on guest debug state which is
8080 not exported here. It is re-evaluated and updated if necessary in hmR0VmxExportSharedState(). */
8081 rc = hmR0VmxExportGuestRip(pVCpu, pMixedCtx);
8082 rc |= hmR0VmxExportGuestRsp(pVCpu, pMixedCtx);
8083 rc |= hmR0VmxExportGuestRflags(pVCpu, pMixedCtx);
8084 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8085
8086 /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
8087 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
8088 | HM_CHANGED_GUEST_CR2
8089 | (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
8090 | HM_CHANGED_GUEST_X87
8091 | HM_CHANGED_GUEST_SSE_AVX
8092 | HM_CHANGED_GUEST_OTHER_XSAVE
8093 | HM_CHANGED_GUEST_XCRx
8094 | HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
8095 | HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
8096 | HM_CHANGED_GUEST_TSC_AUX
8097 | HM_CHANGED_GUEST_OTHER_MSRS
8098 | HM_CHANGED_GUEST_HWVIRT
8099 | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
8100
8101 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
8102 return rc;
8103}
8104
8105
8106/**
8107 * Exports the state shared between the host and guest into the VMCS.
8108 *
8109 * @param pVM The cross context VM structure.
8110 * @param pVCpu The cross context virtual CPU structure.
8111 * @param pCtx Pointer to the guest-CPU context.
8112 *
8113 * @remarks No-long-jump zone!!!
8114 */
8115static void hmR0VmxExportSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8116{
8117 NOREF(pVM);
8118
8119 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8120 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8121
8122 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
8123 {
8124 int rc = hmR0VmxExportSharedDebugState(pVCpu, pCtx);
8125 AssertRC(rc);
8126 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
8127
8128 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8129 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
8130 {
8131 rc = hmR0VmxExportGuestRflags(pVCpu, pCtx);
8132 AssertRC(rc);
8133 }
8134 }
8135
8136 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
8137 {
8138 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8139 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
8140 }
8141
8142 AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
8143 ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8144}
8145
8146
8147/**
8148 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8149 *
8150 * @returns Strict VBox status code (i.e. informational status codes too).
8151 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8152 * without unrestricted guest access and the VMMDev is not presently
8153 * mapped (e.g. EFI32).
8154 *
8155 * @param pVM The cross context VM structure.
8156 * @param pVCpu The cross context virtual CPU structure.
8157 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8158 * out-of-sync. Make sure to update the required fields
8159 * before using them.
8160 *
8161 * @remarks No-long-jump zone!!!
8162 */
8163static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
8164{
8165 HMVMX_ASSERT_PREEMPT_SAFE();
8166 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8167 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8168
8169#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8170 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8171#endif
8172
8173 /*
8174 * For many exits it's only RIP that changes and hence try to export it first
8175 * without going through a lot of change flag checks.
8176 */
8177 VBOXSTRICTRC rcStrict;
8178 uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8179 if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
8180 {
8181 rcStrict = hmR0VmxExportGuestRip(pVCpu, pMixedCtx);
8182 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8183 { /* likely */}
8184 else
8185 AssertMsgFailedReturn(("hmR0VmxExportGuestRip failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8186 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
8187 }
8188 else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8189 {
8190 rcStrict = hmR0VmxExportGuestState(pVM, pVCpu, pMixedCtx);
8191 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8192 { /* likely */}
8193 else
8194 {
8195 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("hmR0VmxExportGuestState failed! rc=%Rrc\n",
8196 VBOXSTRICTRC_VAL(rcStrict)));
8197 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8198 return rcStrict;
8199 }
8200 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
8201 }
8202 else
8203 rcStrict = VINF_SUCCESS;
8204
8205#ifdef VBOX_STRICT
8206 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8207 fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8208 AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
8209 ("fCtxChanged=%#RX64\n", fCtxChanged));
8210#endif
8211 return rcStrict;
8212}
8213
8214
8215/**
8216 * Does the preparations before executing guest code in VT-x.
8217 *
8218 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8219 * recompiler/IEM. We must be cautious what we do here regarding committing
8220 * guest-state information into the VMCS assuming we assuredly execute the
8221 * guest in VT-x mode.
8222 *
8223 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8224 * the common-state (TRPM/forceflags), we must undo those changes so that the
8225 * recompiler/IEM can (and should) use them when it resumes guest execution.
8226 * Otherwise such operations must be done when we can no longer exit to ring-3.
8227 *
8228 * @returns Strict VBox status code (i.e. informational status codes too).
8229 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8230 * have been disabled.
8231 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8232 * double-fault into the guest.
8233 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8234 * dispatched directly.
8235 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8236 *
8237 * @param pVM The cross context VM structure.
8238 * @param pVCpu The cross context virtual CPU structure.
8239 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8240 * out-of-sync. Make sure to update the required fields
8241 * before using them.
8242 * @param pVmxTransient Pointer to the VMX transient structure.
8243 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8244 * us ignore some of the reasons for returning to
8245 * ring-3, and return VINF_EM_DBG_STEPPED if event
8246 * dispatching took place.
8247 */
8248static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8249{
8250 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8251
8252#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8253 PGMRZDynMapFlushAutoSet(pVCpu);
8254#endif
8255
8256 /* Check force flag actions that might require us to go back to ring-3. */
8257 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8258 if (rcStrict == VINF_SUCCESS)
8259 { /* FFs doesn't get set all the time. */ }
8260 else
8261 return rcStrict;
8262
8263 /*
8264 * Setup the virtualized-APIC accesses.
8265 *
8266 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8267 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8268 *
8269 * This is the reason we do it here and not in hmR0VmxExportGuestState().
8270 */
8271 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8272 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8273 && PDMHasApic(pVM))
8274 {
8275 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8276 Assert(u64MsrApicBase);
8277 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8278
8279 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8280
8281 /* Unalias any existing mapping. */
8282 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8283 AssertRCReturn(rc, rc);
8284
8285 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8286 Log4Func(("Mapped HC APIC-access page at %#RGp\n", GCPhysApicBase));
8287 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8288 AssertRCReturn(rc, rc);
8289
8290 /* Update the per-VCPU cache of the APIC base MSR. */
8291 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8292 }
8293
8294 if (TRPMHasTrap(pVCpu))
8295 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8296 uint32_t fIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8297
8298 /*
8299 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8300 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8301 */
8302 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, fIntrState, fStepping);
8303 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8304 { /* likely */ }
8305 else
8306 {
8307 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8308 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8309 return rcStrict;
8310 }
8311
8312 /*
8313 * No longjmps to ring-3 from this point on!!!
8314 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8315 * This also disables flushing of the R0-logger instance (if any).
8316 */
8317 VMMRZCallRing3Disable(pVCpu);
8318
8319 /*
8320 * Export the guest state bits.
8321 *
8322 * We cannot perform longjmps while loading the guest state because we do not preserve the
8323 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8324 * CPU migration.
8325 *
8326 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8327 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8328 * Hence, loading of the guest state needs to be done -after- injection of events.
8329 */
8330 rcStrict = hmR0VmxExportGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8331 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8332 { /* likely */ }
8333 else
8334 {
8335 VMMRZCallRing3Enable(pVCpu);
8336 return rcStrict;
8337 }
8338
8339 /*
8340 * We disable interrupts so that we don't miss any interrupts that would flag
8341 * preemption (IPI/timers etc.) when thread-context hooks aren't used and we've
8342 * been running with preemption disabled for a while. Since this is purly to aid
8343 * the RTThreadPreemptIsPending code, it doesn't matter that it may temporarily
8344 * reenable and disable interrupt on NT.
8345 *
8346 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8347 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8348 *
8349 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8350 * executing guest code.
8351 */
8352 pVmxTransient->fEFlags = ASMIntDisableFlags();
8353
8354 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8355 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8356 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8357 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8358 {
8359 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8360 {
8361 pVCpu->hm.s.Event.fPending = false;
8362
8363 /*
8364 * We've injected any pending events. This is really the point of no return (to ring-3).
8365 *
8366 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
8367 * returns from this function, so don't enable them here.
8368 */
8369 return VINF_SUCCESS;
8370 }
8371
8372 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8373 rcStrict = VINF_EM_RAW_INTERRUPT;
8374 }
8375 else
8376 {
8377 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8378 rcStrict = VINF_EM_RAW_TO_R3;
8379 }
8380
8381 ASMSetFlags(pVmxTransient->fEFlags);
8382 VMMRZCallRing3Enable(pVCpu);
8383
8384 return rcStrict;
8385}
8386
8387
8388/**
8389 * Prepares to run guest code in VT-x and we've committed to doing so. This
8390 * means there is no backing out to ring-3 or anywhere else at this
8391 * point.
8392 *
8393 * @param pVM The cross context VM structure.
8394 * @param pVCpu The cross context virtual CPU structure.
8395 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8396 * out-of-sync. Make sure to update the required fields
8397 * before using them.
8398 * @param pVmxTransient Pointer to the VMX transient structure.
8399 *
8400 * @remarks Called with preemption disabled.
8401 * @remarks No-long-jump zone!!!
8402 */
8403static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8404{
8405 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8406 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8407 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8408
8409 /*
8410 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8411 */
8412 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8413 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8414
8415 if (!CPUMIsGuestFPUStateActive(pVCpu))
8416 {
8417 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8418 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8419 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT;
8420 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8421 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
8422 }
8423
8424 /*
8425 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8426 */
8427 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8428 && pVCpu->hm.s.vmx.cMsrs > 0)
8429 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8430
8431 /*
8432 * Re-save the host state bits as we may've been preempted (only happens when
8433 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8434 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8435 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8436 * See @bugref{8432}.
8437 */
8438 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8439 {
8440 int rc = hmR0VmxExportHostState(pVCpu);
8441 AssertRC(rc);
8442 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
8443 }
8444 Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
8445
8446 /*
8447 * Export the state shared between host and guest (FPU, debug, lazy MSRs).
8448 */
8449 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
8450 hmR0VmxExportSharedState(pVM, pVCpu, pMixedCtx);
8451 AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8452
8453 /* Store status of the shared guest-host state at the time of VM-entry. */
8454#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8455 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8456 {
8457 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8458 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8459 }
8460 else
8461#endif
8462 {
8463 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8464 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8465 }
8466
8467 /*
8468 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8469 */
8470 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8471 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8472
8473 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8474 RTCPUID idCurrentCpu = pCpu->idCpu;
8475 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8476 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8477 {
8478 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8479 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8480 }
8481
8482 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8483 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8484 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8485 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8486
8487 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8488
8489 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8490 to start executing. */
8491
8492 /*
8493 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8494 */
8495 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8496 {
8497 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8498 {
8499 bool fMsrUpdated;
8500 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
8501 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8502 &fMsrUpdated);
8503 AssertRC(rc2);
8504 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8505 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8506 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8507 }
8508 else
8509 {
8510 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8511 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8512 }
8513 }
8514
8515 if (pVM->cpum.ro.GuestFeatures.fIbrs)
8516 {
8517 bool fMsrUpdated;
8518 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_OTHER_MSRS);
8519 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
8520 &fMsrUpdated);
8521 AssertRC(rc2);
8522 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8523 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8524 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8525 }
8526
8527#ifdef VBOX_STRICT
8528 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8529 hmR0VmxCheckHostEferMsr(pVCpu);
8530 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8531#endif
8532#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8533 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8534 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8535 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8536#endif
8537}
8538
8539
8540/**
8541 * Performs some essential restoration of state after running guest code in
8542 * VT-x.
8543 *
8544 * @param pVCpu The cross context virtual CPU structure.
8545 * @param pVmxTransient Pointer to the VMX transient structure.
8546 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8547 *
8548 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8549 *
8550 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8551 * unconditionally when it is safe to do so.
8552 */
8553static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8554{
8555 uint64_t const uHostTsc = ASMReadTSC();
8556 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8557
8558 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8559 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8560 pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8561 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8562 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8563 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8564
8565 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8566 TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TSCOffset);
8567
8568 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8569 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8570 Assert(!ASMIntAreEnabled());
8571 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8572
8573#if HC_ARCH_BITS == 64
8574 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8575#endif
8576#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8577 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8578 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8579 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8580#else
8581 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8582#endif
8583#ifdef VBOX_STRICT
8584 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8585#endif
8586 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8587
8588 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8589 uint32_t uExitReason;
8590 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8591 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8592 AssertRC(rc);
8593 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8594 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8595
8596 if (rcVMRun == VINF_SUCCESS)
8597 {
8598 /*
8599 * Update the VM-exit history array here even if the VM-entry failed due to:
8600 * - Invalid guest state.
8601 * - MSR loading.
8602 * - Machine-check event.
8603 *
8604 * In any of the above cases we will still have a "valid" VM-exit reason
8605 * despite @a fVMEntryFailed being false.
8606 *
8607 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8608 *
8609 * Note! We don't have CS or RIP at this point. Will probably address that later
8610 * by amending the history entry added here.
8611 */
8612 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
8613 UINT64_MAX, uHostTsc);
8614
8615 if (!pVmxTransient->fVMEntryFailed)
8616 {
8617 VMMRZCallRing3Enable(pVCpu);
8618
8619#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8620 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
8621 AssertRC(rc);
8622#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8623 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_RFLAGS);
8624 AssertRC(rc);
8625#else
8626 /*
8627 * Import the guest-interruptibility state always as we need it while evaluating
8628 * injecting events on re-entry.
8629 *
8630 * We don't import CR0 (when Unrestricted guest execution is unavailable) despite
8631 * checking for real-mode while exporting the state because all bits that cause
8632 * mode changes wrt CR0 are intercepted.
8633 */
8634 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
8635 AssertRC(rc);
8636#endif
8637
8638 /*
8639 * Sync the TPR shadow with our APIC state.
8640 */
8641 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8642 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8643 {
8644 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8645 AssertRC(rc);
8646 ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8647 }
8648
8649 return;
8650 }
8651 }
8652 else
8653 {
8654 Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
8655 }
8656
8657 VMMRZCallRing3Enable(pVCpu);
8658}
8659
8660
8661/**
8662 * Runs the guest code using VT-x the normal way.
8663 *
8664 * @returns VBox status code.
8665 * @param pVM The cross context VM structure.
8666 * @param pVCpu The cross context virtual CPU structure.
8667 * @param pCtx Pointer to the guest-CPU context.
8668 *
8669 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8670 */
8671static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8672{
8673 VMXTRANSIENT VmxTransient;
8674 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8675 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8676 uint32_t cLoops = 0;
8677
8678 for (;; cLoops++)
8679 {
8680 Assert(!HMR0SuspendPending());
8681 HMVMX_ASSERT_CPU_SAFE();
8682
8683 /* Preparatory work for running guest code, this may force us to return
8684 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8685 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8686 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8687 if (rcStrict != VINF_SUCCESS)
8688 break;
8689
8690 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8691 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8692 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8693
8694 /* Restore any residual host-state and save any bits shared between host
8695 and guest into the guest-CPU state. Re-enables interrupts! */
8696 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
8697
8698 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8699 if (RT_SUCCESS(rcRun))
8700 { /* very likely */ }
8701 else
8702 {
8703 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8704 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
8705 return rcRun;
8706 }
8707
8708 /* Profile the VM-exit. */
8709 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8710 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8711 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8712 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8713 HMVMX_START_EXIT_DISPATCH_PROF();
8714
8715 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8716
8717 /* Handle the VM-exit. */
8718#ifdef HMVMX_USE_FUNCTION_TABLE
8719 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8720#else
8721 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8722#endif
8723 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8724 if (rcStrict == VINF_SUCCESS)
8725 {
8726 if (cLoops <= pVM->hm.s.cMaxResumeLoops)
8727 continue; /* likely */
8728 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8729 rcStrict = VINF_EM_RAW_INTERRUPT;
8730 }
8731 break;
8732 }
8733
8734 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8735 return rcStrict;
8736}
8737
8738
8739
8740/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8741 * probes.
8742 *
8743 * The following few functions and associated structure contains the bloat
8744 * necessary for providing detailed debug events and dtrace probes as well as
8745 * reliable host side single stepping. This works on the principle of
8746 * "subclassing" the normal execution loop and workers. We replace the loop
8747 * method completely and override selected helpers to add necessary adjustments
8748 * to their core operation.
8749 *
8750 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8751 * any performance for debug and analysis features.
8752 *
8753 * @{
8754 */
8755
8756/**
8757 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
8758 * the debug run loop.
8759 */
8760typedef struct VMXRUNDBGSTATE
8761{
8762 /** The RIP we started executing at. This is for detecting that we stepped. */
8763 uint64_t uRipStart;
8764 /** The CS we started executing with. */
8765 uint16_t uCsStart;
8766
8767 /** Whether we've actually modified the 1st execution control field. */
8768 bool fModifiedProcCtls : 1;
8769 /** Whether we've actually modified the 2nd execution control field. */
8770 bool fModifiedProcCtls2 : 1;
8771 /** Whether we've actually modified the exception bitmap. */
8772 bool fModifiedXcptBitmap : 1;
8773
8774 /** We desire the modified the CR0 mask to be cleared. */
8775 bool fClearCr0Mask : 1;
8776 /** We desire the modified the CR4 mask to be cleared. */
8777 bool fClearCr4Mask : 1;
8778 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
8779 uint32_t fCpe1Extra;
8780 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
8781 uint32_t fCpe1Unwanted;
8782 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
8783 uint32_t fCpe2Extra;
8784 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
8785 uint32_t bmXcptExtra;
8786 /** The sequence number of the Dtrace provider settings the state was
8787 * configured against. */
8788 uint32_t uDtraceSettingsSeqNo;
8789 /** VM-exits to check (one bit per VM-exit). */
8790 uint32_t bmExitsToCheck[3];
8791
8792 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
8793 uint32_t fProcCtlsInitial;
8794 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
8795 uint32_t fProcCtls2Initial;
8796 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
8797 uint32_t bmXcptInitial;
8798} VMXRUNDBGSTATE;
8799AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
8800typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
8801
8802
8803/**
8804 * Initializes the VMXRUNDBGSTATE structure.
8805 *
8806 * @param pVCpu The cross context virtual CPU structure of the
8807 * calling EMT.
8808 * @param pCtx The CPU register context to go with @a pVCpu.
8809 * @param pDbgState The structure to initialize.
8810 */
8811static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
8812{
8813 pDbgState->uRipStart = pCtx->rip;
8814 pDbgState->uCsStart = pCtx->cs.Sel;
8815
8816 pDbgState->fModifiedProcCtls = false;
8817 pDbgState->fModifiedProcCtls2 = false;
8818 pDbgState->fModifiedXcptBitmap = false;
8819 pDbgState->fClearCr0Mask = false;
8820 pDbgState->fClearCr4Mask = false;
8821 pDbgState->fCpe1Extra = 0;
8822 pDbgState->fCpe1Unwanted = 0;
8823 pDbgState->fCpe2Extra = 0;
8824 pDbgState->bmXcptExtra = 0;
8825 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
8826 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
8827 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
8828}
8829
8830
8831/**
8832 * Updates the VMSC fields with changes requested by @a pDbgState.
8833 *
8834 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
8835 * immediately before executing guest code, i.e. when interrupts are disabled.
8836 * We don't check status codes here as we cannot easily assert or return in the
8837 * latter case.
8838 *
8839 * @param pVCpu The cross context virtual CPU structure.
8840 * @param pDbgState The debug state.
8841 */
8842static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
8843{
8844 /*
8845 * Ensure desired flags in VMCS control fields are set.
8846 * (Ignoring write failure here, as we're committed and it's just debug extras.)
8847 *
8848 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
8849 * there should be no stale data in pCtx at this point.
8850 */
8851 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
8852 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
8853 {
8854 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
8855 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
8856 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8857 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
8858 pDbgState->fModifiedProcCtls = true;
8859 }
8860
8861 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
8862 {
8863 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
8864 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
8865 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
8866 pDbgState->fModifiedProcCtls2 = true;
8867 }
8868
8869 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
8870 {
8871 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
8872 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8873 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
8874 pDbgState->fModifiedXcptBitmap = true;
8875 }
8876
8877 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
8878 {
8879 pVCpu->hm.s.vmx.u32CR0Mask = 0;
8880 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
8881 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
8882 }
8883
8884 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
8885 {
8886 pVCpu->hm.s.vmx.u32CR4Mask = 0;
8887 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
8888 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
8889 }
8890}
8891
8892
8893static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
8894{
8895 /*
8896 * Restore VM-exit control settings as we may not reenter this function the
8897 * next time around.
8898 */
8899 /* We reload the initial value, trigger what we can of recalculations the
8900 next time around. From the looks of things, that's all that's required atm. */
8901 if (pDbgState->fModifiedProcCtls)
8902 {
8903 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
8904 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
8905 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
8906 AssertRCReturn(rc2, rc2);
8907 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
8908 }
8909
8910 /* We're currently the only ones messing with this one, so just restore the
8911 cached value and reload the field. */
8912 if ( pDbgState->fModifiedProcCtls2
8913 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
8914 {
8915 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
8916 AssertRCReturn(rc2, rc2);
8917 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
8918 }
8919
8920 /* If we've modified the exception bitmap, we restore it and trigger
8921 reloading and partial recalculation the next time around. */
8922 if (pDbgState->fModifiedXcptBitmap)
8923 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
8924
8925 return rcStrict;
8926}
8927
8928
8929/**
8930 * Configures VM-exit controls for current DBGF and DTrace settings.
8931 *
8932 * This updates @a pDbgState and the VMCS execution control fields to reflect
8933 * the necessary VM-exits demanded by DBGF and DTrace.
8934 *
8935 * @param pVCpu The cross context virtual CPU structure.
8936 * @param pDbgState The debug state.
8937 * @param pVmxTransient Pointer to the VMX transient structure. May update
8938 * fUpdateTscOffsettingAndPreemptTimer.
8939 */
8940static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
8941{
8942 /*
8943 * Take down the dtrace serial number so we can spot changes.
8944 */
8945 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
8946 ASMCompilerBarrier();
8947
8948 /*
8949 * We'll rebuild most of the middle block of data members (holding the
8950 * current settings) as we go along here, so start by clearing it all.
8951 */
8952 pDbgState->bmXcptExtra = 0;
8953 pDbgState->fCpe1Extra = 0;
8954 pDbgState->fCpe1Unwanted = 0;
8955 pDbgState->fCpe2Extra = 0;
8956 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
8957 pDbgState->bmExitsToCheck[i] = 0;
8958
8959 /*
8960 * Software interrupts (INT XXh) - no idea how to trigger these...
8961 */
8962 PVM pVM = pVCpu->CTX_SUFF(pVM);
8963 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
8964 || VBOXVMM_INT_SOFTWARE_ENABLED())
8965 {
8966 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
8967 }
8968
8969 /*
8970 * INT3 breakpoints - triggered by #BP exceptions.
8971 */
8972 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
8973 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
8974
8975 /*
8976 * Exception bitmap and XCPT events+probes.
8977 */
8978 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
8979 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
8980 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
8981
8982 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
8983 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
8984 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
8985 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
8986 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
8987 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
8988 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
8989 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
8990 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
8991 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
8992 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
8993 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
8994 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
8995 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
8996 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
8997 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
8998 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
8999 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9000
9001 if (pDbgState->bmXcptExtra)
9002 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9003
9004 /*
9005 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9006 *
9007 * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
9008 * So, when adding/changing/removing please don't forget to update it.
9009 *
9010 * Some of the macros are picking up local variables to save horizontal space,
9011 * (being able to see it in a table is the lesser evil here).
9012 */
9013#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9014 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9015 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9016#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9017 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9018 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9019 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9020 } else do { } while (0)
9021#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9022 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9023 { \
9024 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9025 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9026 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9027 } else do { } while (0)
9028#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9029 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9030 { \
9031 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9032 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9033 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9034 } else do { } while (0)
9035#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9036 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9037 { \
9038 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9039 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9040 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9041 } else do { } while (0)
9042
9043 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9044 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9045 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9046 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9047 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9048
9049 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9050 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9051 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9052 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9053 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9054 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9055 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9056 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9057 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9058 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9059 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9060 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9061 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9062 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9063 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9064 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9065 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9066 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9067 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9068 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9069 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9070 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9071 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9072 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9073 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9074 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9075 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9076 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9077 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9078 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9079 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9080 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9081 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9082 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9083 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9084 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9085
9086 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9087 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9088 {
9089 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
9090 | CPUMCTX_EXTRN_CR4
9091 | CPUMCTX_EXTRN_APIC_TPR);
9092 AssertRC(rc);
9093
9094#if 0 /** @todo fix me */
9095 pDbgState->fClearCr0Mask = true;
9096 pDbgState->fClearCr4Mask = true;
9097#endif
9098 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9099 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9100 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9101 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9102 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9103 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9104 require clearing here and in the loop if we start using it. */
9105 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9106 }
9107 else
9108 {
9109 if (pDbgState->fClearCr0Mask)
9110 {
9111 pDbgState->fClearCr0Mask = false;
9112 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
9113 }
9114 if (pDbgState->fClearCr4Mask)
9115 {
9116 pDbgState->fClearCr4Mask = false;
9117 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
9118 }
9119 }
9120 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9121 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9122
9123 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9124 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9125 {
9126 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9127 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9128 }
9129 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9130 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9131
9132 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9133 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9134 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9135 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9136 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9137 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9138 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9139 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9140#if 0 /** @todo too slow, fix handler. */
9141 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9142#endif
9143 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9144
9145 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9146 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9147 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9148 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9149 {
9150 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9151 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9152 }
9153 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9154 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9155 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9156 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9157
9158 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9159 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9160 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9161 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9162 {
9163 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9164 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9165 }
9166 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9167 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9168 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9169 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9170
9171 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9172 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9173 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9174 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9175 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9176 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9177 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9178 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9179 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9180 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9181 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9182 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9183 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9184 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9185 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9186 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9187 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9188 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9189 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9190 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9191 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9192 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9193
9194#undef IS_EITHER_ENABLED
9195#undef SET_ONLY_XBM_IF_EITHER_EN
9196#undef SET_CPE1_XBM_IF_EITHER_EN
9197#undef SET_CPEU_XBM_IF_EITHER_EN
9198#undef SET_CPE2_XBM_IF_EITHER_EN
9199
9200 /*
9201 * Sanitize the control stuff.
9202 */
9203 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9204 if (pDbgState->fCpe2Extra)
9205 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9206 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9207 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9208 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9209 {
9210 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9211 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9212 }
9213
9214 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9215 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9216 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9217 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9218}
9219
9220
9221/**
9222 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9223 * appropriate.
9224 *
9225 * The caller has checked the VM-exit against the
9226 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9227 * already, so we don't have to do that either.
9228 *
9229 * @returns Strict VBox status code (i.e. informational status codes too).
9230 * @param pVM The cross context VM structure.
9231 * @param pVCpu The cross context virtual CPU structure.
9232 * @param pMixedCtx Pointer to the guest-CPU context.
9233 * @param pVmxTransient Pointer to the VMX-transient structure.
9234 * @param uExitReason The VM-exit reason.
9235 *
9236 * @remarks The name of this function is displayed by dtrace, so keep it short
9237 * and to the point. No longer than 33 chars long, please.
9238 */
9239static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9240 PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9241{
9242 /*
9243 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9244 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9245 *
9246 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9247 * does. Must add/change/remove both places. Same ordering, please.
9248 *
9249 * Added/removed events must also be reflected in the next section
9250 * where we dispatch dtrace events.
9251 */
9252 bool fDtrace1 = false;
9253 bool fDtrace2 = false;
9254 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9255 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9256 uint32_t uEventArg = 0;
9257#define SET_EXIT(a_EventSubName) \
9258 do { \
9259 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9260 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9261 } while (0)
9262#define SET_BOTH(a_EventSubName) \
9263 do { \
9264 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9265 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9266 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9267 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9268 } while (0)
9269 switch (uExitReason)
9270 {
9271 case VMX_EXIT_MTF:
9272 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9273
9274 case VMX_EXIT_XCPT_OR_NMI:
9275 {
9276 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9277 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9278 {
9279 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9280 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9281 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9282 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9283 {
9284 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9285 {
9286 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9287 uEventArg = pVmxTransient->uExitIntErrorCode;
9288 }
9289 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9290 switch (enmEvent1)
9291 {
9292 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9293 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9294 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9295 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9296 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9297 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9298 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9299 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9300 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9301 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9302 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9303 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9304 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9305 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9306 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9307 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9308 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9309 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9310 default: break;
9311 }
9312 }
9313 else
9314 AssertFailed();
9315 break;
9316
9317 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9318 uEventArg = idxVector;
9319 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9320 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9321 break;
9322 }
9323 break;
9324 }
9325
9326 case VMX_EXIT_TRIPLE_FAULT:
9327 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9328 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9329 break;
9330 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9331 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9332 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9333 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9334 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9335
9336 /* Instruction specific VM-exits: */
9337 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9338 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9339 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9340 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9341 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9342 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9343 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9344 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9345 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9346 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9347 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9348 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9349 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9350 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9351 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9352 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9353 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9354 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9355 case VMX_EXIT_MOV_CRX:
9356 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9357 if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
9358 SET_BOTH(CRX_READ);
9359 else
9360 SET_BOTH(CRX_WRITE);
9361 uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQualification);
9362 break;
9363 case VMX_EXIT_MOV_DRX:
9364 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9365 if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification)
9366 == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
9367 SET_BOTH(DRX_READ);
9368 else
9369 SET_BOTH(DRX_WRITE);
9370 uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification);
9371 break;
9372 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9373 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9374 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9375 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9376 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9377 case VMX_EXIT_XDTR_ACCESS:
9378 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9379 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9380 {
9381 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9382 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9383 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9384 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9385 }
9386 break;
9387
9388 case VMX_EXIT_TR_ACCESS:
9389 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9390 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9391 {
9392 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9393 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9394 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9395 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9396 }
9397 break;
9398
9399 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9400 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9401 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9402 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9403 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9404 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9405 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9406 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9407 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9408 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9409 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9410
9411 /* Events that aren't relevant at this point. */
9412 case VMX_EXIT_EXT_INT:
9413 case VMX_EXIT_INT_WINDOW:
9414 case VMX_EXIT_NMI_WINDOW:
9415 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9416 case VMX_EXIT_PREEMPT_TIMER:
9417 case VMX_EXIT_IO_INSTR:
9418 break;
9419
9420 /* Errors and unexpected events. */
9421 case VMX_EXIT_INIT_SIGNAL:
9422 case VMX_EXIT_SIPI:
9423 case VMX_EXIT_IO_SMI:
9424 case VMX_EXIT_SMI:
9425 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9426 case VMX_EXIT_ERR_MSR_LOAD:
9427 case VMX_EXIT_ERR_MACHINE_CHECK:
9428 break;
9429
9430 default:
9431 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9432 break;
9433 }
9434#undef SET_BOTH
9435#undef SET_EXIT
9436
9437 /*
9438 * Dtrace tracepoints go first. We do them here at once so we don't
9439 * have to copy the guest state saving and stuff a few dozen times.
9440 * Down side is that we've got to repeat the switch, though this time
9441 * we use enmEvent since the probes are a subset of what DBGF does.
9442 */
9443 if (fDtrace1 || fDtrace2)
9444 {
9445 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9446 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9447 switch (enmEvent1)
9448 {
9449 /** @todo consider which extra parameters would be helpful for each probe. */
9450 case DBGFEVENT_END: break;
9451 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9452 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9453 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9454 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9455 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9456 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9457 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9458 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9459 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9460 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9461 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9462 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9463 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9464 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9465 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9466 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9467 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9468 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9469 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9470 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9471 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9472 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9473 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9474 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9475 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9476 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9477 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9478 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9479 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9480 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9481 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9482 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9483 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9484 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9485 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9486 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9487 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9488 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9489 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9490 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9491 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9492 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9493 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9494 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9495 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9496 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9497 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9498 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9499 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9500 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9501 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9502 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9503 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9504 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9505 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9506 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9507 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9508 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9509 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9510 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9511 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9512 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9513 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9514 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9515 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9516 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9517 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9518 }
9519 switch (enmEvent2)
9520 {
9521 /** @todo consider which extra parameters would be helpful for each probe. */
9522 case DBGFEVENT_END: break;
9523 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9524 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9525 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9526 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9527 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9528 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9529 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9530 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9531 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9532 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9533 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9534 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9535 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9536 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9537 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9538 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9539 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9540 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9541 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9542 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9543 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9544 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9545 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9546 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9547 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9548 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9549 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9550 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9551 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9552 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9553 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9554 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9555 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9556 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9557 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9558 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9559 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9560 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9561 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9562 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9563 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9564 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9565 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9566 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9567 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9568 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9569 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9570 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9571 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9572 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9573 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9574 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9575 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9576 }
9577 }
9578
9579 /*
9580 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9581 * the DBGF call will do a full check).
9582 *
9583 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9584 * Note! If we have to events, we prioritize the first, i.e. the instruction
9585 * one, in order to avoid event nesting.
9586 */
9587 if ( enmEvent1 != DBGFEVENT_END
9588 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9589 {
9590 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9591 if (rcStrict != VINF_SUCCESS)
9592 return rcStrict;
9593 }
9594 else if ( enmEvent2 != DBGFEVENT_END
9595 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9596 {
9597 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9598 if (rcStrict != VINF_SUCCESS)
9599 return rcStrict;
9600 }
9601
9602 return VINF_SUCCESS;
9603}
9604
9605
9606/**
9607 * Single-stepping VM-exit filtering.
9608 *
9609 * This is preprocessing the VM-exits and deciding whether we've gotten far
9610 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9611 * handling is performed.
9612 *
9613 * @returns Strict VBox status code (i.e. informational status codes too).
9614 * @param pVM The cross context VM structure.
9615 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9616 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9617 * out-of-sync. Make sure to update the required
9618 * fields before using them.
9619 * @param pVmxTransient Pointer to the VMX-transient structure.
9620 * @param uExitReason The VM-exit reason.
9621 * @param pDbgState The debug state.
9622 */
9623DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9624 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9625{
9626 /*
9627 * Expensive (saves context) generic dtrace VM-exit probe.
9628 */
9629 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9630 { /* more likely */ }
9631 else
9632 {
9633 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9634 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9635 AssertRC(rc);
9636 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9637 }
9638
9639 /*
9640 * Check for host NMI, just to get that out of the way.
9641 */
9642 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9643 { /* normally likely */ }
9644 else
9645 {
9646 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9647 AssertRCReturn(rc2, rc2);
9648 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9649 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9650 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9651 }
9652
9653 /*
9654 * Check for single stepping event if we're stepping.
9655 */
9656 if (pVCpu->hm.s.fSingleInstruction)
9657 {
9658 switch (uExitReason)
9659 {
9660 case VMX_EXIT_MTF:
9661 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9662
9663 /* Various events: */
9664 case VMX_EXIT_XCPT_OR_NMI:
9665 case VMX_EXIT_EXT_INT:
9666 case VMX_EXIT_TRIPLE_FAULT:
9667 case VMX_EXIT_INT_WINDOW:
9668 case VMX_EXIT_NMI_WINDOW:
9669 case VMX_EXIT_TASK_SWITCH:
9670 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9671 case VMX_EXIT_APIC_ACCESS:
9672 case VMX_EXIT_EPT_VIOLATION:
9673 case VMX_EXIT_EPT_MISCONFIG:
9674 case VMX_EXIT_PREEMPT_TIMER:
9675
9676 /* Instruction specific VM-exits: */
9677 case VMX_EXIT_CPUID:
9678 case VMX_EXIT_GETSEC:
9679 case VMX_EXIT_HLT:
9680 case VMX_EXIT_INVD:
9681 case VMX_EXIT_INVLPG:
9682 case VMX_EXIT_RDPMC:
9683 case VMX_EXIT_RDTSC:
9684 case VMX_EXIT_RSM:
9685 case VMX_EXIT_VMCALL:
9686 case VMX_EXIT_VMCLEAR:
9687 case VMX_EXIT_VMLAUNCH:
9688 case VMX_EXIT_VMPTRLD:
9689 case VMX_EXIT_VMPTRST:
9690 case VMX_EXIT_VMREAD:
9691 case VMX_EXIT_VMRESUME:
9692 case VMX_EXIT_VMWRITE:
9693 case VMX_EXIT_VMXOFF:
9694 case VMX_EXIT_VMXON:
9695 case VMX_EXIT_MOV_CRX:
9696 case VMX_EXIT_MOV_DRX:
9697 case VMX_EXIT_IO_INSTR:
9698 case VMX_EXIT_RDMSR:
9699 case VMX_EXIT_WRMSR:
9700 case VMX_EXIT_MWAIT:
9701 case VMX_EXIT_MONITOR:
9702 case VMX_EXIT_PAUSE:
9703 case VMX_EXIT_XDTR_ACCESS:
9704 case VMX_EXIT_TR_ACCESS:
9705 case VMX_EXIT_INVEPT:
9706 case VMX_EXIT_RDTSCP:
9707 case VMX_EXIT_INVVPID:
9708 case VMX_EXIT_WBINVD:
9709 case VMX_EXIT_XSETBV:
9710 case VMX_EXIT_RDRAND:
9711 case VMX_EXIT_INVPCID:
9712 case VMX_EXIT_VMFUNC:
9713 case VMX_EXIT_RDSEED:
9714 case VMX_EXIT_XSAVES:
9715 case VMX_EXIT_XRSTORS:
9716 {
9717 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
9718 | CPUMCTX_EXTRN_CS);
9719 AssertRCReturn(rc, rc);
9720 if ( pMixedCtx->rip != pDbgState->uRipStart
9721 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
9722 return VINF_EM_DBG_STEPPED;
9723 break;
9724 }
9725
9726 /* Errors and unexpected events: */
9727 case VMX_EXIT_INIT_SIGNAL:
9728 case VMX_EXIT_SIPI:
9729 case VMX_EXIT_IO_SMI:
9730 case VMX_EXIT_SMI:
9731 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9732 case VMX_EXIT_ERR_MSR_LOAD:
9733 case VMX_EXIT_ERR_MACHINE_CHECK:
9734 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9735 break;
9736
9737 default:
9738 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9739 break;
9740 }
9741 }
9742
9743 /*
9744 * Check for debugger event breakpoints and dtrace probes.
9745 */
9746 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9747 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9748 {
9749 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9750 if (rcStrict != VINF_SUCCESS)
9751 return rcStrict;
9752 }
9753
9754 /*
9755 * Normal processing.
9756 */
9757#ifdef HMVMX_USE_FUNCTION_TABLE
9758 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9759#else
9760 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9761#endif
9762}
9763
9764
9765/**
9766 * Single steps guest code using VT-x.
9767 *
9768 * @returns Strict VBox status code (i.e. informational status codes too).
9769 * @param pVM The cross context VM structure.
9770 * @param pVCpu The cross context virtual CPU structure.
9771 * @param pCtx Pointer to the guest-CPU context.
9772 *
9773 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
9774 */
9775static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9776{
9777 VMXTRANSIENT VmxTransient;
9778 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9779
9780 /* Set HMCPU indicators. */
9781 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
9782 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
9783 pVCpu->hm.s.fDebugWantRdTscExit = false;
9784 pVCpu->hm.s.fUsingDebugLoop = true;
9785
9786 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
9787 VMXRUNDBGSTATE DbgState;
9788 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
9789 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9790
9791 /*
9792 * The loop.
9793 */
9794 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9795 for (uint32_t cLoops = 0; ; cLoops++)
9796 {
9797 Assert(!HMR0SuspendPending());
9798 HMVMX_ASSERT_CPU_SAFE();
9799 bool fStepping = pVCpu->hm.s.fSingleInstruction;
9800
9801 /*
9802 * Preparatory work for running guest code, this may force us to return
9803 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
9804 */
9805 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9806 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
9807 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
9808 if (rcStrict != VINF_SUCCESS)
9809 break;
9810
9811 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9812 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
9813
9814 /*
9815 * Now we can run the guest code.
9816 */
9817 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9818
9819 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9820
9821 /*
9822 * Restore any residual host-state and save any bits shared between host
9823 * and guest into the guest-CPU state. Re-enables interrupts!
9824 */
9825 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
9826
9827 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9828 if (RT_SUCCESS(rcRun))
9829 { /* very likely */ }
9830 else
9831 {
9832 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9833 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9834 return rcRun;
9835 }
9836
9837 /* Profile the VM-exit. */
9838 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9839 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9840 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9841 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9842 HMVMX_START_EXIT_DISPATCH_PROF();
9843
9844 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9845
9846 /*
9847 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
9848 */
9849 rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
9850 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9851 if (rcStrict != VINF_SUCCESS)
9852 break;
9853 if (cLoops > pVM->hm.s.cMaxResumeLoops)
9854 {
9855 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9856 rcStrict = VINF_EM_RAW_INTERRUPT;
9857 break;
9858 }
9859
9860 /*
9861 * Stepping: Did the RIP change, if so, consider it a single step.
9862 * Otherwise, make sure one of the TFs gets set.
9863 */
9864 if (fStepping)
9865 {
9866 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
9867 | CPUMCTX_EXTRN_CS);
9868 AssertRC(rc);
9869 if ( pCtx->rip != DbgState.uRipStart
9870 || pCtx->cs.Sel != DbgState.uCsStart)
9871 {
9872 rcStrict = VINF_EM_DBG_STEPPED;
9873 break;
9874 }
9875 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
9876 }
9877
9878 /*
9879 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
9880 */
9881 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
9882 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9883 }
9884
9885 /*
9886 * Clear the X86_EFL_TF if necessary.
9887 */
9888 if (pVCpu->hm.s.fClearTrapFlag)
9889 {
9890 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
9891 AssertRC(rc);
9892 pVCpu->hm.s.fClearTrapFlag = false;
9893 pCtx->eflags.Bits.u1TF = 0;
9894 }
9895 /** @todo there seems to be issues with the resume flag when the monitor trap
9896 * flag is pending without being used. Seen early in bios init when
9897 * accessing APIC page in protected mode. */
9898
9899 /*
9900 * Restore VM-exit control settings as we may not reenter this function the
9901 * next time around.
9902 */
9903 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
9904
9905 /* Restore HMCPU indicators. */
9906 pVCpu->hm.s.fUsingDebugLoop = false;
9907 pVCpu->hm.s.fDebugWantRdTscExit = false;
9908 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
9909
9910 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9911 return rcStrict;
9912}
9913
9914
9915/** @} */
9916
9917
9918/**
9919 * Checks if any expensive dtrace probes are enabled and we should go to the
9920 * debug loop.
9921 *
9922 * @returns true if we should use debug loop, false if not.
9923 */
9924static bool hmR0VmxAnyExpensiveProbesEnabled(void)
9925{
9926 /* It's probably faster to OR the raw 32-bit counter variables together.
9927 Since the variables are in an array and the probes are next to one
9928 another (more or less), we have good locality. So, better read
9929 eight-nine cache lines ever time and only have one conditional, than
9930 128+ conditionals, right? */
9931 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
9932 | VBOXVMM_XCPT_DE_ENABLED_RAW()
9933 | VBOXVMM_XCPT_DB_ENABLED_RAW()
9934 | VBOXVMM_XCPT_BP_ENABLED_RAW()
9935 | VBOXVMM_XCPT_OF_ENABLED_RAW()
9936 | VBOXVMM_XCPT_BR_ENABLED_RAW()
9937 | VBOXVMM_XCPT_UD_ENABLED_RAW()
9938 | VBOXVMM_XCPT_NM_ENABLED_RAW()
9939 | VBOXVMM_XCPT_DF_ENABLED_RAW()
9940 | VBOXVMM_XCPT_TS_ENABLED_RAW()
9941 | VBOXVMM_XCPT_NP_ENABLED_RAW()
9942 | VBOXVMM_XCPT_SS_ENABLED_RAW()
9943 | VBOXVMM_XCPT_GP_ENABLED_RAW()
9944 | VBOXVMM_XCPT_PF_ENABLED_RAW()
9945 | VBOXVMM_XCPT_MF_ENABLED_RAW()
9946 | VBOXVMM_XCPT_AC_ENABLED_RAW()
9947 | VBOXVMM_XCPT_XF_ENABLED_RAW()
9948 | VBOXVMM_XCPT_VE_ENABLED_RAW()
9949 | VBOXVMM_XCPT_SX_ENABLED_RAW()
9950 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
9951 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
9952 ) != 0
9953 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
9954 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
9955 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
9956 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
9957 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
9958 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
9959 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
9960 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
9961 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
9962 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
9963 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
9964 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
9965 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
9966 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
9967 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
9968 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
9969 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
9970 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
9971 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
9972 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
9973 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
9974 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
9975 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
9976 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
9977 | VBOXVMM_INSTR_STR_ENABLED_RAW()
9978 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
9979 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
9980 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
9981 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
9982 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
9983 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
9984 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
9985 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
9986 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
9987 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
9988 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
9989 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
9990 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
9991 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
9992 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
9993 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
9994 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
9995 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
9996 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
9997 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
9998 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
9999 ) != 0
10000 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10001 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10002 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10003 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10004 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10005 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10006 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10007 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10008 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10009 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10010 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10011 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10012 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10013 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10014 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10015 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10016 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10017 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10018 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10019 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10020 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10021 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10022 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10023 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10024 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10025 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10026 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10027 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10028 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10029 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10030 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10031 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10032 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10033 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10034 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10035 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10036 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10037 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10038 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10039 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10040 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10041 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10042 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10043 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10044 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10045 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10046 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10047 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10048 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10049 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10050 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10051 ) != 0;
10052}
10053
10054
10055/**
10056 * Runs the guest code using VT-x.
10057 *
10058 * @returns Strict VBox status code (i.e. informational status codes too).
10059 * @param pVM The cross context VM structure.
10060 * @param pVCpu The cross context virtual CPU structure.
10061 * @param pCtx Pointer to the guest-CPU context.
10062 */
10063VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10064{
10065 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10066 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10067 HMVMX_ASSERT_PREEMPT_SAFE();
10068
10069 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10070
10071 VBOXSTRICTRC rcStrict;
10072 if ( !pVCpu->hm.s.fUseDebugLoop
10073 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10074 && !DBGFIsStepping(pVCpu)
10075 && !pVM->dbgf.ro.cEnabledInt3Breakpoints)
10076 rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10077 else
10078 rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10079
10080 if (rcStrict == VERR_EM_INTERPRETER)
10081 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10082 else if (rcStrict == VINF_EM_RESET)
10083 rcStrict = VINF_EM_TRIPLE_FAULT;
10084
10085 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10086 if (RT_FAILURE(rc2))
10087 {
10088 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10089 rcStrict = rc2;
10090 }
10091 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10092 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10093 return rcStrict;
10094}
10095
10096
10097#ifndef HMVMX_USE_FUNCTION_TABLE
10098DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10099{
10100#ifdef DEBUG_ramshankar
10101#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
10102 do { \
10103 if (a_fSave != 0) \
10104 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
10105 VBOXSTRICTRC rcStrict = a_CallExpr; \
10106 if (a_fSave != 0) \
10107 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
10108 return rcStrict; \
10109 } while (0)
10110#else
10111# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
10112#endif
10113 switch (rcReason)
10114 {
10115 case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10116 case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10117 case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10118 case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10119 case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10120 case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10121 case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10122 case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10123 case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10124 case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10125 case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10126 case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10127 case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10128 case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10129 case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10130 case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10131 case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10132 case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10133 case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10134 case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10135 case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10136 case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10137 case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10138 case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10139 case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10140 case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10141 case VMX_EXIT_XDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10142 case VMX_EXIT_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10143 case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10144 case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10145 case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10146 case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10147 case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10148 case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10149
10150 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10151 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10152 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10153 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10154 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10155 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10156 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10157 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10158 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10159
10160 case VMX_EXIT_VMCLEAR:
10161 case VMX_EXIT_VMLAUNCH:
10162 case VMX_EXIT_VMPTRLD:
10163 case VMX_EXIT_VMPTRST:
10164 case VMX_EXIT_VMREAD:
10165 case VMX_EXIT_VMRESUME:
10166 case VMX_EXIT_VMWRITE:
10167 case VMX_EXIT_VMXOFF:
10168 case VMX_EXIT_VMXON:
10169 case VMX_EXIT_INVEPT:
10170 case VMX_EXIT_INVVPID:
10171 case VMX_EXIT_VMFUNC:
10172 case VMX_EXIT_XSAVES:
10173 case VMX_EXIT_XRSTORS:
10174 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10175
10176 case VMX_EXIT_ENCLS:
10177 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10178 case VMX_EXIT_PML_FULL:
10179 default:
10180 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10181 }
10182#undef VMEXIT_CALL_RET
10183}
10184#endif /* !HMVMX_USE_FUNCTION_TABLE */
10185
10186
10187#ifdef VBOX_STRICT
10188/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10189# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10190 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10191
10192# define HMVMX_ASSERT_PREEMPT_CPUID() \
10193 do { \
10194 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10195 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10196 } while (0)
10197
10198# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10199 do { \
10200 AssertPtr(pVCpu); \
10201 AssertPtr(pMixedCtx); \
10202 AssertPtr(pVmxTransient); \
10203 Assert(pVmxTransient->fVMEntryFailed == false); \
10204 Assert(ASMIntAreEnabled()); \
10205 HMVMX_ASSERT_PREEMPT_SAFE(); \
10206 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10207 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10208 HMVMX_ASSERT_PREEMPT_SAFE(); \
10209 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10210 HMVMX_ASSERT_PREEMPT_CPUID(); \
10211 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10212 } while (0)
10213
10214# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10215 do { \
10216 Log4Func(("\n")); \
10217 } while (0)
10218#else /* nonstrict builds: */
10219# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10220 do { \
10221 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10222 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10223 } while (0)
10224# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10225#endif
10226
10227
10228/**
10229 * Advances the guest RIP by the specified number of bytes.
10230 *
10231 * @param pVCpu The cross context virtual CPU structure.
10232 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10233 * out-of-sync. Make sure to update the required fields
10234 * before using them.
10235 * @param cbInstr Number of bytes to advance the RIP by.
10236 *
10237 * @remarks No-long-jump zone!!!
10238 */
10239DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10240{
10241 /* Advance the RIP. */
10242 pMixedCtx->rip += cbInstr;
10243 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
10244
10245 /* Update interrupt inhibition. */
10246 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10247 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10248 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10249}
10250
10251
10252/**
10253 * Advances the guest RIP after reading it from the VMCS.
10254 *
10255 * @returns VBox status code, no informational status codes.
10256 * @param pVCpu The cross context virtual CPU structure.
10257 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10258 * out-of-sync. Make sure to update the required fields
10259 * before using them.
10260 * @param pVmxTransient Pointer to the VMX transient structure.
10261 *
10262 * @remarks No-long-jump zone!!!
10263 */
10264static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10265{
10266 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10267 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
10268 | CPUMCTX_EXTRN_RFLAGS);
10269 AssertRCReturn(rc, rc);
10270
10271 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10272
10273 /*
10274 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10275 * pending debug exception field as it takes care of priority of events.
10276 *
10277 * See Intel spec. 32.2.1 "Debug Exceptions".
10278 */
10279 if ( !pVCpu->hm.s.fSingleInstruction
10280 && pMixedCtx->eflags.Bits.u1TF)
10281 {
10282 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
10283 AssertRCReturn(rc, rc);
10284 }
10285
10286 return VINF_SUCCESS;
10287}
10288
10289
10290/**
10291 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10292 * and update error record fields accordingly.
10293 *
10294 * @return VMX_IGS_* return codes.
10295 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10296 * wrong with the guest state.
10297 *
10298 * @param pVM The cross context VM structure.
10299 * @param pVCpu The cross context virtual CPU structure.
10300 * @param pCtx Pointer to the guest-CPU state.
10301 *
10302 * @remarks This function assumes our cache of the VMCS controls
10303 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10304 */
10305static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10306{
10307#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10308#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10309 uError = (err); \
10310 break; \
10311 } else do { } while (0)
10312
10313 int rc;
10314 uint32_t uError = VMX_IGS_ERROR;
10315 uint32_t u32Val;
10316 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10317
10318 do
10319 {
10320 /*
10321 * CR0.
10322 */
10323 uint32_t fSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10324 uint32_t fZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10325 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10326 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10327 if (fUnrestrictedGuest)
10328 fSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
10329
10330 uint32_t uGuestCR0;
10331 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uGuestCR0);
10332 AssertRCBreak(rc);
10333 HMVMX_CHECK_BREAK((uGuestCR0 & fSetCR0) == fSetCR0, VMX_IGS_CR0_FIXED1);
10334 HMVMX_CHECK_BREAK(!(uGuestCR0 & ~fZapCR0), VMX_IGS_CR0_FIXED0);
10335 if ( !fUnrestrictedGuest
10336 && (uGuestCR0 & X86_CR0_PG)
10337 && !(uGuestCR0 & X86_CR0_PE))
10338 {
10339 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10340 }
10341
10342 /*
10343 * CR4.
10344 */
10345 uint64_t fSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10346 uint64_t fZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10347
10348 uint32_t uGuestCR4;
10349 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uGuestCR4);
10350 AssertRCBreak(rc);
10351 HMVMX_CHECK_BREAK((uGuestCR4 & fSetCR4) == fSetCR4, VMX_IGS_CR4_FIXED1);
10352 HMVMX_CHECK_BREAK(!(uGuestCR4 & ~fZapCR4), VMX_IGS_CR4_FIXED0);
10353
10354 /*
10355 * IA32_DEBUGCTL MSR.
10356 */
10357 uint64_t u64Val;
10358 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10359 AssertRCBreak(rc);
10360 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10361 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10362 {
10363 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10364 }
10365 uint64_t u64DebugCtlMsr = u64Val;
10366
10367#ifdef VBOX_STRICT
10368 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10369 AssertRCBreak(rc);
10370 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10371#endif
10372 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10373
10374 /*
10375 * RIP and RFLAGS.
10376 */
10377 uint32_t u32Eflags;
10378#if HC_ARCH_BITS == 64
10379 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10380 AssertRCBreak(rc);
10381 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10382 if ( !fLongModeGuest
10383 || !pCtx->cs.Attr.n.u1Long)
10384 {
10385 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10386 }
10387 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10388 * must be identical if the "IA-32e mode guest" VM-entry
10389 * control is 1 and CS.L is 1. No check applies if the
10390 * CPU supports 64 linear-address bits. */
10391
10392 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10393 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10394 AssertRCBreak(rc);
10395 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10396 VMX_IGS_RFLAGS_RESERVED);
10397 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10398 u32Eflags = u64Val;
10399#else
10400 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10401 AssertRCBreak(rc);
10402 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10403 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10404#endif
10405
10406 if ( fLongModeGuest
10407 || ( fUnrestrictedGuest
10408 && !(uGuestCR0 & X86_CR0_PE)))
10409 {
10410 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10411 }
10412
10413 uint32_t u32EntryInfo;
10414 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10415 AssertRCBreak(rc);
10416 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10417 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10418 {
10419 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10420 }
10421
10422 /*
10423 * 64-bit checks.
10424 */
10425#if HC_ARCH_BITS == 64
10426 if (fLongModeGuest)
10427 {
10428 HMVMX_CHECK_BREAK(uGuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10429 HMVMX_CHECK_BREAK(uGuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10430 }
10431
10432 if ( !fLongModeGuest
10433 && (uGuestCR4 & X86_CR4_PCIDE))
10434 {
10435 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10436 }
10437
10438 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10439 * 51:32 beyond the processor's physical-address width are 0. */
10440
10441 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10442 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10443 {
10444 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10445 }
10446
10447 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10448 AssertRCBreak(rc);
10449 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10450
10451 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10452 AssertRCBreak(rc);
10453 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10454#endif
10455
10456 /*
10457 * PERF_GLOBAL MSR.
10458 */
10459 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10460 {
10461 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10462 AssertRCBreak(rc);
10463 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10464 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10465 }
10466
10467 /*
10468 * PAT MSR.
10469 */
10470 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10471 {
10472 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10473 AssertRCBreak(rc);
10474 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10475 for (unsigned i = 0; i < 8; i++)
10476 {
10477 uint8_t u8Val = (u64Val & 0xff);
10478 if ( u8Val != 0 /* UC */
10479 && u8Val != 1 /* WC */
10480 && u8Val != 4 /* WT */
10481 && u8Val != 5 /* WP */
10482 && u8Val != 6 /* WB */
10483 && u8Val != 7 /* UC- */)
10484 {
10485 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10486 }
10487 u64Val >>= 8;
10488 }
10489 }
10490
10491 /*
10492 * EFER MSR.
10493 */
10494 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10495 {
10496 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10497 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10498 AssertRCBreak(rc);
10499 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10500 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10501 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10502 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10503 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10504 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10505 || !(uGuestCR0 & X86_CR0_PG)
10506 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10507 VMX_IGS_EFER_LMA_LME_MISMATCH);
10508 }
10509
10510 /*
10511 * Segment registers.
10512 */
10513 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10514 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10515 if (!(u32Eflags & X86_EFL_VM))
10516 {
10517 /* CS */
10518 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10519 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10520 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10521 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10522 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10523 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10524 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10525 /* CS cannot be loaded with NULL in protected mode. */
10526 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10527 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10528 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10529 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10530 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10531 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10532 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10533 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10534 else
10535 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10536
10537 /* SS */
10538 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10539 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10540 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10541 if ( !(pCtx->cr0 & X86_CR0_PE)
10542 || pCtx->cs.Attr.n.u4Type == 3)
10543 {
10544 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10545 }
10546 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10547 {
10548 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10549 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10550 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10551 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10552 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10553 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10554 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10555 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10556 }
10557
10558 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
10559 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10560 {
10561 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10562 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10563 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10564 || pCtx->ds.Attr.n.u4Type > 11
10565 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10566 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10567 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10568 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10569 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10570 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10571 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10572 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10573 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10574 }
10575 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10576 {
10577 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10578 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10579 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10580 || pCtx->es.Attr.n.u4Type > 11
10581 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10582 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10583 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10584 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10585 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10586 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10587 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10588 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10589 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10590 }
10591 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10592 {
10593 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10594 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10595 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10596 || pCtx->fs.Attr.n.u4Type > 11
10597 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10598 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10599 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10600 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10601 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10602 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10603 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10604 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10605 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10606 }
10607 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10608 {
10609 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10610 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10611 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10612 || pCtx->gs.Attr.n.u4Type > 11
10613 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10614 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10615 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10616 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10617 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10618 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10619 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10620 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10621 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10622 }
10623 /* 64-bit capable CPUs. */
10624#if HC_ARCH_BITS == 64
10625 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10626 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10627 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10628 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10629 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10630 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10631 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10632 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10633 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10634 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10635 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10636#endif
10637 }
10638 else
10639 {
10640 /* V86 mode checks. */
10641 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10642 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10643 {
10644 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10645 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10646 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10647 }
10648 else
10649 {
10650 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10651 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10652 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10653 }
10654
10655 /* CS */
10656 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10657 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10658 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10659 /* SS */
10660 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10661 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10662 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10663 /* DS */
10664 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10665 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10666 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10667 /* ES */
10668 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10669 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10670 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10671 /* FS */
10672 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10673 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10674 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10675 /* GS */
10676 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10677 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10678 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10679 /* 64-bit capable CPUs. */
10680#if HC_ARCH_BITS == 64
10681 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10682 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10683 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10684 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10685 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10686 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10687 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10688 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10689 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10690 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10691 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10692#endif
10693 }
10694
10695 /*
10696 * TR.
10697 */
10698 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10699 /* 64-bit capable CPUs. */
10700#if HC_ARCH_BITS == 64
10701 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10702#endif
10703 if (fLongModeGuest)
10704 {
10705 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10706 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10707 }
10708 else
10709 {
10710 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10711 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10712 VMX_IGS_TR_ATTR_TYPE_INVALID);
10713 }
10714 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10715 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10716 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10717 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10718 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10719 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10720 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10721 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10722
10723 /*
10724 * GDTR and IDTR.
10725 */
10726#if HC_ARCH_BITS == 64
10727 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10728 AssertRCBreak(rc);
10729 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10730
10731 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10732 AssertRCBreak(rc);
10733 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10734#endif
10735
10736 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10737 AssertRCBreak(rc);
10738 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10739
10740 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10741 AssertRCBreak(rc);
10742 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10743
10744 /*
10745 * Guest Non-Register State.
10746 */
10747 /* Activity State. */
10748 uint32_t u32ActivityState;
10749 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10750 AssertRCBreak(rc);
10751 HMVMX_CHECK_BREAK( !u32ActivityState
10752 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
10753 VMX_IGS_ACTIVITY_STATE_INVALID);
10754 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10755 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10756 uint32_t u32IntrState;
10757 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
10758 AssertRCBreak(rc);
10759 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
10760 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10761 {
10762 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10763 }
10764
10765 /** @todo Activity state and injecting interrupts. Left as a todo since we
10766 * currently don't use activity states but ACTIVE. */
10767
10768 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10769 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10770
10771 /* Guest interruptibility-state. */
10772 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10773 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10774 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
10775 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10776 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10777 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10778 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10779 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10780 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10781 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
10782 {
10783 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10784 {
10785 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10786 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10787 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10788 }
10789 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10790 {
10791 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10792 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10793 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10794 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10795 }
10796 }
10797 /** @todo Assumes the processor is not in SMM. */
10798 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10799 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10800 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10801 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10802 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10803 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
10804 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10805 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10806 {
10807 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
10808 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10809 }
10810
10811 /* Pending debug exceptions. */
10812#if HC_ARCH_BITS == 64
10813 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
10814 AssertRCBreak(rc);
10815 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10816 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10817 u32Val = u64Val; /* For pending debug exceptions checks below. */
10818#else
10819 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
10820 AssertRCBreak(rc);
10821 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10822 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10823#endif
10824
10825 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10826 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
10827 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10828 {
10829 if ( (u32Eflags & X86_EFL_TF)
10830 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10831 {
10832 /* Bit 14 is PendingDebug.BS. */
10833 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10834 }
10835 if ( !(u32Eflags & X86_EFL_TF)
10836 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10837 {
10838 /* Bit 14 is PendingDebug.BS. */
10839 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10840 }
10841 }
10842
10843 /* VMCS link pointer. */
10844 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10845 AssertRCBreak(rc);
10846 if (u64Val != UINT64_C(0xffffffffffffffff))
10847 {
10848 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10849 /** @todo Bits beyond the processor's physical-address width MBZ. */
10850 /** @todo 32-bit located in memory referenced by value of this field (as a
10851 * physical address) must contain the processor's VMCS revision ID. */
10852 /** @todo SMM checks. */
10853 }
10854
10855 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10856 * not using Nested Paging? */
10857 if ( pVM->hm.s.fNestedPaging
10858 && !fLongModeGuest
10859 && CPUMIsGuestInPAEModeEx(pCtx))
10860 {
10861 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10862 AssertRCBreak(rc);
10863 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10864
10865 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10866 AssertRCBreak(rc);
10867 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10868
10869 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10870 AssertRCBreak(rc);
10871 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10872
10873 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10874 AssertRCBreak(rc);
10875 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10876 }
10877
10878 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10879 if (uError == VMX_IGS_ERROR)
10880 uError = VMX_IGS_REASON_NOT_FOUND;
10881 } while (0);
10882
10883 pVCpu->hm.s.u32HMError = uError;
10884 return uError;
10885
10886#undef HMVMX_ERROR_BREAK
10887#undef HMVMX_CHECK_BREAK
10888}
10889
10890/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10891/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
10892/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10893
10894/** @name VM-exit handlers.
10895 * @{
10896 */
10897
10898/**
10899 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
10900 */
10901HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10902{
10903 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10904 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
10905 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
10906 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
10907 return VINF_SUCCESS;
10908 return VINF_EM_RAW_INTERRUPT;
10909}
10910
10911
10912/**
10913 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
10914 */
10915HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10916{
10917 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10918 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
10919
10920 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10921 AssertRCReturn(rc, rc);
10922
10923 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10924 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
10925 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
10926 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
10927
10928 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10929 {
10930 /*
10931 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
10932 * injected it ourselves and anything we inject is not going to cause a VM-exit directly
10933 * for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
10934 *
10935 * [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
10936 * [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
10937 */
10938 VMXDispatchHostNmi();
10939 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
10940 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10941 return VINF_SUCCESS;
10942 }
10943
10944 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10945 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10946 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
10947 { /* likely */ }
10948 else
10949 {
10950 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
10951 rcStrictRc1 = VINF_SUCCESS;
10952 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10953 return rcStrictRc1;
10954 }
10955
10956 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
10957 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
10958 switch (uIntType)
10959 {
10960 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
10961 Assert(uVector == X86_XCPT_DB);
10962 RT_FALL_THRU();
10963 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
10964 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
10965 RT_FALL_THRU();
10966 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
10967 {
10968 /*
10969 * If there's any exception caused as a result of event injection, the resulting
10970 * secondary/final execption will be pending, we shall continue guest execution
10971 * after injecting the event. The page-fault case is complicated and we manually
10972 * handle any currently pending event in hmR0VmxExitXcptPF.
10973 */
10974 if (!pVCpu->hm.s.Event.fPending)
10975 { /* likely */ }
10976 else if (uVector != X86_XCPT_PF)
10977 {
10978 rc = VINF_SUCCESS;
10979 break;
10980 }
10981
10982 switch (uVector)
10983 {
10984 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
10985 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
10986 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
10987 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
10988 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
10989 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
10990
10991 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10992 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10993 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
10994 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10995 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
10996 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10997 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
10998 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10999 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11000 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11001 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11002 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11003 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11004 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11005 default:
11006 {
11007 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11008 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11009 {
11010 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11011 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11012 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11013
11014 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
11015 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11016 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11017 AssertRCReturn(rc, rc);
11018 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11019 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11020 0 /* GCPtrFaultAddress */);
11021 }
11022 else
11023 {
11024 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11025 pVCpu->hm.s.u32HMError = uVector;
11026 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11027 }
11028 break;
11029 }
11030 }
11031 break;
11032 }
11033
11034 default:
11035 {
11036 pVCpu->hm.s.u32HMError = uExitIntInfo;
11037 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11038 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11039 break;
11040 }
11041 }
11042 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11043 return rc;
11044}
11045
11046
11047/**
11048 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11049 */
11050HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11051{
11052 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11053
11054 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11055 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11056
11057 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11058 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11059 return VINF_SUCCESS;
11060}
11061
11062
11063/**
11064 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11065 */
11066HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11067{
11068 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11069 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11070 {
11071 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11072 HMVMX_RETURN_UNEXPECTED_EXIT();
11073 }
11074
11075 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11076
11077 /*
11078 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11079 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11080 */
11081 uint32_t fIntrState = 0;
11082 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11083 AssertRCReturn(rc, rc);
11084
11085 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11086 if ( fBlockSti
11087 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11088 {
11089 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11090 }
11091
11092 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11093 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11094
11095 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11096 return VINF_SUCCESS;
11097}
11098
11099
11100/**
11101 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11102 */
11103HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11104{
11105 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11106 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11107}
11108
11109
11110/**
11111 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11112 */
11113HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11114{
11115 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11116 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11117}
11118
11119
11120/**
11121 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11122 */
11123HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11124{
11125 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11126 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
11127
11128 /*
11129 * Get the state we need and update the exit history entry.
11130 */
11131 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11132 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
11133 | CPUMCTX_EXTRN_CS);
11134 AssertRCReturn(rc, rc);
11135
11136 VBOXSTRICTRC rcStrict;
11137 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11138 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
11139 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11140 if (!pExitRec)
11141 {
11142 /*
11143 * Regular CPUID instruction execution.
11144 */
11145 PVM pVM = pVCpu->CTX_SUFF(pVM);
11146 rcStrict = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11147 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11148 {
11149 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11150 Assert(pVmxTransient->cbInstr == 2);
11151 }
11152 else
11153 {
11154 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11155 rcStrict = VERR_EM_INTERPRETER;
11156 }
11157 }
11158 else
11159 {
11160 /*
11161 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
11162 */
11163 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
11164 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11165 AssertRCReturn(rc2, rc2);
11166
11167 Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11168 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11169
11170 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11171 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
11172
11173 Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11174 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11175 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11176 }
11177 return VBOXSTRICTRC_TODO(rcStrict);
11178}
11179
11180
11181/**
11182 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11183 */
11184HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11185{
11186 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11187 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4);
11188 AssertRCReturn(rc, rc);
11189
11190 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11191 return VINF_EM_RAW_EMULATE_INSTR;
11192
11193 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11194 HMVMX_RETURN_UNEXPECTED_EXIT();
11195}
11196
11197
11198/**
11199 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11200 */
11201HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11202{
11203 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11204 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11205 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11206 AssertRCReturn(rc, rc);
11207
11208 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11209 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11210 {
11211 /* If we get a spurious VM-exit when offsetting is enabled,
11212 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11213 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11214 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11215 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
11216 | HM_CHANGED_GUEST_RFLAGS);
11217 }
11218 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11219 {
11220 rcStrict = VINF_SUCCESS;
11221 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11222 }
11223 return rcStrict;
11224}
11225
11226
11227/**
11228 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11229 */
11230HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11231{
11232 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11233 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
11234 | CPUMCTX_EXTRN_TSC_AUX);
11235 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11236 AssertRCReturn(rc, rc);
11237
11238 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11239 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11240 {
11241 /* If we get a spurious VM-exit when offsetting is enabled,
11242 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11243 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11244 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11245 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
11246 | HM_CHANGED_GUEST_RFLAGS);
11247 }
11248 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11249 {
11250 rcStrict = VINF_SUCCESS;
11251 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11252 }
11253 return rcStrict;
11254}
11255
11256
11257/**
11258 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11259 */
11260HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11261{
11262 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11263 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4
11264 | CPUMCTX_EXTRN_CR0
11265 | CPUMCTX_EXTRN_RFLAGS
11266 | CPUMCTX_EXTRN_SS);
11267 AssertRCReturn(rc, rc);
11268
11269 PVM pVM = pVCpu->CTX_SUFF(pVM);
11270 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11271 if (RT_LIKELY(rc == VINF_SUCCESS))
11272 {
11273 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11274 Assert(pVmxTransient->cbInstr == 2);
11275 }
11276 else
11277 {
11278 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11279 rc = VERR_EM_INTERPRETER;
11280 }
11281 return rc;
11282}
11283
11284
11285/**
11286 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11287 */
11288HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11289{
11290 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11291
11292 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11293 if (EMAreHypercallInstructionsEnabled(pVCpu))
11294 {
11295 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
11296 | CPUMCTX_EXTRN_RFLAGS
11297 | CPUMCTX_EXTRN_CR0
11298 | CPUMCTX_EXTRN_SS
11299 | CPUMCTX_EXTRN_CS
11300 | CPUMCTX_EXTRN_EFER);
11301 AssertRCReturn(rc, rc);
11302
11303 /* Perform the hypercall. */
11304 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11305 if (rcStrict == VINF_SUCCESS)
11306 {
11307 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11308 AssertRCReturn(rc, rc);
11309 }
11310 else
11311 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11312 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11313 || RT_FAILURE(rcStrict));
11314
11315 /* If the hypercall changes anything other than guest's general-purpose registers,
11316 we would need to reload the guest changed bits here before VM-entry. */
11317 }
11318 else
11319 Log4Func(("Hypercalls not enabled\n"));
11320
11321 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11322 if (RT_FAILURE(rcStrict))
11323 {
11324 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11325 rcStrict = VINF_SUCCESS;
11326 }
11327
11328 return rcStrict;
11329}
11330
11331
11332/**
11333 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11334 */
11335HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11336{
11337 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11338 PVM pVM = pVCpu->CTX_SUFF(pVM);
11339 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11340
11341 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11342 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK);
11343 AssertRCReturn(rc, rc);
11344
11345 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11346 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11347 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11348 else
11349 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11350 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11351 return rcStrict;
11352}
11353
11354
11355/**
11356 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11357 */
11358HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11359{
11360 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11361 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11362 | CPUMCTX_EXTRN_RFLAGS
11363 | CPUMCTX_EXTRN_SS);
11364 AssertRCReturn(rc, rc);
11365
11366 PVM pVM = pVCpu->CTX_SUFF(pVM);
11367 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11368 if (RT_LIKELY(rc == VINF_SUCCESS))
11369 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11370 else
11371 {
11372 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11373 rc = VERR_EM_INTERPRETER;
11374 }
11375 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11376 return rc;
11377}
11378
11379
11380/**
11381 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11382 */
11383HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11384{
11385 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11386 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11387 | CPUMCTX_EXTRN_RFLAGS
11388 | CPUMCTX_EXTRN_SS);
11389 AssertRCReturn(rc, rc);
11390
11391 PVM pVM = pVCpu->CTX_SUFF(pVM);
11392 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11393 rc = VBOXSTRICTRC_VAL(rc2);
11394 if (RT_LIKELY( rc == VINF_SUCCESS
11395 || rc == VINF_EM_HALT))
11396 {
11397 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11398 AssertRCReturn(rc3, rc3);
11399
11400 if ( rc == VINF_EM_HALT
11401 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11402 rc = VINF_SUCCESS;
11403 }
11404 else
11405 {
11406 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11407 rc = VERR_EM_INTERPRETER;
11408 }
11409 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11410 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11411 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11412 return rc;
11413}
11414
11415
11416/**
11417 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11418 */
11419HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11420{
11421 /*
11422 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
11423 * mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
11424 * treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
11425 * VMX root operation. If we get here, something funny is going on.
11426 *
11427 * See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
11428 */
11429 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11430 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11431 HMVMX_RETURN_UNEXPECTED_EXIT();
11432}
11433
11434
11435/**
11436 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11437 */
11438HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11439{
11440 /*
11441 * This can only happen if we support dual-monitor treatment of SMI, which can be activated
11442 * by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
11443 * this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
11444 * an SMI. If we get here, something funny is going on.
11445 *
11446 * See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11447 * See Intel spec. 25.3 "Other Causes of VM-Exits"
11448 */
11449 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11450 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11451 HMVMX_RETURN_UNEXPECTED_EXIT();
11452}
11453
11454
11455/**
11456 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11457 */
11458HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11459{
11460 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11461 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11462 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11463 HMVMX_RETURN_UNEXPECTED_EXIT();
11464}
11465
11466
11467/**
11468 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11469 */
11470HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11471{
11472 /*
11473 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
11474 * We don't make use of it as our guests don't have direct access to the host LAPIC.
11475 * See Intel spec. 25.3 "Other Causes of VM-exits".
11476 */
11477 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11478 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11479 HMVMX_RETURN_UNEXPECTED_EXIT();
11480}
11481
11482
11483/**
11484 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11485 * VM-exit.
11486 */
11487HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11488{
11489 /*
11490 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11491 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11492 *
11493 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11494 * See Intel spec. "23.8 Restrictions on VMX operation".
11495 */
11496 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11497 return VINF_SUCCESS;
11498}
11499
11500
11501/**
11502 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11503 * VM-exit.
11504 */
11505HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11506{
11507 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11508 return VINF_EM_RESET;
11509}
11510
11511
11512/**
11513 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11514 */
11515HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11516{
11517 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11518 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11519
11520 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11521 AssertRCReturn(rc, rc);
11522
11523 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11524 rc = VINF_SUCCESS;
11525 else
11526 rc = VINF_EM_HALT;
11527
11528 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11529 if (rc != VINF_SUCCESS)
11530 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11531 return rc;
11532}
11533
11534
11535/**
11536 * VM-exit handler for instructions that result in a \#UD exception delivered to
11537 * the guest.
11538 */
11539HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11540{
11541 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11542 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11543 return VINF_SUCCESS;
11544}
11545
11546
11547/**
11548 * VM-exit handler for expiry of the VMX preemption timer.
11549 */
11550HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11551{
11552 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11553
11554 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11555 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11556
11557 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11558 PVM pVM = pVCpu->CTX_SUFF(pVM);
11559 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11560 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11561 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11562}
11563
11564
11565/**
11566 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11567 */
11568HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11569{
11570 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11571
11572 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11573 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
11574 | CPUMCTX_EXTRN_CR4);
11575 AssertRCReturn(rc, rc);
11576
11577 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11578 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11579 : HM_CHANGED_XCPT_RAISED_MASK);
11580
11581 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11582
11583 return rcStrict;
11584}
11585
11586
11587/**
11588 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11589 */
11590HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11591{
11592 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11593 /** @todo Use VM-exit instruction information. */
11594 return VERR_EM_INTERPRETER;
11595}
11596
11597
11598/**
11599 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11600 * Error VM-exit.
11601 */
11602HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11603{
11604 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11605 rc |= hmR0VmxCheckVmcsCtls(pVCpu);
11606 AssertRCReturn(rc, rc);
11607
11608 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11609 NOREF(uInvalidReason);
11610
11611#ifdef VBOX_STRICT
11612 uint32_t fIntrState;
11613 RTHCUINTREG uHCReg;
11614 uint64_t u64Val;
11615 uint32_t u32Val;
11616
11617 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11618 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11619 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11620 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11621 AssertRCReturn(rc, rc);
11622
11623 Log4(("uInvalidReason %u\n", uInvalidReason));
11624 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11625 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11626 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11627 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", fIntrState));
11628
11629 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11630 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11631 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11632 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11633 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11634 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11635 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11636 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11637 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11638 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11639 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11640 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11641#else
11642 NOREF(pVmxTransient);
11643#endif
11644
11645 hmR0DumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11646 return VERR_VMX_INVALID_GUEST_STATE;
11647}
11648
11649
11650/**
11651 * VM-exit handler for VM-entry failure due to an MSR-load
11652 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11653 */
11654HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11655{
11656 NOREF(pVmxTransient);
11657 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11658 HMVMX_RETURN_UNEXPECTED_EXIT();
11659}
11660
11661
11662/**
11663 * VM-exit handler for VM-entry failure due to a machine-check event
11664 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11665 */
11666HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11667{
11668 NOREF(pVmxTransient);
11669 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11670 HMVMX_RETURN_UNEXPECTED_EXIT();
11671}
11672
11673
11674/**
11675 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11676 * theory.
11677 */
11678HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11679{
11680 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
11681 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
11682 return VERR_VMX_UNDEFINED_EXIT_CODE;
11683}
11684
11685
11686/**
11687 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11688 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11689 * Conditional VM-exit.
11690 */
11691HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11692{
11693 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11694
11695 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
11696 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11697 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
11698 return VERR_EM_INTERPRETER;
11699 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11700 HMVMX_RETURN_UNEXPECTED_EXIT();
11701}
11702
11703
11704/**
11705 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11706 */
11707HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11708{
11709 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11710
11711 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
11712 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
11713 return VERR_EM_INTERPRETER;
11714 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11715 HMVMX_RETURN_UNEXPECTED_EXIT();
11716}
11717
11718
11719/**
11720 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11721 */
11722HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11723{
11724 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11725
11726 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
11727 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11728 | CPUMCTX_EXTRN_RFLAGS
11729 | CPUMCTX_EXTRN_SS);
11730 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11731 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
11732 AssertRCReturn(rc, rc);
11733 Log4Func(("ecx=%#RX32\n", pMixedCtx->ecx));
11734
11735#ifdef VBOX_STRICT
11736 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
11737 {
11738 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
11739 && pMixedCtx->ecx != MSR_K6_EFER)
11740 {
11741 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11742 pMixedCtx->ecx));
11743 HMVMX_RETURN_UNEXPECTED_EXIT();
11744 }
11745 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11746 {
11747 VMXMSREXITREAD enmRead;
11748 VMXMSREXITWRITE enmWrite;
11749 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
11750 AssertRCReturn(rc2, rc2);
11751 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11752 {
11753 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11754 HMVMX_RETURN_UNEXPECTED_EXIT();
11755 }
11756 }
11757 }
11758#endif
11759
11760 PVM pVM = pVCpu->CTX_SUFF(pVM);
11761 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11762 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
11763 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
11764 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11765 if (RT_SUCCESS(rc))
11766 {
11767 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11768 Assert(pVmxTransient->cbInstr == 2);
11769 }
11770 return rc;
11771}
11772
11773
11774/**
11775 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11776 */
11777HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11778{
11779 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11780 PVM pVM = pVCpu->CTX_SUFF(pVM);
11781 int rc = VINF_SUCCESS;
11782
11783 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
11784 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11785 | CPUMCTX_EXTRN_RFLAGS
11786 | CPUMCTX_EXTRN_SS);
11787 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11788 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
11789 AssertRCReturn(rc, rc);
11790 Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
11791
11792 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11793 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
11794 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
11795
11796 if (RT_SUCCESS(rc))
11797 {
11798 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11799
11800 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
11801 if ( pMixedCtx->ecx == MSR_IA32_APICBASE
11802 || ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
11803 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
11804 {
11805 /*
11806 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
11807 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
11808 * EMInterpretWrmsr() changes it.
11809 */
11810 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11811 }
11812 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
11813 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11814 else if (pMixedCtx->ecx == MSR_K6_EFER)
11815 {
11816 /*
11817 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
11818 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
11819 * the other bits as well, SCE and NXE. See @bugref{7368}.
11820 */
11821 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
11822 | HM_CHANGED_VMX_ENTRY_CTLS
11823 | HM_CHANGED_VMX_EXIT_CTLS);
11824 }
11825
11826 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
11827 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11828 {
11829 switch (pMixedCtx->ecx)
11830 {
11831 case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
11832 case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
11833 case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
11834 case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
11835 case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
11836 case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
11837 default:
11838 {
11839 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11840 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
11841 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11842 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
11843 break;
11844 }
11845 }
11846 }
11847#ifdef VBOX_STRICT
11848 else
11849 {
11850 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
11851 switch (pMixedCtx->ecx)
11852 {
11853 case MSR_IA32_SYSENTER_CS:
11854 case MSR_IA32_SYSENTER_EIP:
11855 case MSR_IA32_SYSENTER_ESP:
11856 case MSR_K8_FS_BASE:
11857 case MSR_K8_GS_BASE:
11858 {
11859 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
11860 HMVMX_RETURN_UNEXPECTED_EXIT();
11861 }
11862
11863 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
11864 default:
11865 {
11866 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11867 {
11868 /* EFER writes are always intercepted, see hmR0VmxExportGuestMsrs(). */
11869 if (pMixedCtx->ecx != MSR_K6_EFER)
11870 {
11871 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11872 pMixedCtx->ecx));
11873 HMVMX_RETURN_UNEXPECTED_EXIT();
11874 }
11875 }
11876
11877 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11878 {
11879 VMXMSREXITREAD enmRead;
11880 VMXMSREXITWRITE enmWrite;
11881 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
11882 AssertRCReturn(rc2, rc2);
11883 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
11884 {
11885 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11886 HMVMX_RETURN_UNEXPECTED_EXIT();
11887 }
11888 }
11889 break;
11890 }
11891 }
11892 }
11893#endif /* VBOX_STRICT */
11894 }
11895 return rc;
11896}
11897
11898
11899/**
11900 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
11901 */
11902HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11903{
11904 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11905 /** @todo The guest has likely hit a contended spinlock. We might want to
11906 * poke a schedule different guest VCPU. */
11907 return VINF_EM_RAW_INTERRUPT;
11908}
11909
11910
11911/**
11912 * VM-exit handler for when the TPR value is lowered below the specified
11913 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
11914 */
11915HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11916{
11917 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11918 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
11919
11920 /*
11921 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
11922 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
11923 */
11924 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
11925 return VINF_SUCCESS;
11926}
11927
11928
11929/**
11930 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
11931 * VM-exit.
11932 *
11933 * @retval VINF_SUCCESS when guest execution can continue.
11934 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
11935 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
11936 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
11937 * interpreter.
11938 */
11939HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11940{
11941 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11942 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
11943
11944 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11945 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11946 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11947 AssertRCReturn(rc, rc);
11948
11949 VBOXSTRICTRC rcStrict;
11950 PVM pVM = pVCpu->CTX_SUFF(pVM);
11951 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
11952 uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQualification);
11953 switch (uAccessType)
11954 {
11955 case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
11956 {
11957 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
11958 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
11959 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification));
11960 AssertMsg( rcStrict == VINF_SUCCESS
11961 || rcStrict == VINF_IEM_RAISED_XCPT
11962 || rcStrict == VINF_PGM_CHANGE_MODE
11963 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11964
11965 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
11966 {
11967 case 0:
11968 {
11969 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11970 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
11971 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
11972 break;
11973 }
11974
11975 case 2:
11976 {
11977 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
11978 /* Nothing to do here, CR2 it's not part of the VMCS. */
11979 break;
11980 }
11981
11982 case 3:
11983 {
11984 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
11985 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
11986 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR3);
11987 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
11988 break;
11989 }
11990
11991 case 4:
11992 {
11993 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
11994 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11995 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
11996 pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
11997 break;
11998 }
11999
12000 case 8:
12001 {
12002 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
12003 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12004 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
12005 break;
12006 }
12007 default:
12008 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification)));
12009 break;
12010 }
12011 break;
12012 }
12013
12014 case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
12015 {
12016 Assert( !pVM->hm.s.fNestedPaging
12017 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12018 || pVCpu->hm.s.fUsingDebugLoop
12019 || VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 3);
12020 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12021 Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 8
12022 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12023
12024 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12025 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification),
12026 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification));
12027 AssertMsg( rcStrict == VINF_SUCCESS
12028 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12029#ifdef VBOX_WITH_STATISTICS
12030 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
12031 {
12032 case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
12033 case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
12034 case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
12035 case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
12036 case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
12037 }
12038#endif
12039 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
12040 VBOXSTRICTRC_VAL(rcStrict)));
12041 if (VMX_EXIT_QUAL_CRX_GENREG(uExitQualification) == X86_GREG_xSP)
12042 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RSP);
12043 break;
12044 }
12045
12046 case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12047 {
12048 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12049 AssertMsg( rcStrict == VINF_SUCCESS
12050 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12051
12052 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12053 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12054 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12055 break;
12056 }
12057
12058 case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12059 {
12060 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12061 VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQualification));
12062 AssertMsg( rcStrict == VINF_SUCCESS
12063 || rcStrict == VINF_IEM_RAISED_XCPT
12064 || rcStrict == VINF_PGM_CHANGE_MODE,
12065 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12066
12067 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12068 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12069 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12070 break;
12071 }
12072
12073 default:
12074 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12075 VERR_VMX_UNEXPECTED_EXCEPTION);
12076 }
12077
12078 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
12079 : HM_CHANGED_XCPT_RAISED_MASK);
12080 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12081 NOREF(pVM);
12082 return rcStrict;
12083}
12084
12085
12086/**
12087 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12088 * VM-exit.
12089 */
12090HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12091{
12092 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12093 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12094 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
12095
12096 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12097 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12098 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
12099 | CPUMCTX_EXTRN_SREG_MASK
12100 | CPUMCTX_EXTRN_EFER);
12101 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12102 AssertRCReturn(rc, rc);
12103
12104 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12105 uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQualification);
12106 uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQualification);
12107 bool fIOWrite = ( VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQualification)
12108 == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12109 bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQualification);
12110 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12111 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12112 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12113
12114 /*
12115 * Update exit history to see if this exit can be optimized.
12116 */
12117 VBOXSTRICTRC rcStrict;
12118 PCEMEXITREC pExitRec = NULL;
12119 if ( !fGstStepping
12120 && !fDbgStepping)
12121 pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12122 !fIOString
12123 ? !fIOWrite
12124 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12125 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12126 : !fIOWrite
12127 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12128 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12129 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12130 if (!pExitRec)
12131 {
12132 /* I/O operation lookup arrays. */
12133 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12134 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12135 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12136 uint32_t const cbInstr = pVmxTransient->cbInstr;
12137 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12138 PVM pVM = pVCpu->CTX_SUFF(pVM);
12139 if (fIOString)
12140 {
12141 /*
12142 * INS/OUTS - I/O String instruction.
12143 *
12144 * Use instruction-information if available, otherwise fall back on
12145 * interpreting the instruction.
12146 */
12147 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12148 fIOWrite ? 'w' : 'r'));
12149 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12150 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12151 {
12152 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12153 AssertRCReturn(rc2, rc2);
12154 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12155 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12156 IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12157 bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification);
12158 if (fIOWrite)
12159 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12160 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12161 else
12162 {
12163 /*
12164 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12165 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12166 * See Intel Instruction spec. for "INS".
12167 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12168 */
12169 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12170 }
12171 }
12172 else
12173 rcStrict = IEMExecOne(pVCpu);
12174
12175 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12176 fUpdateRipAlready = true;
12177 }
12178 else
12179 {
12180 /*
12181 * IN/OUT - I/O instruction.
12182 */
12183 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12184 fIOWrite ? 'w' : 'r'));
12185 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12186 Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification));
12187 if (fIOWrite)
12188 {
12189 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12190 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12191 }
12192 else
12193 {
12194 uint32_t u32Result = 0;
12195 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12196 if (IOM_SUCCESS(rcStrict))
12197 {
12198 /* Save result of I/O IN instr. in AL/AX/EAX. */
12199 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12200 }
12201 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12202 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12203 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12204 }
12205 }
12206
12207 if (IOM_SUCCESS(rcStrict))
12208 {
12209 if (!fUpdateRipAlready)
12210 {
12211 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12212 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12213 }
12214
12215 /*
12216 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12217 * while booting Fedora 17 64-bit guest.
12218 *
12219 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12220 */
12221 if (fIOString)
12222 {
12223 /** @todo Single-step for INS/OUTS with REP prefix? */
12224 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12225 }
12226 else if ( !fDbgStepping
12227 && fGstStepping)
12228 {
12229 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
12230 AssertRCReturn(rc, rc);
12231 }
12232
12233 /*
12234 * If any I/O breakpoints are armed, we need to check if one triggered
12235 * and take appropriate action.
12236 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12237 */
12238 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12239 AssertRCReturn(rc, rc);
12240
12241 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12242 * execution engines about whether hyper BPs and such are pending. */
12243 uint32_t const uDr7 = pMixedCtx->dr[7];
12244 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12245 && X86_DR7_ANY_RW_IO(uDr7)
12246 && (pMixedCtx->cr4 & X86_CR4_DE))
12247 || DBGFBpIsHwIoArmed(pVM)))
12248 {
12249 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12250
12251 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12252 VMMRZCallRing3Disable(pVCpu);
12253 HM_DISABLE_PREEMPT();
12254
12255 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12256
12257 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12258 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12259 {
12260 /* Raise #DB. */
12261 if (fIsGuestDbgActive)
12262 ASMSetDR6(pMixedCtx->dr[6]);
12263 if (pMixedCtx->dr[7] != uDr7)
12264 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_DR7;
12265
12266 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12267 }
12268 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12269 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12270 else if ( rcStrict2 != VINF_SUCCESS
12271 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12272 rcStrict = rcStrict2;
12273 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12274
12275 HM_RESTORE_PREEMPT();
12276 VMMRZCallRing3Enable(pVCpu);
12277 }
12278 }
12279
12280#ifdef VBOX_STRICT
12281 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12282 Assert(!fIOWrite);
12283 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12284 Assert(fIOWrite);
12285 else
12286 {
12287# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12288 * statuses, that the VMM device and some others may return. See
12289 * IOM_SUCCESS() for guidance. */
12290 AssertMsg( RT_FAILURE(rcStrict)
12291 || rcStrict == VINF_SUCCESS
12292 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12293 || rcStrict == VINF_EM_DBG_BREAKPOINT
12294 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12295 || rcStrict == VINF_EM_RAW_TO_R3
12296 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12297# endif
12298 }
12299#endif
12300 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12301 }
12302 else
12303 {
12304 /*
12305 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12306 */
12307 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12308 AssertRCReturn(rc2, rc2);
12309 STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
12310 : fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
12311 Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12312 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12313 VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification) ? "REP " : "",
12314 fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
12315
12316 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12317 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12318
12319 Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12320 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12321 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12322 }
12323 return rcStrict;
12324}
12325
12326
12327/**
12328 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12329 * VM-exit.
12330 */
12331HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12332{
12333 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12334
12335 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12336 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12337 AssertRCReturn(rc, rc);
12338 if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12339 {
12340 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12341 AssertRCReturn(rc, rc);
12342 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12343 {
12344 uint32_t uErrCode;
12345 RTGCUINTPTR GCPtrFaultAddress;
12346 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12347 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12348 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12349 if (fErrorCodeValid)
12350 {
12351 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12352 AssertRCReturn(rc, rc);
12353 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12354 }
12355 else
12356 uErrCode = 0;
12357
12358 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12359 && uVector == X86_XCPT_PF)
12360 GCPtrFaultAddress = pMixedCtx->cr2;
12361 else
12362 GCPtrFaultAddress = 0;
12363
12364 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12365 0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
12366
12367 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12368 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12369 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12370 }
12371 }
12372
12373 /* Fall back to the interpreter to emulate the task-switch. */
12374 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12375 return VERR_EM_INTERPRETER;
12376}
12377
12378
12379/**
12380 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12381 */
12382HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12383{
12384 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12385 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12386 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12387 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12388 AssertRCReturn(rc, rc);
12389 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12390 return VINF_EM_DBG_STEPPED;
12391}
12392
12393
12394/**
12395 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12396 */
12397HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12398{
12399 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12400
12401 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12402
12403 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12404 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12405 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12406 {
12407 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12408 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12409 {
12410 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12411 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12412 }
12413 }
12414 else
12415 {
12416 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12417 rcStrict1 = VINF_SUCCESS;
12418 return rcStrict1;
12419 }
12420
12421 /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12422 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12423 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12424 AssertRCReturn(rc, rc);
12425
12426 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12427 uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12428 VBOXSTRICTRC rcStrict2;
12429 switch (uAccessType)
12430 {
12431 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12432 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12433 {
12434 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12435 || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
12436 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12437
12438 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12439 GCPhys &= PAGE_BASE_GC_MASK;
12440 GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12441 PVM pVM = pVCpu->CTX_SUFF(pVM);
12442 Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12443 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12444
12445 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12446 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12447 CPUMCTX2CORE(pMixedCtx), GCPhys);
12448 Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12449 if ( rcStrict2 == VINF_SUCCESS
12450 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12451 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12452 {
12453 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12454 | HM_CHANGED_GUEST_RSP
12455 | HM_CHANGED_GUEST_RFLAGS
12456 | HM_CHANGED_GUEST_APIC_TPR);
12457 rcStrict2 = VINF_SUCCESS;
12458 }
12459 break;
12460 }
12461
12462 default:
12463 Log4Func(("uAccessType=%#x\n", uAccessType));
12464 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12465 break;
12466 }
12467
12468 if (rcStrict2 != VINF_SUCCESS)
12469 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12470 return rcStrict2;
12471}
12472
12473
12474/**
12475 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12476 * VM-exit.
12477 */
12478HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12479{
12480 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12481
12482 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12483 if (pVmxTransient->fWasGuestDebugStateActive)
12484 {
12485 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12486 HMVMX_RETURN_UNEXPECTED_EXIT();
12487 }
12488
12489 if ( !pVCpu->hm.s.fSingleInstruction
12490 && !pVmxTransient->fWasHyperDebugStateActive)
12491 {
12492 Assert(!DBGFIsStepping(pVCpu));
12493 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12494
12495 /* Don't intercept MOV DRx any more. */
12496 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12497 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12498 AssertRCReturn(rc, rc);
12499
12500 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12501 VMMRZCallRing3Disable(pVCpu);
12502 HM_DISABLE_PREEMPT();
12503
12504 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12505 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12506 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12507
12508 HM_RESTORE_PREEMPT();
12509 VMMRZCallRing3Enable(pVCpu);
12510
12511#ifdef VBOX_WITH_STATISTICS
12512 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12513 AssertRCReturn(rc, rc);
12514 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12515 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12516 else
12517 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12518#endif
12519 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12520 return VINF_SUCCESS;
12521 }
12522
12523 /*
12524 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12525 * Update the segment registers and DR7 from the CPU.
12526 */
12527 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12528 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK
12529 | CPUMCTX_EXTRN_DR7);
12530 AssertRCReturn(rc, rc);
12531 Log4Func(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12532
12533 PVM pVM = pVCpu->CTX_SUFF(pVM);
12534 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12535 {
12536 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12537 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification),
12538 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification));
12539 if (RT_SUCCESS(rc))
12540 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12541 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12542 }
12543 else
12544 {
12545 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12546 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification),
12547 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification));
12548 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12549 }
12550
12551 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12552 if (RT_SUCCESS(rc))
12553 {
12554 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12555 AssertRCReturn(rc2, rc2);
12556 return VINF_SUCCESS;
12557 }
12558 return rc;
12559}
12560
12561
12562/**
12563 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12564 * Conditional VM-exit.
12565 */
12566HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12567{
12568 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12569 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12570
12571 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12572 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12573 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12574 {
12575 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12576 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12577 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12578 {
12579 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12580 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12581 }
12582 }
12583 else
12584 {
12585 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12586 rcStrict1 = VINF_SUCCESS;
12587 return rcStrict1;
12588 }
12589
12590 /*
12591 * Get sufficent state and update the exit history entry.
12592 */
12593 RTGCPHYS GCPhys;
12594 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12595 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12596 AssertRCReturn(rc, rc);
12597
12598 VBOXSTRICTRC rcStrict;
12599 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12600 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
12601 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12602 if (!pExitRec)
12603 {
12604 /*
12605 * If we succeed, resume guest execution.
12606 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12607 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12608 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12609 * weird case. See @bugref{6043}.
12610 */
12611 PVM pVM = pVCpu->CTX_SUFF(pVM);
12612 rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12613 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12614 if ( rcStrict == VINF_SUCCESS
12615 || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12616 || rcStrict == VERR_PAGE_NOT_PRESENT)
12617 {
12618 /* Successfully handled MMIO operation. */
12619 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12620 | HM_CHANGED_GUEST_RSP
12621 | HM_CHANGED_GUEST_RFLAGS
12622 | HM_CHANGED_GUEST_APIC_TPR);
12623 rcStrict = VINF_SUCCESS;
12624 }
12625 }
12626 else
12627 {
12628 /*
12629 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12630 */
12631 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
12632 int rc2 = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12633 AssertRCReturn(rc2, rc2);
12634
12635 Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12636 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12637
12638 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12639 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12640
12641 Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12642 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12643 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12644 }
12645 return VBOXSTRICTRC_TODO(rcStrict);
12646}
12647
12648
12649/**
12650 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12651 * VM-exit.
12652 */
12653HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12654{
12655 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12656 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12657
12658 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12659 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12660 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12661 {
12662 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12663 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12664 Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12665 }
12666 else
12667 {
12668 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12669 rcStrict1 = VINF_SUCCESS;
12670 return rcStrict1;
12671 }
12672
12673 RTGCPHYS GCPhys;
12674 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12675 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12676 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12677 AssertRCReturn(rc, rc);
12678
12679 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12680 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12681
12682 RTGCUINT uErrorCode = 0;
12683 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
12684 uErrorCode |= X86_TRAP_PF_ID;
12685 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_DATA_WRITE)
12686 uErrorCode |= X86_TRAP_PF_RW;
12687 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
12688 uErrorCode |= X86_TRAP_PF_P;
12689
12690 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12691
12692 Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12693 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12694
12695 /* Handle the pagefault trap for the nested shadow table. */
12696 PVM pVM = pVCpu->CTX_SUFF(pVM);
12697 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12698 TRPMResetTrap(pVCpu);
12699
12700 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12701 if ( rcStrict2 == VINF_SUCCESS
12702 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12703 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12704 {
12705 /* Successfully synced our nested page tables. */
12706 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12707 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12708 | HM_CHANGED_GUEST_RSP
12709 | HM_CHANGED_GUEST_RFLAGS);
12710 return VINF_SUCCESS;
12711 }
12712
12713 Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12714 return rcStrict2;
12715}
12716
12717/** @} */
12718
12719/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12720/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
12721/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12722
12723/** @name VM-exit exception handlers.
12724 * @{
12725 */
12726
12727/**
12728 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12729 */
12730static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12731{
12732 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12733 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12734
12735 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
12736 AssertRCReturn(rc, rc);
12737
12738 if (!(pMixedCtx->cr0 & X86_CR0_NE))
12739 {
12740 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12741 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12742
12743 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12744 * provides VM-exit instruction length. If this causes problem later,
12745 * disassemble the instruction like it's done on AMD-V. */
12746 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12747 AssertRCReturn(rc2, rc2);
12748 return rc;
12749 }
12750
12751 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12752 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12753 return rc;
12754}
12755
12756
12757/**
12758 * VM-exit exception handler for \#BP (Breakpoint exception).
12759 */
12760static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12761{
12762 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12763 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12764
12765 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12766 AssertRCReturn(rc, rc);
12767
12768 rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx));
12769 if (rc == VINF_EM_RAW_GUEST_TRAP)
12770 {
12771 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12772 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12773 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12774 AssertRCReturn(rc, rc);
12775
12776 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12777 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12778 }
12779
12780 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
12781 return rc;
12782}
12783
12784
12785/**
12786 * VM-exit exception handler for \#AC (alignment check exception).
12787 */
12788static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12789{
12790 RT_NOREF_PV(pMixedCtx);
12791 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12792
12793 /*
12794 * Re-inject it. We'll detect any nesting before getting here.
12795 */
12796 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12797 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12798 AssertRCReturn(rc, rc);
12799 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
12800
12801 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12802 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12803 return VINF_SUCCESS;
12804}
12805
12806
12807/**
12808 * VM-exit exception handler for \#DB (Debug exception).
12809 */
12810static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12811{
12812 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12813 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
12814
12815 /*
12816 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
12817 * for processing.
12818 */
12819 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12820
12821 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
12822 uint64_t uDR6 = X86_DR6_INIT_VAL;
12823 uDR6 |= ( pVmxTransient->uExitQualification
12824 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
12825
12826 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
12827 Log6Func(("rc=%Rrc\n", rc));
12828 if (rc == VINF_EM_RAW_GUEST_TRAP)
12829 {
12830 /*
12831 * The exception was for the guest. Update DR6, DR7.GD and
12832 * IA32_DEBUGCTL.LBR before forwarding it.
12833 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
12834 */
12835 VMMRZCallRing3Disable(pVCpu);
12836 HM_DISABLE_PREEMPT();
12837
12838 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
12839 pMixedCtx->dr[6] |= uDR6;
12840 if (CPUMIsGuestDebugStateActive(pVCpu))
12841 ASMSetDR6(pMixedCtx->dr[6]);
12842
12843 HM_RESTORE_PREEMPT();
12844 VMMRZCallRing3Enable(pVCpu);
12845
12846 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12847 AssertRCReturn(rc, rc);
12848
12849 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
12850 pMixedCtx->dr[7] &= ~X86_DR7_GD;
12851
12852 /* Paranoia. */
12853 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
12854 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
12855
12856 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
12857 AssertRCReturn(rc, rc);
12858
12859 /*
12860 * Raise #DB in the guest.
12861 *
12862 * It is important to reflect exactly what the VM-exit gave us (preserving the
12863 * interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
12864 * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
12865 * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
12866 *
12867 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
12868 * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
12869 */
12870 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12871 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12872 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12873 AssertRCReturn(rc, rc);
12874 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12875 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12876 return VINF_SUCCESS;
12877 }
12878
12879 /*
12880 * Not a guest trap, must be a hypervisor related debug event then.
12881 * Update DR6 in case someone is interested in it.
12882 */
12883 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
12884 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
12885 CPUMSetHyperDR6(pVCpu, uDR6);
12886
12887 return rc;
12888}
12889
12890/**
12891 * VM-exit exception handler for \#GP (General-protection exception).
12892 *
12893 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
12894 */
12895static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12896{
12897 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12898 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
12899
12900 int rc;
12901 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
12902 { /* likely */ }
12903 else
12904 {
12905#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12906 Assert(pVCpu->hm.s.fUsingDebugLoop);
12907#endif
12908 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
12909 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12910 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12911 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12912 rc |= hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12913 AssertRCReturn(rc, rc);
12914 Log4Func(("Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
12915 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
12916 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12917 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12918 return rc;
12919 }
12920
12921 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
12922 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
12923
12924 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
12925 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12926 AssertRCReturn(rc, rc);
12927
12928 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12929 uint32_t cbOp = 0;
12930 PVM pVM = pVCpu->CTX_SUFF(pVM);
12931 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12932 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
12933 if (RT_SUCCESS(rc))
12934 {
12935 rc = VINF_SUCCESS;
12936 Assert(cbOp == pDis->cbInstr);
12937 Log4Func(("Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12938 switch (pDis->pCurInstr->uOpcode)
12939 {
12940 case OP_CLI:
12941 {
12942 pMixedCtx->eflags.Bits.u1IF = 0;
12943 pMixedCtx->eflags.Bits.u1RF = 0;
12944 pMixedCtx->rip += pDis->cbInstr;
12945 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12946 if ( !fDbgStepping
12947 && pMixedCtx->eflags.Bits.u1TF)
12948 {
12949 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
12950 AssertRCReturn(rc, rc);
12951 }
12952 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
12953 break;
12954 }
12955
12956 case OP_STI:
12957 {
12958 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
12959 pMixedCtx->eflags.Bits.u1IF = 1;
12960 pMixedCtx->eflags.Bits.u1RF = 0;
12961 pMixedCtx->rip += pDis->cbInstr;
12962 if (!fOldIF)
12963 {
12964 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
12965 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
12966 }
12967 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12968 if ( !fDbgStepping
12969 && pMixedCtx->eflags.Bits.u1TF)
12970 {
12971 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
12972 AssertRCReturn(rc, rc);
12973 }
12974 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
12975 break;
12976 }
12977
12978 case OP_HLT:
12979 {
12980 rc = VINF_EM_HALT;
12981 pMixedCtx->rip += pDis->cbInstr;
12982 pMixedCtx->eflags.Bits.u1RF = 0;
12983 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12984 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
12985 break;
12986 }
12987
12988 case OP_POPF:
12989 {
12990 Log4Func(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12991 uint32_t cbParm;
12992 uint32_t uMask;
12993 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12994 if (pDis->fPrefix & DISPREFIX_OPSIZE)
12995 {
12996 cbParm = 4;
12997 uMask = 0xffffffff;
12998 }
12999 else
13000 {
13001 cbParm = 2;
13002 uMask = 0xffff;
13003 }
13004
13005 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13006 RTGCPTR GCPtrStack = 0;
13007 X86EFLAGS Eflags;
13008 Eflags.u32 = 0;
13009 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13010 &GCPtrStack);
13011 if (RT_SUCCESS(rc))
13012 {
13013 Assert(sizeof(Eflags.u32) >= cbParm);
13014 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13015 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13016 }
13017 if (RT_FAILURE(rc))
13018 {
13019 rc = VERR_EM_INTERPRETER;
13020 break;
13021 }
13022 Log4Func(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13023 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13024 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13025 pMixedCtx->esp += cbParm;
13026 pMixedCtx->esp &= uMask;
13027 pMixedCtx->rip += pDis->cbInstr;
13028 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13029 | HM_CHANGED_GUEST_RSP
13030 | HM_CHANGED_GUEST_RFLAGS);
13031 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13032 POPF restores EFLAGS.TF. */
13033 if ( !fDbgStepping
13034 && fGstStepping)
13035 {
13036 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13037 AssertRCReturn(rc, rc);
13038 }
13039 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13040 break;
13041 }
13042
13043 case OP_PUSHF:
13044 {
13045 uint32_t cbParm;
13046 uint32_t uMask;
13047 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13048 {
13049 cbParm = 4;
13050 uMask = 0xffffffff;
13051 }
13052 else
13053 {
13054 cbParm = 2;
13055 uMask = 0xffff;
13056 }
13057
13058 /* Get the stack pointer & push the contents of eflags onto the stack. */
13059 RTGCPTR GCPtrStack = 0;
13060 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13061 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13062 if (RT_FAILURE(rc))
13063 {
13064 rc = VERR_EM_INTERPRETER;
13065 break;
13066 }
13067 X86EFLAGS Eflags = pMixedCtx->eflags;
13068 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13069 Eflags.Bits.u1RF = 0;
13070 Eflags.Bits.u1VM = 0;
13071
13072 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13073 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13074 {
13075 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13076 rc = VERR_EM_INTERPRETER;
13077 break;
13078 }
13079 Log4Func(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13080 pMixedCtx->esp -= cbParm;
13081 pMixedCtx->esp &= uMask;
13082 pMixedCtx->rip += pDis->cbInstr;
13083 pMixedCtx->eflags.Bits.u1RF = 0;
13084 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13085 | HM_CHANGED_GUEST_RSP
13086 | HM_CHANGED_GUEST_RFLAGS);
13087 if ( !fDbgStepping
13088 && pMixedCtx->eflags.Bits.u1TF)
13089 {
13090 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13091 AssertRCReturn(rc, rc);
13092 }
13093 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13094 break;
13095 }
13096
13097 case OP_IRET:
13098 {
13099 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13100 * instruction reference. */
13101 RTGCPTR GCPtrStack = 0;
13102 uint32_t uMask = 0xffff;
13103 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13104 uint16_t aIretFrame[3];
13105 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13106 {
13107 rc = VERR_EM_INTERPRETER;
13108 break;
13109 }
13110 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13111 &GCPtrStack);
13112 if (RT_SUCCESS(rc))
13113 {
13114 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13115 PGMACCESSORIGIN_HM));
13116 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13117 }
13118 if (RT_FAILURE(rc))
13119 {
13120 rc = VERR_EM_INTERPRETER;
13121 break;
13122 }
13123 pMixedCtx->eip = 0;
13124 pMixedCtx->ip = aIretFrame[0];
13125 pMixedCtx->cs.Sel = aIretFrame[1];
13126 pMixedCtx->cs.ValidSel = aIretFrame[1];
13127 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13128 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13129 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13130 pMixedCtx->sp += sizeof(aIretFrame);
13131 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13132 | HM_CHANGED_GUEST_CS
13133 | HM_CHANGED_GUEST_RSP
13134 | HM_CHANGED_GUEST_RFLAGS);
13135 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13136 if ( !fDbgStepping
13137 && fGstStepping)
13138 {
13139 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13140 AssertRCReturn(rc, rc);
13141 }
13142 Log4Func(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13143 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13144 break;
13145 }
13146
13147 case OP_INT:
13148 {
13149 uint16_t uVector = pDis->Param1.uValue & 0xff;
13150 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13151 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13152 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13153 break;
13154 }
13155
13156 case OP_INTO:
13157 {
13158 if (pMixedCtx->eflags.Bits.u1OF)
13159 {
13160 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13161 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13162 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13163 }
13164 else
13165 {
13166 pMixedCtx->eflags.Bits.u1RF = 0;
13167 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
13168 }
13169 break;
13170 }
13171
13172 default:
13173 {
13174 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13175 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13176 EMCODETYPE_SUPERVISOR);
13177 rc = VBOXSTRICTRC_VAL(rc2);
13178 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13179 /** @todo We have to set pending-debug exceptions here when the guest is
13180 * single-stepping depending on the instruction that was interpreted. */
13181 Log4Func(("#GP rc=%Rrc\n", rc));
13182 break;
13183 }
13184 }
13185 }
13186 else
13187 rc = VERR_EM_INTERPRETER;
13188
13189 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13190 ("#GP Unexpected rc=%Rrc\n", rc));
13191 return rc;
13192}
13193
13194
13195/**
13196 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13197 * the exception reported in the VMX transient structure back into the VM.
13198 *
13199 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13200 * up-to-date.
13201 */
13202static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13203{
13204 RT_NOREF_PV(pMixedCtx);
13205 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13206#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13207 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13208 ("uVector=%#x u32XcptBitmap=%#X32\n",
13209 VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13210#endif
13211
13212 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13213 hmR0VmxCheckExitDueToEventDelivery(). */
13214 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13215 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13216 AssertRCReturn(rc, rc);
13217 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13218
13219#ifdef DEBUG_ramshankar
13220 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS
13221 | CPUMCTX_EXTRN_RIP);
13222 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13223 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13224#endif
13225
13226 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13227 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13228 return VINF_SUCCESS;
13229}
13230
13231
13232/**
13233 * VM-exit exception handler for \#PF (Page-fault exception).
13234 */
13235static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13236{
13237 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13238 PVM pVM = pVCpu->CTX_SUFF(pVM);
13239 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13240 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13241 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13242 AssertRCReturn(rc, rc);
13243
13244 if (!pVM->hm.s.fNestedPaging)
13245 { /* likely */ }
13246 else
13247 {
13248#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13249 Assert(pVCpu->hm.s.fUsingDebugLoop);
13250#endif
13251 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13252 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13253 {
13254 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13255 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13256 }
13257 else
13258 {
13259 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13260 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13261 Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13262 }
13263 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13264 return rc;
13265 }
13266
13267 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13268 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13269 if (pVmxTransient->fVectoringPF)
13270 {
13271 Assert(pVCpu->hm.s.Event.fPending);
13272 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13273 }
13274
13275 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13276 AssertRCReturn(rc, rc);
13277
13278 Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13279 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13280
13281 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13282 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13283 (RTGCPTR)pVmxTransient->uExitQualification);
13284
13285 Log4Func(("#PF: rc=%Rrc\n", rc));
13286 if (rc == VINF_SUCCESS)
13287 {
13288 /*
13289 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13290 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13291 */
13292 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13293 TRPMResetTrap(pVCpu);
13294 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13295 return rc;
13296 }
13297
13298 if (rc == VINF_EM_RAW_GUEST_TRAP)
13299 {
13300 if (!pVmxTransient->fVectoringDoublePF)
13301 {
13302 /* It's a guest page fault and needs to be reflected to the guest. */
13303 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13304 TRPMResetTrap(pVCpu);
13305 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13306 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13307 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13308 }
13309 else
13310 {
13311 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13312 TRPMResetTrap(pVCpu);
13313 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13314 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13315 Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13316 }
13317
13318 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13319 return VINF_SUCCESS;
13320 }
13321
13322 TRPMResetTrap(pVCpu);
13323 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13324 return rc;
13325}
13326
13327/** @} */
13328
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette