VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 72777

Last change on this file since 72777 was 72777, checked in by vboxsync, 6 years ago

HM: nit
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 572.9 KB
Line 
1/* $Id: HMVMXR0.cpp 72777 2018-06-29 18:04:40Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/x86.h>
25#include <iprt/asm-amd64-x86.h>
26#include <iprt/thread.h>
27
28#include <VBox/vmm/pdmapi.h>
29#include <VBox/vmm/dbgf.h>
30#include <VBox/vmm/iem.h>
31#include <VBox/vmm/iom.h>
32#include <VBox/vmm/selm.h>
33#include <VBox/vmm/tm.h>
34#include <VBox/vmm/gim.h>
35#include <VBox/vmm/apic.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#include "HMInternal.h"
40#include <VBox/vmm/vm.h>
41#include "HMVMXR0.h"
42#include "dtrace/VBoxVMM.h"
43
44#ifdef DEBUG_ramshankar
45# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
47# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_CHECK_GUEST_STATE
49# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
50# define HMVMX_ALWAYS_TRAP_PF
51# define HMVMX_ALWAYS_FLUSH_TLB
52# define HMVMX_ALWAYS_SWAP_EFER
53#endif
54
55
56/*********************************************************************************************************************************
57* Defined Constants And Macros *
58*********************************************************************************************************************************/
59/** Use the function table. */
60#define HMVMX_USE_FUNCTION_TABLE
61
62/** Determine which tagged-TLB flush handler to use. */
63#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68/** @name HMVMX_READ_XXX
69 * Flags to skip redundant reads of some common VMCS fields that are not part of
70 * the guest-CPU or VCPU state but are needed while handling VM-exits.
71 */
72#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
73#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
74#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
75#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
76#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
77#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
78#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
79/** @} */
80
81/**
82 * States of the VMCS.
83 *
84 * This does not reflect all possible VMCS states but currently only those
85 * needed for maintaining the VMCS consistently even when thread-context hooks
86 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
87 */
88#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
89#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
90#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
91
92/**
93 * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
94 * guest using hardware-assisted VMX.
95 *
96 * This excludes state like GPRs (other than RSP) which are always are
97 * swapped and restored across the world-switch and also registers like EFER,
98 * MSR which cannot be modified by the guest without causing a VM-exit.
99 */
100#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
101 | CPUMCTX_EXTRN_RFLAGS \
102 | CPUMCTX_EXTRN_RSP \
103 | CPUMCTX_EXTRN_SREG_MASK \
104 | CPUMCTX_EXTRN_TABLE_MASK \
105 | CPUMCTX_EXTRN_KERNEL_GS_BASE \
106 | CPUMCTX_EXTRN_SYSCALL_MSRS \
107 | CPUMCTX_EXTRN_SYSENTER_MSRS \
108 | CPUMCTX_EXTRN_TSC_AUX \
109 | CPUMCTX_EXTRN_OTHER_MSRS \
110 | CPUMCTX_EXTRN_CR0 \
111 | CPUMCTX_EXTRN_CR3 \
112 | CPUMCTX_EXTRN_CR4 \
113 | CPUMCTX_EXTRN_DR7 \
114 | CPUMCTX_EXTRN_HM_VMX_MASK)
115
116/**
117 * Exception bitmap mask for real-mode guests (real-on-v86).
118 *
119 * We need to intercept all exceptions manually except:
120 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
121 * due to bugs in Intel CPUs.
122 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
123 * support.
124 */
125#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
126 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
127 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
128 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
129 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
130 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
131 | RT_BIT(X86_XCPT_XF))
132
133/**
134 * Exception bitmap mask for all contributory exceptions.
135 *
136 * Page fault is deliberately excluded here as it's conditional as to whether
137 * it's contributory or benign. Page faults are handled separately.
138 */
139#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
140 | RT_BIT(X86_XCPT_DE))
141
142/** Maximum VM-instruction error number. */
143#define HMVMX_INSTR_ERROR_MAX 28
144
145/** Profiling macro. */
146#ifdef HM_PROFILE_EXIT_DISPATCH
147# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
148# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
149#else
150# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
151# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
152#endif
153
154/** Assert that preemption is disabled or covered by thread-context hooks. */
155#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
156 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
157
158/** Assert that we haven't migrated CPUs when thread-context hooks are not
159 * used. */
160#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
161 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
162 ("Illegal migration! Entered on CPU %u Current %u\n", \
163 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
164
165/** Helper macro for VM-exit handlers called unexpectedly. */
166#define HMVMX_RETURN_UNEXPECTED_EXIT() \
167 do { \
168 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
169 return VERR_VMX_UNEXPECTED_EXIT; \
170 } while (0)
171
172/** Macro for saving segment registers from VMCS into the guest-CPU
173 * context. */
174#ifdef VMX_USE_CACHED_VMCS_ACCESSES
175# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
176 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
177 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
178#else
179# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
180 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
181 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
182#endif
183
184
185/*********************************************************************************************************************************
186* Structures and Typedefs *
187*********************************************************************************************************************************/
188/**
189 * VMX transient state.
190 *
191 * A state structure for holding miscellaneous information across
192 * VMX non-root operation and restored after the transition.
193 */
194typedef struct VMXTRANSIENT
195{
196 /** The host's rflags/eflags. */
197 RTCCUINTREG fEFlags;
198#if HC_ARCH_BITS == 32
199 uint32_t u32Alignment0;
200#endif
201 /** The guest's TPR value used for TPR shadowing. */
202 uint8_t u8GuestTpr;
203 /** Alignment. */
204 uint8_t abAlignment0[7];
205
206 /** The basic VM-exit reason. */
207 uint16_t uExitReason;
208 /** Alignment. */
209 uint16_t u16Alignment0;
210 /** The VM-exit interruption error code. */
211 uint32_t uExitIntErrorCode;
212 /** The VM-exit exit code qualification. */
213 uint64_t uExitQualification;
214
215 /** The VM-exit interruption-information field. */
216 uint32_t uExitIntInfo;
217 /** The VM-exit instruction-length field. */
218 uint32_t cbInstr;
219 /** The VM-exit instruction-information field. */
220 union
221 {
222 /** Plain unsigned int representation. */
223 uint32_t u;
224 /** INS and OUTS information. */
225 struct
226 {
227 uint32_t u7Reserved0 : 7;
228 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
229 uint32_t u3AddrSize : 3;
230 uint32_t u5Reserved1 : 5;
231 /** The segment register (X86_SREG_XXX). */
232 uint32_t iSegReg : 3;
233 uint32_t uReserved2 : 14;
234 } StrIo;
235 /** INVEPT, INVVPID, INVPCID information. */
236 struct
237 {
238 /** Scaling; 0=no scaling, 1=scale-by-2, 2=scale-by-4, 3=scale-by-8. */
239 uint32_t u2Scaling : 2;
240 uint32_t u5Reserved0 : 5;
241 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
242 uint32_t u3AddrSize : 3;
243 uint32_t u1Reserved0 : 1;
244 uint32_t u4Reserved0 : 4;
245 /** The segment register (X86_SREG_XXX). */
246 uint32_t iSegReg : 3;
247 /** The index register (X86_GREG_XXX). */
248 uint32_t iIdxReg : 4;
249 /** Set if index register is invalid. */
250 uint32_t fIdxRegValid : 1;
251 /** The base register (X86_GREG_XXX). */
252 uint32_t iBaseReg : 4;
253 /** Set if base register is invalid. */
254 uint32_t fBaseRegValid : 1;
255 /** Register 2 (X86_GREG_XXX). */
256 uint32_t iReg2 : 4;
257 } Inv;
258 } ExitInstrInfo;
259 /** Whether the VM-entry failed or not. */
260 bool fVMEntryFailed;
261 /** Alignment. */
262 uint8_t abAlignment1[3];
263
264 /** The VM-entry interruption-information field. */
265 uint32_t uEntryIntInfo;
266 /** The VM-entry exception error code field. */
267 uint32_t uEntryXcptErrorCode;
268 /** The VM-entry instruction length field. */
269 uint32_t cbEntryInstr;
270
271 /** IDT-vectoring information field. */
272 uint32_t uIdtVectoringInfo;
273 /** IDT-vectoring error code. */
274 uint32_t uIdtVectoringErrorCode;
275
276 /** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
277 uint32_t fVmcsFieldsRead;
278
279 /** Whether the guest debug state was active at the time of VM-exit. */
280 bool fWasGuestDebugStateActive;
281 /** Whether the hyper debug state was active at the time of VM-exit. */
282 bool fWasHyperDebugStateActive;
283 /** Whether TSC-offsetting should be setup before VM-entry. */
284 bool fUpdateTscOffsettingAndPreemptTimer;
285 /** Whether the VM-exit was caused by a page-fault during delivery of a
286 * contributory exception or a page-fault. */
287 bool fVectoringDoublePF;
288 /** Whether the VM-exit was caused by a page-fault during delivery of an
289 * external interrupt or NMI. */
290 bool fVectoringPF;
291} VMXTRANSIENT;
292AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
293AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
294AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
295AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
296AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
297/** Pointer to VMX transient state. */
298typedef VMXTRANSIENT *PVMXTRANSIENT;
299
300
301/**
302 * MSR-bitmap read permissions.
303 */
304typedef enum VMXMSREXITREAD
305{
306 /** Reading this MSR causes a VM-exit. */
307 VMXMSREXIT_INTERCEPT_READ = 0xb,
308 /** Reading this MSR does not cause a VM-exit. */
309 VMXMSREXIT_PASSTHRU_READ
310} VMXMSREXITREAD;
311/** Pointer to MSR-bitmap read permissions. */
312typedef VMXMSREXITREAD* PVMXMSREXITREAD;
313
314/**
315 * MSR-bitmap write permissions.
316 */
317typedef enum VMXMSREXITWRITE
318{
319 /** Writing to this MSR causes a VM-exit. */
320 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
321 /** Writing to this MSR does not cause a VM-exit. */
322 VMXMSREXIT_PASSTHRU_WRITE
323} VMXMSREXITWRITE;
324/** Pointer to MSR-bitmap write permissions. */
325typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
326
327
328/**
329 * VMX VM-exit handler.
330 *
331 * @returns Strict VBox status code (i.e. informational status codes too).
332 * @param pVCpu The cross context virtual CPU structure.
333 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
334 * out-of-sync. Make sure to update the required
335 * fields before using them.
336 * @param pVmxTransient Pointer to the VMX-transient structure.
337 */
338#ifndef HMVMX_USE_FUNCTION_TABLE
339typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
340#else
341typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
342/** Pointer to VM-exit handler. */
343typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
344#endif
345
346/**
347 * VMX VM-exit handler, non-strict status code.
348 *
349 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
350 *
351 * @returns VBox status code, no informational status code returned.
352 * @param pVCpu The cross context virtual CPU structure.
353 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
354 * out-of-sync. Make sure to update the required
355 * fields before using them.
356 * @param pVmxTransient Pointer to the VMX-transient structure.
357 *
358 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
359 * use of that status code will be replaced with VINF_EM_SOMETHING
360 * later when switching over to IEM.
361 */
362#ifndef HMVMX_USE_FUNCTION_TABLE
363typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
364#else
365typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
366#endif
367
368
369/*********************************************************************************************************************************
370* Internal Functions *
371*********************************************************************************************************************************/
372static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
373static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
374static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
375static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat);
376static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
377 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState);
378#if HC_ARCH_BITS == 32
379static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
380#endif
381#ifndef HMVMX_USE_FUNCTION_TABLE
382DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
383# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
384# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
385#else
386# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
387# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
388#endif
389
390
391/** @name VM-exit handlers.
392 * @{
393 */
394static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
395static FNVMXEXITHANDLER hmR0VmxExitExtInt;
396static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
397static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
398static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
399static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
400static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
401static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
402static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
403static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
404static FNVMXEXITHANDLER hmR0VmxExitCpuid;
405static FNVMXEXITHANDLER hmR0VmxExitGetsec;
406static FNVMXEXITHANDLER hmR0VmxExitHlt;
407static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
408static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
409static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
410static FNVMXEXITHANDLER hmR0VmxExitVmcall;
411static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
412static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
413static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
414static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
415static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
416static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
417static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
418static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
419static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
420static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
421static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
422static FNVMXEXITHANDLER hmR0VmxExitMwait;
423static FNVMXEXITHANDLER hmR0VmxExitMtf;
424static FNVMXEXITHANDLER hmR0VmxExitMonitor;
425static FNVMXEXITHANDLER hmR0VmxExitPause;
426static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
427static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
428static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
429static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
430static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
431static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
432static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
433static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
434static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
435static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
436static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
437static FNVMXEXITHANDLER hmR0VmxExitRdrand;
438static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
439/** @} */
440
441static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
442static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
443static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
444static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
445static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
446static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
447static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
448static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
449
450
451/*********************************************************************************************************************************
452* Global Variables *
453*********************************************************************************************************************************/
454#ifdef HMVMX_USE_FUNCTION_TABLE
455
456/**
457 * VMX_EXIT dispatch table.
458 */
459static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
460{
461 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
462 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
463 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
464 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
465 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
466 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
467 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
468 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
469 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
470 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
471 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
472 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
473 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
474 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
475 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
476 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
477 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
478 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
479 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
480 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
481 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
482 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
483 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
484 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
485 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
486 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
487 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
488 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
489 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
490 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
491 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
492 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
493 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
494 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
495 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
496 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
497 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
498 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
499 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
500 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
501 /* 40 UNDEFINED */ hmR0VmxExitPause,
502 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
503 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
504 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
505 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
506 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
507 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
508 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
509 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
510 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
511 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
512 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
513 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
514 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
515 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
516 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
517 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
518 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
519 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
520 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
521 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
522 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
523 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
524 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
525 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
526};
527#endif /* HMVMX_USE_FUNCTION_TABLE */
528
529#ifdef VBOX_STRICT
530static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
531{
532 /* 0 */ "(Not Used)",
533 /* 1 */ "VMCALL executed in VMX root operation.",
534 /* 2 */ "VMCLEAR with invalid physical address.",
535 /* 3 */ "VMCLEAR with VMXON pointer.",
536 /* 4 */ "VMLAUNCH with non-clear VMCS.",
537 /* 5 */ "VMRESUME with non-launched VMCS.",
538 /* 6 */ "VMRESUME after VMXOFF",
539 /* 7 */ "VM-entry with invalid control fields.",
540 /* 8 */ "VM-entry with invalid host state fields.",
541 /* 9 */ "VMPTRLD with invalid physical address.",
542 /* 10 */ "VMPTRLD with VMXON pointer.",
543 /* 11 */ "VMPTRLD with incorrect revision identifier.",
544 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
545 /* 13 */ "VMWRITE to read-only VMCS component.",
546 /* 14 */ "(Not Used)",
547 /* 15 */ "VMXON executed in VMX root operation.",
548 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
549 /* 17 */ "VM-entry with non-launched executing VMCS.",
550 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
551 /* 19 */ "VMCALL with non-clear VMCS.",
552 /* 20 */ "VMCALL with invalid VM-exit control fields.",
553 /* 21 */ "(Not Used)",
554 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
555 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
556 /* 24 */ "VMCALL with invalid SMM-monitor features.",
557 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
558 /* 26 */ "VM-entry with events blocked by MOV SS.",
559 /* 27 */ "(Not Used)",
560 /* 28 */ "Invalid operand to INVEPT/INVVPID."
561};
562#endif /* VBOX_STRICT */
563
564
565
566/**
567 * Updates the VM's last error record.
568 *
569 * If there was a VMX instruction error, reads the error data from the VMCS and
570 * updates VCPU's last error record as well.
571 *
572 * @param pVM The cross context VM structure.
573 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
574 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
575 * VERR_VMX_INVALID_VMCS_FIELD.
576 * @param rc The error code.
577 */
578static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
579{
580 AssertPtr(pVM);
581 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
582 || rc == VERR_VMX_UNABLE_TO_START_VM)
583 {
584 AssertPtrReturnVoid(pVCpu);
585 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
586 }
587 pVM->hm.s.lLastError = rc;
588}
589
590
591/**
592 * Reads the VM-entry interruption-information field from the VMCS into the VMX
593 * transient structure.
594 *
595 * @returns VBox status code.
596 * @param pVmxTransient Pointer to the VMX transient structure.
597 *
598 * @remarks No-long-jump zone!!!
599 */
600DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
601{
602 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
603 AssertRCReturn(rc, rc);
604 return VINF_SUCCESS;
605}
606
607#ifdef VBOX_STRICT
608/**
609 * Reads the VM-entry exception error code field from the VMCS into
610 * the VMX transient structure.
611 *
612 * @returns VBox status code.
613 * @param pVmxTransient Pointer to the VMX transient structure.
614 *
615 * @remarks No-long-jump zone!!!
616 */
617DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
618{
619 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
620 AssertRCReturn(rc, rc);
621 return VINF_SUCCESS;
622}
623
624
625/**
626 * Reads the VM-entry exception error code field from the VMCS into
627 * the VMX transient structure.
628 *
629 * @returns VBox status code.
630 * @param pVmxTransient Pointer to the VMX transient structure.
631 *
632 * @remarks No-long-jump zone!!!
633 */
634DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
635{
636 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
637 AssertRCReturn(rc, rc);
638 return VINF_SUCCESS;
639}
640#endif /* VBOX_STRICT */
641
642
643/**
644 * Reads the VM-exit interruption-information field from the VMCS into the VMX
645 * transient structure.
646 *
647 * @returns VBox status code.
648 * @param pVmxTransient Pointer to the VMX transient structure.
649 */
650DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
651{
652 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
653 {
654 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
655 AssertRCReturn(rc,rc);
656 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_INFO;
657 }
658 return VINF_SUCCESS;
659}
660
661
662/**
663 * Reads the VM-exit interruption error code from the VMCS into the VMX
664 * transient structure.
665 *
666 * @returns VBox status code.
667 * @param pVmxTransient Pointer to the VMX transient structure.
668 */
669DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
670{
671 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
672 {
673 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
674 AssertRCReturn(rc, rc);
675 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
676 }
677 return VINF_SUCCESS;
678}
679
680
681/**
682 * Reads the VM-exit instruction length field from the VMCS into the VMX
683 * transient structure.
684 *
685 * @returns VBox status code.
686 * @param pVmxTransient Pointer to the VMX transient structure.
687 */
688DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
689{
690 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
691 {
692 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
693 AssertRCReturn(rc, rc);
694 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_LEN;
695 }
696 return VINF_SUCCESS;
697}
698
699
700/**
701 * Reads the VM-exit instruction-information field from the VMCS into
702 * the VMX transient structure.
703 *
704 * @returns VBox status code.
705 * @param pVmxTransient Pointer to the VMX transient structure.
706 */
707DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
708{
709 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
710 {
711 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
712 AssertRCReturn(rc, rc);
713 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_INFO;
714 }
715 return VINF_SUCCESS;
716}
717
718
719/**
720 * Reads the exit code qualification from the VMCS into the VMX transient
721 * structure.
722 *
723 * @returns VBox status code.
724 * @param pVCpu The cross context virtual CPU structure of the
725 * calling EMT. (Required for the VMCS cache case.)
726 * @param pVmxTransient Pointer to the VMX transient structure.
727 */
728DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
729{
730 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
731 {
732 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
733 AssertRCReturn(rc, rc);
734 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION;
735 }
736 return VINF_SUCCESS;
737}
738
739
740/**
741 * Reads the IDT-vectoring information field from the VMCS into the VMX
742 * transient structure.
743 *
744 * @returns VBox status code.
745 * @param pVmxTransient Pointer to the VMX transient structure.
746 *
747 * @remarks No-long-jump zone!!!
748 */
749DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
750{
751 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
752 {
753 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
754 AssertRCReturn(rc, rc);
755 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_INFO;
756 }
757 return VINF_SUCCESS;
758}
759
760
761/**
762 * Reads the IDT-vectoring error code from the VMCS into the VMX
763 * transient structure.
764 *
765 * @returns VBox status code.
766 * @param pVmxTransient Pointer to the VMX transient structure.
767 */
768DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
769{
770 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
771 {
772 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
773 AssertRCReturn(rc, rc);
774 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
775 }
776 return VINF_SUCCESS;
777}
778
779
780/**
781 * Enters VMX root mode operation on the current CPU.
782 *
783 * @returns VBox status code.
784 * @param pVM The cross context VM structure. Can be
785 * NULL, after a resume.
786 * @param HCPhysCpuPage Physical address of the VMXON region.
787 * @param pvCpuPage Pointer to the VMXON region.
788 */
789static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
790{
791 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
792 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
793 Assert(pvCpuPage);
794 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
795
796 if (pVM)
797 {
798 /* Write the VMCS revision dword to the VMXON region. */
799 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
800 }
801
802 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
803 RTCCUINTREG fEFlags = ASMIntDisableFlags();
804
805 /* Enable the VMX bit in CR4 if necessary. */
806 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
807
808 /* Enter VMX root mode. */
809 int rc = VMXEnable(HCPhysCpuPage);
810 if (RT_FAILURE(rc))
811 {
812 if (!(uOldCr4 & X86_CR4_VMXE))
813 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
814
815 if (pVM)
816 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
817 }
818
819 /* Restore interrupts. */
820 ASMSetFlags(fEFlags);
821 return rc;
822}
823
824
825/**
826 * Exits VMX root mode operation on the current CPU.
827 *
828 * @returns VBox status code.
829 */
830static int hmR0VmxLeaveRootMode(void)
831{
832 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
833
834 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
835 RTCCUINTREG fEFlags = ASMIntDisableFlags();
836
837 /* If we're for some reason not in VMX root mode, then don't leave it. */
838 RTCCUINTREG uHostCR4 = ASMGetCR4();
839
840 int rc;
841 if (uHostCR4 & X86_CR4_VMXE)
842 {
843 /* Exit VMX root mode and clear the VMX bit in CR4. */
844 VMXDisable();
845 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
846 rc = VINF_SUCCESS;
847 }
848 else
849 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
850
851 /* Restore interrupts. */
852 ASMSetFlags(fEFlags);
853 return rc;
854}
855
856
857/**
858 * Allocates and maps one physically contiguous page. The allocated page is
859 * zero'd out. (Used by various VT-x structures).
860 *
861 * @returns IPRT status code.
862 * @param pMemObj Pointer to the ring-0 memory object.
863 * @param ppVirt Where to store the virtual address of the
864 * allocation.
865 * @param pHCPhys Where to store the physical address of the
866 * allocation.
867 */
868static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
869{
870 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
871 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
872 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
873
874 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
875 if (RT_FAILURE(rc))
876 return rc;
877 *ppVirt = RTR0MemObjAddress(*pMemObj);
878 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
879 ASMMemZero32(*ppVirt, PAGE_SIZE);
880 return VINF_SUCCESS;
881}
882
883
884/**
885 * Frees and unmaps an allocated physical page.
886 *
887 * @param pMemObj Pointer to the ring-0 memory object.
888 * @param ppVirt Where to re-initialize the virtual address of
889 * allocation as 0.
890 * @param pHCPhys Where to re-initialize the physical address of the
891 * allocation as 0.
892 */
893static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
894{
895 AssertPtr(pMemObj);
896 AssertPtr(ppVirt);
897 AssertPtr(pHCPhys);
898 if (*pMemObj != NIL_RTR0MEMOBJ)
899 {
900 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
901 AssertRC(rc);
902 *pMemObj = NIL_RTR0MEMOBJ;
903 *ppVirt = 0;
904 *pHCPhys = 0;
905 }
906}
907
908
909/**
910 * Worker function to free VT-x related structures.
911 *
912 * @returns IPRT status code.
913 * @param pVM The cross context VM structure.
914 */
915static void hmR0VmxStructsFree(PVM pVM)
916{
917 for (VMCPUID i = 0; i < pVM->cCpus; i++)
918 {
919 PVMCPU pVCpu = &pVM->aCpus[i];
920 AssertPtr(pVCpu);
921
922 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
923 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
924
925 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
926 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
927
928 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
929 }
930
931 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
932#ifdef VBOX_WITH_CRASHDUMP_MAGIC
933 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
934#endif
935}
936
937
938/**
939 * Worker function to allocate VT-x related VM structures.
940 *
941 * @returns IPRT status code.
942 * @param pVM The cross context VM structure.
943 */
944static int hmR0VmxStructsAlloc(PVM pVM)
945{
946 /*
947 * Initialize members up-front so we can cleanup properly on allocation failure.
948 */
949#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
950 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
951 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
952 pVM->hm.s.vmx.HCPhys##a_Name = 0;
953
954#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
955 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
956 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
957 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
958
959#ifdef VBOX_WITH_CRASHDUMP_MAGIC
960 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
961#endif
962 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
963
964 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
965 for (VMCPUID i = 0; i < pVM->cCpus; i++)
966 {
967 PVMCPU pVCpu = &pVM->aCpus[i];
968 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
969 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
970 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
971 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
972 }
973#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
974#undef VMXLOCAL_INIT_VM_MEMOBJ
975
976 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
977 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
978 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
979 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
980
981 /*
982 * Allocate all the VT-x structures.
983 */
984 int rc = VINF_SUCCESS;
985#ifdef VBOX_WITH_CRASHDUMP_MAGIC
986 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
987 if (RT_FAILURE(rc))
988 goto cleanup;
989 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
990 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
991#endif
992
993 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
994 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
995 {
996 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
997 &pVM->hm.s.vmx.HCPhysApicAccess);
998 if (RT_FAILURE(rc))
999 goto cleanup;
1000 }
1001
1002 /*
1003 * Initialize per-VCPU VT-x structures.
1004 */
1005 for (VMCPUID i = 0; i < pVM->cCpus; i++)
1006 {
1007 PVMCPU pVCpu = &pVM->aCpus[i];
1008 AssertPtr(pVCpu);
1009
1010 /* Allocate the VM control structure (VMCS). */
1011 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1012 if (RT_FAILURE(rc))
1013 goto cleanup;
1014
1015 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1016 if ( PDMHasApic(pVM)
1017 && (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW))
1018 {
1019 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1020 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1021 if (RT_FAILURE(rc))
1022 goto cleanup;
1023 }
1024
1025 /*
1026 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1027 * transparent accesses of specific MSRs.
1028 *
1029 * If the condition for enabling MSR bitmaps changes here, don't forget to
1030 * update HMAreMsrBitmapsAvailable().
1031 */
1032 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1033 {
1034 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1035 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1036 if (RT_FAILURE(rc))
1037 goto cleanup;
1038 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1039 }
1040
1041 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1042 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1043 if (RT_FAILURE(rc))
1044 goto cleanup;
1045
1046 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1047 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1048 if (RT_FAILURE(rc))
1049 goto cleanup;
1050 }
1051
1052 return VINF_SUCCESS;
1053
1054cleanup:
1055 hmR0VmxStructsFree(pVM);
1056 return rc;
1057}
1058
1059
1060/**
1061 * Does global VT-x initialization (called during module initialization).
1062 *
1063 * @returns VBox status code.
1064 */
1065VMMR0DECL(int) VMXR0GlobalInit(void)
1066{
1067#ifdef HMVMX_USE_FUNCTION_TABLE
1068 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1069# ifdef VBOX_STRICT
1070 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1071 Assert(g_apfnVMExitHandlers[i]);
1072# endif
1073#endif
1074 return VINF_SUCCESS;
1075}
1076
1077
1078/**
1079 * Does global VT-x termination (called during module termination).
1080 */
1081VMMR0DECL(void) VMXR0GlobalTerm()
1082{
1083 /* Nothing to do currently. */
1084}
1085
1086
1087/**
1088 * Sets up and activates VT-x on the current CPU.
1089 *
1090 * @returns VBox status code.
1091 * @param pCpu Pointer to the global CPU info struct.
1092 * @param pVM The cross context VM structure. Can be
1093 * NULL after a host resume operation.
1094 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1095 * fEnabledByHost is @c true).
1096 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1097 * @a fEnabledByHost is @c true).
1098 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1099 * enable VT-x on the host.
1100 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1101 */
1102VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1103 void *pvMsrs)
1104{
1105 Assert(pCpu);
1106 Assert(pvMsrs);
1107 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1108
1109 /* Enable VT-x if it's not already enabled by the host. */
1110 if (!fEnabledByHost)
1111 {
1112 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1113 if (RT_FAILURE(rc))
1114 return rc;
1115 }
1116
1117 /*
1118 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
1119 * using EPTPs) so we don't retain any stale guest-physical mappings which won't get
1120 * invalidated when flushing by VPID.
1121 */
1122 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1123 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1124 {
1125 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1126 pCpu->fFlushAsidBeforeUse = false;
1127 }
1128 else
1129 pCpu->fFlushAsidBeforeUse = true;
1130
1131 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1132 ++pCpu->cTlbFlushes;
1133
1134 return VINF_SUCCESS;
1135}
1136
1137
1138/**
1139 * Deactivates VT-x on the current CPU.
1140 *
1141 * @returns VBox status code.
1142 * @param pCpu Pointer to the global CPU info struct.
1143 * @param pvCpuPage Pointer to the VMXON region.
1144 * @param HCPhysCpuPage Physical address of the VMXON region.
1145 *
1146 * @remarks This function should never be called when SUPR0EnableVTx() or
1147 * similar was used to enable VT-x on the host.
1148 */
1149VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1150{
1151 NOREF(pCpu);
1152 NOREF(pvCpuPage);
1153 NOREF(HCPhysCpuPage);
1154
1155 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1156 return hmR0VmxLeaveRootMode();
1157}
1158
1159
1160/**
1161 * Sets the permission bits for the specified MSR in the MSR bitmap.
1162 *
1163 * @param pVCpu The cross context virtual CPU structure.
1164 * @param uMsr The MSR value.
1165 * @param enmRead Whether reading this MSR causes a VM-exit.
1166 * @param enmWrite Whether writing this MSR causes a VM-exit.
1167 */
1168static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1169{
1170 int32_t iBit;
1171 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1172
1173 /*
1174 * Layout:
1175 * 0x000 - 0x3ff - Low MSR read bits
1176 * 0x400 - 0x7ff - High MSR read bits
1177 * 0x800 - 0xbff - Low MSR write bits
1178 * 0xc00 - 0xfff - High MSR write bits
1179 */
1180 if (uMsr <= 0x00001FFF)
1181 iBit = uMsr;
1182 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1183 {
1184 iBit = uMsr - UINT32_C(0xC0000000);
1185 pbMsrBitmap += 0x400;
1186 }
1187 else
1188 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1189
1190 Assert(iBit <= 0x1fff);
1191 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1192 ASMBitSet(pbMsrBitmap, iBit);
1193 else
1194 ASMBitClear(pbMsrBitmap, iBit);
1195
1196 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1197 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1198 else
1199 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1200}
1201
1202
1203#ifdef VBOX_STRICT
1204/**
1205 * Gets the permission bits for the specified MSR in the MSR bitmap.
1206 *
1207 * @returns VBox status code.
1208 * @retval VINF_SUCCESS if the specified MSR is found.
1209 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1210 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1211 *
1212 * @param pVCpu The cross context virtual CPU structure.
1213 * @param uMsr The MSR.
1214 * @param penmRead Where to store the read permissions.
1215 * @param penmWrite Where to store the write permissions.
1216 */
1217static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1218{
1219 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1220 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1221 int32_t iBit;
1222 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1223
1224 /* See hmR0VmxSetMsrPermission() for the layout. */
1225 if (uMsr <= 0x00001FFF)
1226 iBit = uMsr;
1227 else if ( uMsr >= 0xC0000000
1228 && uMsr <= 0xC0001FFF)
1229 {
1230 iBit = (uMsr - 0xC0000000);
1231 pbMsrBitmap += 0x400;
1232 }
1233 else
1234 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1235
1236 Assert(iBit <= 0x1fff);
1237 if (ASMBitTest(pbMsrBitmap, iBit))
1238 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1239 else
1240 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1241
1242 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1243 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1244 else
1245 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1246 return VINF_SUCCESS;
1247}
1248#endif /* VBOX_STRICT */
1249
1250
1251/**
1252 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1253 * area.
1254 *
1255 * @returns VBox status code.
1256 * @param pVCpu The cross context virtual CPU structure.
1257 * @param cMsrs The number of MSRs.
1258 */
1259static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1260{
1261 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1262 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1263 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1264 {
1265 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1266 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1267 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1268 }
1269
1270 /* Update number of guest MSRs to load/store across the world-switch. */
1271 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1272 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1273
1274 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1275 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1276 AssertRCReturn(rc, rc);
1277
1278 /* Update the VCPU's copy of the MSR count. */
1279 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1280
1281 return VINF_SUCCESS;
1282}
1283
1284
1285/**
1286 * Adds a new (or updates the value of an existing) guest/host MSR
1287 * pair to be swapped during the world-switch as part of the
1288 * auto-load/store MSR area in the VMCS.
1289 *
1290 * @returns VBox status code.
1291 * @param pVCpu The cross context virtual CPU structure.
1292 * @param uMsr The MSR.
1293 * @param uGuestMsrValue Value of the guest MSR.
1294 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1295 * necessary.
1296 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1297 * its value was updated. Optional, can be NULL.
1298 */
1299static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1300 bool *pfAddedAndUpdated)
1301{
1302 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1303 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1304 uint32_t i;
1305 for (i = 0; i < cMsrs; i++)
1306 {
1307 if (pGuestMsr->u32Msr == uMsr)
1308 break;
1309 pGuestMsr++;
1310 }
1311
1312 bool fAdded = false;
1313 if (i == cMsrs)
1314 {
1315 ++cMsrs;
1316 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1317 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1318
1319 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1320 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1321 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1322
1323 fAdded = true;
1324 }
1325
1326 /* Update the MSR values in the auto-load/store MSR area. */
1327 pGuestMsr->u32Msr = uMsr;
1328 pGuestMsr->u64Value = uGuestMsrValue;
1329
1330 /* Create/update the MSR slot in the host MSR area. */
1331 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1332 pHostMsr += i;
1333 pHostMsr->u32Msr = uMsr;
1334
1335 /*
1336 * Update the host MSR only when requested by the caller AND when we're
1337 * adding it to the auto-load/store area. Otherwise, it would have been
1338 * updated by hmR0VmxExportHostMsrs(). We do this for performance reasons.
1339 */
1340 bool fUpdatedMsrValue = false;
1341 if ( fAdded
1342 && fUpdateHostMsr)
1343 {
1344 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1345 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1346 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1347 fUpdatedMsrValue = true;
1348 }
1349
1350 if (pfAddedAndUpdated)
1351 *pfAddedAndUpdated = fUpdatedMsrValue;
1352 return VINF_SUCCESS;
1353}
1354
1355
1356/**
1357 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1358 * auto-load/store MSR area in the VMCS.
1359 *
1360 * @returns VBox status code.
1361 * @param pVCpu The cross context virtual CPU structure.
1362 * @param uMsr The MSR.
1363 */
1364static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1365{
1366 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1367 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1368 for (uint32_t i = 0; i < cMsrs; i++)
1369 {
1370 /* Find the MSR. */
1371 if (pGuestMsr->u32Msr == uMsr)
1372 {
1373 /* If it's the last MSR, simply reduce the count. */
1374 if (i == cMsrs - 1)
1375 {
1376 --cMsrs;
1377 break;
1378 }
1379
1380 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1381 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1382 pLastGuestMsr += cMsrs - 1;
1383 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1384 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1385
1386 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1387 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1388 pLastHostMsr += cMsrs - 1;
1389 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1390 pHostMsr->u64Value = pLastHostMsr->u64Value;
1391 --cMsrs;
1392 break;
1393 }
1394 pGuestMsr++;
1395 }
1396
1397 /* Update the VMCS if the count changed (meaning the MSR was found). */
1398 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1399 {
1400 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1401 AssertRCReturn(rc, rc);
1402
1403 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1404 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1405 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1406
1407 Log4Func(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1408 return VINF_SUCCESS;
1409 }
1410
1411 return VERR_NOT_FOUND;
1412}
1413
1414
1415/**
1416 * Checks if the specified guest MSR is part of the auto-load/store area in
1417 * the VMCS.
1418 *
1419 * @returns true if found, false otherwise.
1420 * @param pVCpu The cross context virtual CPU structure.
1421 * @param uMsr The MSR to find.
1422 */
1423static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1424{
1425 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1426 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1427
1428 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1429 {
1430 if (pGuestMsr->u32Msr == uMsr)
1431 return true;
1432 }
1433 return false;
1434}
1435
1436
1437/**
1438 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1439 *
1440 * @param pVCpu The cross context virtual CPU structure.
1441 *
1442 * @remarks No-long-jump zone!!!
1443 */
1444static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1445{
1446 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1447 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1448 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1449 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1450
1451 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1452 {
1453 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1454
1455 /*
1456 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1457 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1458 */
1459 if (pHostMsr->u32Msr == MSR_K6_EFER)
1460 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1461 else
1462 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1463 }
1464
1465 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1466}
1467
1468
1469/**
1470 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1471 * perform lazy restoration of the host MSRs while leaving VT-x.
1472 *
1473 * @param pVCpu The cross context virtual CPU structure.
1474 *
1475 * @remarks No-long-jump zone!!!
1476 */
1477static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1478{
1479 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1480
1481 /*
1482 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1483 */
1484 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1485 {
1486 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1487#if HC_ARCH_BITS == 64
1488 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1489 {
1490 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1491 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1492 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1493 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1494 }
1495#endif
1496 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1497 }
1498}
1499
1500
1501/**
1502 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1503 * lazily while leaving VT-x.
1504 *
1505 * @returns true if it does, false otherwise.
1506 * @param pVCpu The cross context virtual CPU structure.
1507 * @param uMsr The MSR to check.
1508 */
1509static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1510{
1511 NOREF(pVCpu);
1512#if HC_ARCH_BITS == 64
1513 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1514 {
1515 switch (uMsr)
1516 {
1517 case MSR_K8_LSTAR:
1518 case MSR_K6_STAR:
1519 case MSR_K8_SF_MASK:
1520 case MSR_K8_KERNEL_GS_BASE:
1521 return true;
1522 }
1523 }
1524#else
1525 RT_NOREF(pVCpu, uMsr);
1526#endif
1527 return false;
1528}
1529
1530
1531/**
1532 * Loads a set of guests MSRs to allow read/passthru to the guest.
1533 *
1534 * The name of this function is slightly confusing. This function does NOT
1535 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1536 * common prefix for functions dealing with "lazy restoration" of the shared
1537 * MSRs.
1538 *
1539 * @param pVCpu The cross context virtual CPU structure.
1540 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1541 * out-of-sync. Make sure to update the required fields
1542 * before using them.
1543 *
1544 * @remarks No-long-jump zone!!!
1545 */
1546static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1547{
1548 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1549 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1550
1551 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1552#if HC_ARCH_BITS == 64
1553 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1554 {
1555 /*
1556 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1557 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1558 * we can skip a few MSR writes.
1559 *
1560 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1561 * guest MSR values in the guest-CPU context might be different to what's currently
1562 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1563 * CPU, see @bugref{8728}.
1564 */
1565 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1566 && pMixedCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1567 && pMixedCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1568 && pMixedCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1569 && pMixedCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1570 {
1571#ifdef VBOX_STRICT
1572 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pMixedCtx->msrKERNELGSBASE);
1573 Assert(ASMRdMsr(MSR_K8_LSTAR) == pMixedCtx->msrLSTAR);
1574 Assert(ASMRdMsr(MSR_K6_STAR) == pMixedCtx->msrSTAR);
1575 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pMixedCtx->msrSFMASK);
1576#endif
1577 }
1578 else
1579 {
1580 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1581 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1582 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1583 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1584 }
1585 }
1586#else
1587 RT_NOREF(pMixedCtx);
1588#endif
1589 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1590}
1591
1592
1593/**
1594 * Performs lazy restoration of the set of host MSRs if they were previously
1595 * loaded with guest MSR values.
1596 *
1597 * @param pVCpu The cross context virtual CPU structure.
1598 *
1599 * @remarks No-long-jump zone!!!
1600 * @remarks The guest MSRs should have been saved back into the guest-CPU
1601 * context by hmR0VmxImportGuestState()!!!
1602 */
1603static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1604{
1605 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1606 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1607
1608 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1609 {
1610 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1611#if HC_ARCH_BITS == 64
1612 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1613 {
1614 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1615 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1616 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1617 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1618 }
1619#endif
1620 }
1621 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1622}
1623
1624
1625/**
1626 * Verifies that our cached values of the VMCS controls are all
1627 * consistent with what's actually present in the VMCS.
1628 *
1629 * @returns VBox status code.
1630 * @param pVCpu The cross context virtual CPU structure.
1631 */
1632static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1633{
1634 uint32_t u32Val;
1635 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1636 AssertRCReturn(rc, rc);
1637 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1638 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1639
1640 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1641 AssertRCReturn(rc, rc);
1642 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1643 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1644
1645 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1646 AssertRCReturn(rc, rc);
1647 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1648 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1649
1650 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1651 AssertRCReturn(rc, rc);
1652 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1653 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1654
1655 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1656 {
1657 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1658 AssertRCReturn(rc, rc);
1659 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1660 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1661 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1662 }
1663
1664 return VINF_SUCCESS;
1665}
1666
1667
1668#ifdef VBOX_STRICT
1669/**
1670 * Verifies that our cached host EFER value has not changed
1671 * since we cached it.
1672 *
1673 * @param pVCpu The cross context virtual CPU structure.
1674 */
1675static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1676{
1677 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1678
1679 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1680 {
1681 uint64_t u64Val;
1682 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1683 AssertRC(rc);
1684
1685 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1686 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1687 }
1688}
1689
1690
1691/**
1692 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1693 * VMCS are correct.
1694 *
1695 * @param pVCpu The cross context virtual CPU structure.
1696 */
1697static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1698{
1699 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1700
1701 /* Verify MSR counts in the VMCS are what we think it should be. */
1702 uint32_t cMsrs;
1703 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1704 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1705
1706 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1707 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1708
1709 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1710 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1711
1712 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1713 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1714 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1715 {
1716 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1717 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1718 pGuestMsr->u32Msr, cMsrs));
1719
1720 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1721 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1722 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1723
1724 /* Verify that the permissions are as expected in the MSR bitmap. */
1725 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1726 {
1727 VMXMSREXITREAD enmRead;
1728 VMXMSREXITWRITE enmWrite;
1729 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1730 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1731 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1732 {
1733 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1734 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1735 }
1736 else
1737 {
1738 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1739 pGuestMsr->u32Msr, cMsrs));
1740 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1741 pGuestMsr->u32Msr, cMsrs));
1742 }
1743 }
1744 }
1745}
1746#endif /* VBOX_STRICT */
1747
1748
1749/**
1750 * Flushes the TLB using EPT.
1751 *
1752 * @returns VBox status code.
1753 * @param pVCpu The cross context virtual CPU structure of the calling
1754 * EMT. Can be NULL depending on @a enmFlush.
1755 * @param enmFlush Type of flush.
1756 *
1757 * @remarks Caller is responsible for making sure this function is called only
1758 * when NestedPaging is supported and providing @a enmFlush that is
1759 * supported by the CPU.
1760 * @remarks Can be called with interrupts disabled.
1761 */
1762static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1763{
1764 uint64_t au64Descriptor[2];
1765 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1766 au64Descriptor[0] = 0;
1767 else
1768 {
1769 Assert(pVCpu);
1770 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1771 }
1772 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1773
1774 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1775 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1776 rc));
1777 if ( RT_SUCCESS(rc)
1778 && pVCpu)
1779 {
1780 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1781 }
1782}
1783
1784
1785/**
1786 * Flushes the TLB using VPID.
1787 *
1788 * @returns VBox status code.
1789 * @param pVM The cross context VM structure.
1790 * @param pVCpu The cross context virtual CPU structure of the calling
1791 * EMT. Can be NULL depending on @a enmFlush.
1792 * @param enmFlush Type of flush.
1793 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1794 * on @a enmFlush).
1795 *
1796 * @remarks Can be called with interrupts disabled.
1797 */
1798static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1799{
1800 NOREF(pVM);
1801 AssertPtr(pVM);
1802 Assert(pVM->hm.s.vmx.fVpid);
1803
1804 uint64_t au64Descriptor[2];
1805 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1806 {
1807 au64Descriptor[0] = 0;
1808 au64Descriptor[1] = 0;
1809 }
1810 else
1811 {
1812 AssertPtr(pVCpu);
1813 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1814 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1815 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1816 au64Descriptor[1] = GCPtr;
1817 }
1818
1819 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1820 AssertMsg(rc == VINF_SUCCESS,
1821 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1822 if ( RT_SUCCESS(rc)
1823 && pVCpu)
1824 {
1825 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1826 }
1827}
1828
1829
1830/**
1831 * Invalidates a guest page by guest virtual address. Only relevant for
1832 * EPT/VPID, otherwise there is nothing really to invalidate.
1833 *
1834 * @returns VBox status code.
1835 * @param pVM The cross context VM structure.
1836 * @param pVCpu The cross context virtual CPU structure.
1837 * @param GCVirt Guest virtual address of the page to invalidate.
1838 */
1839VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1840{
1841 AssertPtr(pVM);
1842 AssertPtr(pVCpu);
1843 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1844
1845 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1846 if (!fFlushPending)
1847 {
1848 /*
1849 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1850 * the EPT case. See @bugref{6043} and @bugref{6177}.
1851 *
1852 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1853 * as this function maybe called in a loop with individual addresses.
1854 */
1855 if (pVM->hm.s.vmx.fVpid)
1856 {
1857 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1858
1859#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1860 /*
1861 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1862 * where executing INVVPID outside 64-bit mode does not flush translations of
1863 * 64-bit linear addresses, see @bugref{6208#c72}.
1864 */
1865 if (RT_HI_U32(GCVirt))
1866 fVpidFlush = false;
1867#endif
1868
1869 if (fVpidFlush)
1870 {
1871 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1872 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1873 }
1874 else
1875 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1876 }
1877 else if (pVM->hm.s.fNestedPaging)
1878 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1879 }
1880
1881 return VINF_SUCCESS;
1882}
1883
1884
1885/**
1886 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1887 * case where neither EPT nor VPID is supported by the CPU.
1888 *
1889 * @param pVM The cross context VM structure.
1890 * @param pVCpu The cross context virtual CPU structure.
1891 * @param pCpu Pointer to the global HM struct.
1892 *
1893 * @remarks Called with interrupts disabled.
1894 */
1895static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1896{
1897 AssertPtr(pVCpu);
1898 AssertPtr(pCpu);
1899 NOREF(pVM);
1900
1901 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1902
1903 Assert(pCpu->idCpu != NIL_RTCPUID);
1904 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1905 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1906 pVCpu->hm.s.fForceTLBFlush = false;
1907 return;
1908}
1909
1910
1911/**
1912 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1913 *
1914 * @param pVM The cross context VM structure.
1915 * @param pVCpu The cross context virtual CPU structure.
1916 * @param pCpu Pointer to the global HM CPU struct.
1917 * @remarks All references to "ASID" in this function pertains to "VPID" in
1918 * Intel's nomenclature. The reason is, to avoid confusion in compare
1919 * statements since the host-CPU copies are named "ASID".
1920 *
1921 * @remarks Called with interrupts disabled.
1922 */
1923static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1924{
1925#ifdef VBOX_WITH_STATISTICS
1926 bool fTlbFlushed = false;
1927# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1928# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1929 if (!fTlbFlushed) \
1930 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1931 } while (0)
1932#else
1933# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1934# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1935#endif
1936
1937 AssertPtr(pVM);
1938 AssertPtr(pCpu);
1939 AssertPtr(pVCpu);
1940 Assert(pCpu->idCpu != NIL_RTCPUID);
1941
1942 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1943 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1944 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1945
1946 /*
1947 * Force a TLB flush for the first world-switch if the current CPU differs from the one we
1948 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1949 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1950 * cannot reuse the current ASID anymore.
1951 */
1952 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1953 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1954 {
1955 ++pCpu->uCurrentAsid;
1956 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1957 {
1958 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1959 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1960 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1961 }
1962
1963 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1964 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1965 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1966
1967 /*
1968 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1969 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1970 */
1971 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1972 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1973 HMVMX_SET_TAGGED_TLB_FLUSHED();
1974 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1975 }
1976
1977 /* Check for explicit TLB flushes. */
1978 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1979 {
1980 /*
1981 * Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
1982 * creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
1983 * tables when EPT is in use. Flushing-by-VPID will only flush linear (only
1984 * VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
1985 * mappings, see @bugref{6568}.
1986 *
1987 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
1988 */
1989 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1990 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1991 HMVMX_SET_TAGGED_TLB_FLUSHED();
1992 }
1993
1994 pVCpu->hm.s.fForceTLBFlush = false;
1995 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1996
1997 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1998 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1999 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2000 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2001 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2002 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2003 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2004 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2005 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2006
2007 /* Update VMCS with the VPID. */
2008 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2009 AssertRC(rc);
2010
2011#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2012}
2013
2014
2015/**
2016 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2017 *
2018 * @returns VBox status code.
2019 * @param pVM The cross context VM structure.
2020 * @param pVCpu The cross context virtual CPU structure.
2021 * @param pCpu Pointer to the global HM CPU struct.
2022 *
2023 * @remarks Called with interrupts disabled.
2024 */
2025static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2026{
2027 AssertPtr(pVM);
2028 AssertPtr(pVCpu);
2029 AssertPtr(pCpu);
2030 Assert(pCpu->idCpu != NIL_RTCPUID);
2031 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2032 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2033
2034 /*
2035 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2036 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2037 */
2038 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2039 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2040 {
2041 pVCpu->hm.s.fForceTLBFlush = true;
2042 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2043 }
2044
2045 /* Check for explicit TLB flushes. */
2046 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2047 {
2048 pVCpu->hm.s.fForceTLBFlush = true;
2049 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2050 }
2051
2052 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2053 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2054
2055 if (pVCpu->hm.s.fForceTLBFlush)
2056 {
2057 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2058 pVCpu->hm.s.fForceTLBFlush = false;
2059 }
2060}
2061
2062
2063/**
2064 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2065 *
2066 * @returns VBox status code.
2067 * @param pVM The cross context VM structure.
2068 * @param pVCpu The cross context virtual CPU structure.
2069 * @param pCpu Pointer to the global HM CPU struct.
2070 *
2071 * @remarks Called with interrupts disabled.
2072 */
2073static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2074{
2075 AssertPtr(pVM);
2076 AssertPtr(pVCpu);
2077 AssertPtr(pCpu);
2078 Assert(pCpu->idCpu != NIL_RTCPUID);
2079 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2080 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2081
2082 /*
2083 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2084 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2085 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2086 */
2087 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2088 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2089 {
2090 pVCpu->hm.s.fForceTLBFlush = true;
2091 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2092 }
2093
2094 /* Check for explicit TLB flushes. */
2095 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2096 {
2097 /*
2098 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2099 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2100 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2101 */
2102 pVCpu->hm.s.fForceTLBFlush = true;
2103 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2104 }
2105
2106 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2107 if (pVCpu->hm.s.fForceTLBFlush)
2108 {
2109 ++pCpu->uCurrentAsid;
2110 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2111 {
2112 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2113 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2114 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2115 }
2116
2117 pVCpu->hm.s.fForceTLBFlush = false;
2118 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2119 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2120 if (pCpu->fFlushAsidBeforeUse)
2121 {
2122 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2123 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2124 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2125 {
2126 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2127 pCpu->fFlushAsidBeforeUse = false;
2128 }
2129 else
2130 {
2131 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2132 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2133 }
2134 }
2135 }
2136
2137 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2138 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2139 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2140 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2141 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2142 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2143 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2144
2145 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2146 AssertRC(rc);
2147}
2148
2149
2150/**
2151 * Flushes the guest TLB entry based on CPU capabilities.
2152 *
2153 * @param pVCpu The cross context virtual CPU structure.
2154 * @param pCpu Pointer to the global HM CPU struct.
2155 */
2156DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2157{
2158#ifdef HMVMX_ALWAYS_FLUSH_TLB
2159 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2160#endif
2161 PVM pVM = pVCpu->CTX_SUFF(pVM);
2162 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2163 {
2164 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2165 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2166 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2167 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2168 default:
2169 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2170 break;
2171 }
2172 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2173}
2174
2175
2176/**
2177 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2178 * TLB entries from the host TLB before VM-entry.
2179 *
2180 * @returns VBox status code.
2181 * @param pVM The cross context VM structure.
2182 */
2183static int hmR0VmxSetupTaggedTlb(PVM pVM)
2184{
2185 /*
2186 * Determine optimal flush type for Nested Paging.
2187 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2188 * guest execution (see hmR3InitFinalizeR0()).
2189 */
2190 if (pVM->hm.s.fNestedPaging)
2191 {
2192 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2193 {
2194 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2195 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2196 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2197 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2198 else
2199 {
2200 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2201 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2202 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2203 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2204 }
2205
2206 /* Make sure the write-back cacheable memory type for EPT is supported. */
2207 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2208 {
2209 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2210 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2211 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2212 }
2213
2214 /* EPT requires a page-walk length of 4. */
2215 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2216 {
2217 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2218 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2219 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2220 }
2221 }
2222 else
2223 {
2224 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2225 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2226 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2227 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2228 }
2229 }
2230
2231 /*
2232 * Determine optimal flush type for VPID.
2233 */
2234 if (pVM->hm.s.vmx.fVpid)
2235 {
2236 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2237 {
2238 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2239 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2240 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2241 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2242 else
2243 {
2244 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2245 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2246 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2247 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2248 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2249 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2250 pVM->hm.s.vmx.fVpid = false;
2251 }
2252 }
2253 else
2254 {
2255 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2256 Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2257 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2258 pVM->hm.s.vmx.fVpid = false;
2259 }
2260 }
2261
2262 /*
2263 * Setup the handler for flushing tagged-TLBs.
2264 */
2265 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2266 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2267 else if (pVM->hm.s.fNestedPaging)
2268 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2269 else if (pVM->hm.s.vmx.fVpid)
2270 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2271 else
2272 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2273 return VINF_SUCCESS;
2274}
2275
2276
2277/**
2278 * Sets up pin-based VM-execution controls in the VMCS.
2279 *
2280 * @returns VBox status code.
2281 * @param pVM The cross context VM structure.
2282 * @param pVCpu The cross context virtual CPU structure.
2283 */
2284static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2285{
2286 AssertPtr(pVM);
2287 AssertPtr(pVCpu);
2288
2289 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2290 uint32_t fZap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2291
2292 fVal |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2293 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2294
2295 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2296 fVal |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2297
2298 /* Enable the VMX preemption timer. */
2299 if (pVM->hm.s.vmx.fUsePreemptTimer)
2300 {
2301 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2302 fVal |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2303 }
2304
2305#if 0
2306 /* Enable posted-interrupt processing. */
2307 if (pVM->hm.s.fPostedIntrs)
2308 {
2309 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2310 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2311 fVal |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2312 }
2313#endif
2314
2315 if ((fVal & fZap) != fVal)
2316 {
2317 LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! Cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2318 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, fVal, fZap));
2319 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2320 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2321 }
2322
2323 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2324 AssertRCReturn(rc, rc);
2325
2326 pVCpu->hm.s.vmx.u32PinCtls = fVal;
2327 return rc;
2328}
2329
2330
2331/**
2332 * Sets up processor-based VM-execution controls in the VMCS.
2333 *
2334 * @returns VBox status code.
2335 * @param pVM The cross context VM structure.
2336 * @param pVCpu The cross context virtual CPU structure.
2337 */
2338static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2339{
2340 AssertPtr(pVM);
2341 AssertPtr(pVCpu);
2342
2343 int rc = VERR_INTERNAL_ERROR_5;
2344 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2345 uint32_t fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2346
2347 fVal |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2348 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2349 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2350 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2351 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2352 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2353 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2354
2355 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2356 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2357 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2358 {
2359 LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2360 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2361 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2362 }
2363
2364 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2365 if (!pVM->hm.s.fNestedPaging)
2366 {
2367 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2368 fVal |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2369 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2370 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2371 }
2372
2373 /* Use TPR shadowing if supported by the CPU. */
2374 if ( PDMHasApic(pVM)
2375 && pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2376 {
2377 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2378 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2379 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2380 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2381 AssertRCReturn(rc, rc);
2382
2383 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2384 /* CR8 writes cause a VM-exit based on TPR threshold. */
2385 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2386 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2387 }
2388 else
2389 {
2390 /*
2391 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2392 * Set this control only for 64-bit guests.
2393 */
2394 if (pVM->hm.s.fAllow64BitGuests)
2395 {
2396 fVal |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2397 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2398 }
2399 }
2400
2401 /* Use MSR-bitmaps if supported by the CPU. */
2402 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2403 {
2404 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2405
2406 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2407 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2408 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2409 AssertRCReturn(rc, rc);
2410
2411 /*
2412 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2413 * automatically using dedicated fields in the VMCS.
2414 */
2415 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2416 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2417 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2418 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2419 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2420
2421#if HC_ARCH_BITS == 64
2422 /*
2423 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2424 */
2425 if (pVM->hm.s.fAllow64BitGuests)
2426 {
2427 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2428 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2429 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2430 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2431 }
2432#endif
2433 /*
2434 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2435 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2436 */
2437 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2438 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2439
2440 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2441 }
2442
2443 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2444 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2445 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2446
2447 if ((fVal & fZap) != fVal)
2448 {
2449 LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2450 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, fVal, fZap));
2451 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2452 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2453 }
2454
2455 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2456 AssertRCReturn(rc, rc);
2457
2458 pVCpu->hm.s.vmx.u32ProcCtls = fVal;
2459
2460 /*
2461 * Secondary processor-based VM-execution controls.
2462 */
2463 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2464 {
2465 fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2466 fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2467
2468 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2469 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2470
2471 if (pVM->hm.s.fNestedPaging)
2472 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2473
2474 /*
2475 * Enable the INVPCID instruction if supported by the hardware and we expose
2476 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2477 */
2478 if ( (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2479 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2480 {
2481 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2482 }
2483
2484 if (pVM->hm.s.vmx.fVpid)
2485 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2486
2487 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2488 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2489
2490#if 0
2491 if (pVM->hm.s.fVirtApicRegs)
2492 {
2493 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2494 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2495
2496 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2497 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2498 }
2499#endif
2500
2501 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2502 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2503 * done dynamically. */
2504 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2505 {
2506 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2507 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2508 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2509 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2510 AssertRCReturn(rc, rc);
2511 }
2512
2513 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2514 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2515
2516 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2517 && pVM->hm.s.vmx.cPleGapTicks
2518 && pVM->hm.s.vmx.cPleWindowTicks)
2519 {
2520 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2521
2522 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2523 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2524 AssertRCReturn(rc, rc);
2525 }
2526
2527 if ((fVal & fZap) != fVal)
2528 {
2529 LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2530 "cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, fVal, fZap));
2531 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2532 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2533 }
2534
2535 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2536 AssertRCReturn(rc, rc);
2537
2538 pVCpu->hm.s.vmx.u32ProcCtls2 = fVal;
2539 }
2540 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2541 {
2542 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2543 "available\n"));
2544 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2545 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2546 }
2547
2548 return VINF_SUCCESS;
2549}
2550
2551
2552/**
2553 * Sets up miscellaneous (everything other than Pin & Processor-based
2554 * VM-execution) control fields in the VMCS.
2555 *
2556 * @returns VBox status code.
2557 * @param pVM The cross context VM structure.
2558 * @param pVCpu The cross context virtual CPU structure.
2559 */
2560static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2561{
2562 NOREF(pVM);
2563 AssertPtr(pVM);
2564 AssertPtr(pVCpu);
2565
2566 int rc = VERR_GENERAL_FAILURE;
2567
2568 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2569#if 0
2570 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxExportGuestCR3AndCR4())*/
2571 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2572 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2573
2574 /*
2575 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2576 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2577 * We thus use the exception bitmap to control it rather than use both.
2578 */
2579 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2580 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2581
2582 /* All IO & IOIO instructions cause VM-exits. */
2583 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2584 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2585
2586 /* Initialize the MSR-bitmap area. */
2587 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2588 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2589 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2590 AssertRCReturn(rc, rc);
2591#endif
2592
2593 /* Setup MSR auto-load/store area. */
2594 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2595 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2596 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2597 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2598 AssertRCReturn(rc, rc);
2599
2600 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2601 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2602 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2603 AssertRCReturn(rc, rc);
2604
2605 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2606 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2607 AssertRCReturn(rc, rc);
2608
2609 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2610#if 0
2611 /* Setup debug controls */
2612 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2613 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2614 AssertRCReturn(rc, rc);
2615#endif
2616
2617 return rc;
2618}
2619
2620
2621/**
2622 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2623 *
2624 * We shall setup those exception intercepts that don't change during the
2625 * lifetime of the VM here. The rest are done dynamically while loading the
2626 * guest state.
2627 *
2628 * @returns VBox status code.
2629 * @param pVM The cross context VM structure.
2630 * @param pVCpu The cross context virtual CPU structure.
2631 */
2632static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2633{
2634 AssertPtr(pVM);
2635 AssertPtr(pVCpu);
2636
2637 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2638
2639 uint32_t u32XcptBitmap = 0;
2640
2641 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2642 u32XcptBitmap |= RT_BIT_32(X86_XCPT_AC);
2643
2644 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2645 and writes, and because recursive #DBs can cause the CPU hang, we must always
2646 intercept #DB. */
2647 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2648
2649 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2650 if (!pVM->hm.s.fNestedPaging)
2651 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2652
2653 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2654 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2655 AssertRCReturn(rc, rc);
2656 return rc;
2657}
2658
2659
2660/**
2661 * Does per-VM VT-x initialization.
2662 *
2663 * @returns VBox status code.
2664 * @param pVM The cross context VM structure.
2665 */
2666VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2667{
2668 LogFlowFunc(("pVM=%p\n", pVM));
2669
2670 int rc = hmR0VmxStructsAlloc(pVM);
2671 if (RT_FAILURE(rc))
2672 {
2673 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2674 return rc;
2675 }
2676
2677 return VINF_SUCCESS;
2678}
2679
2680
2681/**
2682 * Does per-VM VT-x termination.
2683 *
2684 * @returns VBox status code.
2685 * @param pVM The cross context VM structure.
2686 */
2687VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2688{
2689 LogFlowFunc(("pVM=%p\n", pVM));
2690
2691#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2692 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2693 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2694#endif
2695 hmR0VmxStructsFree(pVM);
2696 return VINF_SUCCESS;
2697}
2698
2699
2700/**
2701 * Sets up the VM for execution under VT-x.
2702 * This function is only called once per-VM during initialization.
2703 *
2704 * @returns VBox status code.
2705 * @param pVM The cross context VM structure.
2706 */
2707VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2708{
2709 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2710 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2711
2712 LogFlowFunc(("pVM=%p\n", pVM));
2713
2714 /*
2715 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be
2716 * allocated. We no longer support the highly unlikely case of UnrestrictedGuest without
2717 * pRealModeTSS, see hmR3InitFinalizeR0Intel().
2718 */
2719 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2720 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2721 || !pVM->hm.s.vmx.pRealModeTSS))
2722 {
2723 LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2724 return VERR_INTERNAL_ERROR;
2725 }
2726
2727 /* Initialize these always, see hmR3InitFinalizeR0().*/
2728 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2729 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2730
2731 /* Setup the tagged-TLB flush handlers. */
2732 int rc = hmR0VmxSetupTaggedTlb(pVM);
2733 if (RT_FAILURE(rc))
2734 {
2735 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2736 return rc;
2737 }
2738
2739 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2740 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2741#if HC_ARCH_BITS == 64
2742 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2743 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2744 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2745 {
2746 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2747 }
2748#endif
2749
2750 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2751 RTCCUINTREG uHostCR4 = ASMGetCR4();
2752 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2753 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2754
2755 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2756 {
2757 PVMCPU pVCpu = &pVM->aCpus[i];
2758 AssertPtr(pVCpu);
2759 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2760
2761 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2762 Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2763
2764 /* Set revision dword at the beginning of the VMCS structure. */
2765 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2766
2767 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2768 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2769 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2770 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2771
2772 /* Load this VMCS as the current VMCS. */
2773 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2774 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2775 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2776
2777 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2778 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2779 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2780
2781 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2782 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2783 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2784
2785 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2786 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2787 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2788
2789 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2790 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2791 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2792
2793#if HC_ARCH_BITS == 32
2794 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2795 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2796 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2797#endif
2798
2799 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2800 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2801 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2802 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2803
2804 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2805
2806 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2807 }
2808
2809 return VINF_SUCCESS;
2810}
2811
2812
2813/**
2814 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2815 * the VMCS.
2816 *
2817 * @returns VBox status code.
2818 */
2819static int hmR0VmxExportHostControlRegs(void)
2820{
2821 RTCCUINTREG uReg = ASMGetCR0();
2822 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2823 AssertRCReturn(rc, rc);
2824
2825 uReg = ASMGetCR3();
2826 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2827 AssertRCReturn(rc, rc);
2828
2829 uReg = ASMGetCR4();
2830 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2831 AssertRCReturn(rc, rc);
2832 return rc;
2833}
2834
2835
2836/**
2837 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2838 * the host-state area in the VMCS.
2839 *
2840 * @returns VBox status code.
2841 * @param pVCpu The cross context virtual CPU structure.
2842 */
2843static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
2844{
2845#if HC_ARCH_BITS == 64
2846/**
2847 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2848 * requirements. See hmR0VmxExportHostSegmentRegs().
2849 */
2850# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2851 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2852 { \
2853 bool fValidSelector = true; \
2854 if ((selValue) & X86_SEL_LDT) \
2855 { \
2856 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2857 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2858 } \
2859 if (fValidSelector) \
2860 { \
2861 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2862 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2863 } \
2864 (selValue) = 0; \
2865 }
2866
2867 /*
2868 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2869 * should -not- save the messed up state without restoring the original host-state,
2870 * see @bugref{7240}.
2871 *
2872 * This apparently can happen (most likely the FPU changes), deal with it rather than
2873 * asserting. Was observed booting Solaris 10u10 32-bit guest.
2874 */
2875 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2876 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2877 {
2878 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2879 pVCpu->idCpu));
2880 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2881 }
2882 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2883#else
2884 RT_NOREF(pVCpu);
2885#endif
2886
2887 /*
2888 * Host DS, ES, FS and GS segment registers.
2889 */
2890#if HC_ARCH_BITS == 64
2891 RTSEL uSelDS = ASMGetDS();
2892 RTSEL uSelES = ASMGetES();
2893 RTSEL uSelFS = ASMGetFS();
2894 RTSEL uSelGS = ASMGetGS();
2895#else
2896 RTSEL uSelDS = 0;
2897 RTSEL uSelES = 0;
2898 RTSEL uSelFS = 0;
2899 RTSEL uSelGS = 0;
2900#endif
2901
2902 /*
2903 * Host CS and SS segment registers.
2904 */
2905 RTSEL uSelCS = ASMGetCS();
2906 RTSEL uSelSS = ASMGetSS();
2907
2908 /*
2909 * Host TR segment register.
2910 */
2911 RTSEL uSelTR = ASMGetTR();
2912
2913#if HC_ARCH_BITS == 64
2914 /*
2915 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
2916 * gain VM-entry and restore them before we get preempted.
2917 *
2918 * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2919 */
2920 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2921 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2922 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2923 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2924# undef VMXLOCAL_ADJUST_HOST_SEG
2925#endif
2926
2927 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2928 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2929 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2930 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2931 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2932 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2933 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2934 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2935 Assert(uSelCS);
2936 Assert(uSelTR);
2937
2938 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2939#if 0
2940 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2941 Assert(uSelSS != 0);
2942#endif
2943
2944 /* Write these host selector fields into the host-state area in the VMCS. */
2945 int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2946 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2947#if HC_ARCH_BITS == 64
2948 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2949 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2950 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2951 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2952#else
2953 NOREF(uSelDS);
2954 NOREF(uSelES);
2955 NOREF(uSelFS);
2956 NOREF(uSelGS);
2957#endif
2958 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2959 AssertRCReturn(rc, rc);
2960
2961 /*
2962 * Host GDTR and IDTR.
2963 */
2964 RTGDTR Gdtr;
2965 RTIDTR Idtr;
2966 RT_ZERO(Gdtr);
2967 RT_ZERO(Idtr);
2968 ASMGetGDTR(&Gdtr);
2969 ASMGetIDTR(&Idtr);
2970 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2971 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2972 AssertRCReturn(rc, rc);
2973
2974#if HC_ARCH_BITS == 64
2975 /*
2976 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
2977 * them to the maximum limit (0xffff) on every VM-exit.
2978 */
2979 if (Gdtr.cbGdt != 0xffff)
2980 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2981
2982 /*
2983 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
2984 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the
2985 * limit as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU
2986 * behavior. However, several hosts either insists on 0xfff being the limit (Windows
2987 * Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
2988 * but botches sidt alignment in at least one consumer). So, we're only allowing the
2989 * IDTR.LIMIT to be left at 0xffff on hosts where we are sure it won't cause trouble.
2990 */
2991# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
2992 if (Idtr.cbIdt < 0x0fff)
2993# else
2994 if (Idtr.cbIdt != 0xffff)
2995# endif
2996 {
2997 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
2998 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2999 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3000 }
3001#endif
3002
3003 /*
3004 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3005 * and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3006 * RPL should be too in most cases.
3007 */
3008 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3009 ("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3010
3011 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3012#if HC_ARCH_BITS == 64
3013 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3014
3015 /*
3016 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3017 * all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3018 * restoration if the host has something else. Task switching is not supported in 64-bit
3019 * mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3020 * limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3021 *
3022 * [1] See Intel spec. 3.5 "System Descriptor Types".
3023 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3024 */
3025 PVM pVM = pVCpu->CTX_SUFF(pVM);
3026 Assert(pDesc->System.u4Type == 11);
3027 if ( pDesc->System.u16LimitLow != 0x67
3028 || pDesc->System.u4LimitHigh)
3029 {
3030 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3031 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3032 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3033 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3034 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3035 }
3036
3037 /*
3038 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3039 */
3040 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3041 {
3042 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3043 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3044 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3045 {
3046 /* The GDT is read-only but the writable GDT is available. */
3047 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3048 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3049 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3050 AssertRCReturn(rc, rc);
3051 }
3052 }
3053#else
3054 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3055#endif
3056 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3057 AssertRCReturn(rc, rc);
3058
3059 /*
3060 * Host FS base and GS base.
3061 */
3062#if HC_ARCH_BITS == 64
3063 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3064 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3065 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3066 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3067 AssertRCReturn(rc, rc);
3068
3069 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3070 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3071 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3072 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3073 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3074#endif
3075 return VINF_SUCCESS;
3076}
3077
3078
3079/**
3080 * Exports certain host MSRs in the VM-exit MSR-load area and some in the
3081 * host-state area of the VMCS.
3082 *
3083 * Theses MSRs will be automatically restored on the host after every successful
3084 * VM-exit.
3085 *
3086 * @returns VBox status code.
3087 * @param pVCpu The cross context virtual CPU structure.
3088 *
3089 * @remarks No-long-jump zone!!!
3090 */
3091static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
3092{
3093 AssertPtr(pVCpu);
3094 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3095
3096 /*
3097 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3098 * rather than swapping them on every VM-entry.
3099 */
3100 hmR0VmxLazySaveHostMsrs(pVCpu);
3101
3102 /*
3103 * Host Sysenter MSRs.
3104 */
3105 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3106#if HC_ARCH_BITS == 32
3107 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3108 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3109#else
3110 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3111 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3112#endif
3113 AssertRCReturn(rc, rc);
3114
3115 /*
3116 * Host EFER MSR.
3117 *
3118 * If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3119 * done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3120 */
3121 PVM pVM = pVCpu->CTX_SUFF(pVM);
3122 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3123 {
3124 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3125 AssertRCReturn(rc, rc);
3126 }
3127
3128 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see hmR0VmxExportGuestExitCtls(). */
3129
3130 return VINF_SUCCESS;
3131}
3132
3133
3134/**
3135 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3136 *
3137 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3138 * these two bits are handled by VM-entry, see hmR0VmxExportGuestExitCtls() and
3139 * hmR0VMxExportGuestEntryCtls().
3140 *
3141 * @returns true if we need to load guest EFER, false otherwise.
3142 * @param pVCpu The cross context virtual CPU structure.
3143 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3144 * out-of-sync. Make sure to update the required fields
3145 * before using them.
3146 *
3147 * @remarks Requires EFER, CR4.
3148 * @remarks No-long-jump zone!!!
3149 */
3150static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3151{
3152#ifdef HMVMX_ALWAYS_SWAP_EFER
3153 return true;
3154#endif
3155
3156#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3157 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3158 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3159 return false;
3160#endif
3161
3162 PVM pVM = pVCpu->CTX_SUFF(pVM);
3163 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3164 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3165
3166 /*
3167 * For 64-bit guests, if EFER.SCE bit differs, we need to swap EFER to ensure that the
3168 * guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3169 */
3170 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3171 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3172 {
3173 return true;
3174 }
3175
3176 /*
3177 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3178 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3179 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3180 */
3181 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3182 && (pMixedCtx->cr0 & X86_CR0_PG)
3183 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3184 {
3185 /* Assert that host is PAE capable. */
3186 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3187 return true;
3188 }
3189
3190 return false;
3191}
3192
3193
3194/**
3195 * Exports the guest state with appropriate VM-entry controls in the VMCS.
3196 *
3197 * These controls can affect things done on VM-exit; e.g. "load debug controls",
3198 * see Intel spec. 24.8.1 "VM-entry controls".
3199 *
3200 * @returns VBox status code.
3201 * @param pVCpu The cross context virtual CPU structure.
3202 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3203 * out-of-sync. Make sure to update the required fields
3204 * before using them.
3205 *
3206 * @remarks Requires EFER.
3207 * @remarks No-long-jump zone!!!
3208 */
3209static int hmR0VmxExportGuestEntryCtls(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3210{
3211 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_CTLS)
3212 {
3213 PVM pVM = pVCpu->CTX_SUFF(pVM);
3214 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3215 uint32_t fZap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3216
3217 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3218 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3219
3220 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3221 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3222 {
3223 fVal |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3224 Log4Func(("VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n"));
3225 }
3226 else
3227 Assert(!(fVal & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3228
3229 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3230 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3231 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3232 {
3233 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3234 Log4Func(("VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n"));
3235 }
3236
3237 /*
3238 * The following should -not- be set (since we're not in SMM mode):
3239 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3240 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3241 */
3242
3243 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3244 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3245
3246 if ((fVal & fZap) != fVal)
3247 {
3248 Log4Func(("Invalid VM-entry controls combo! Cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3249 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, fVal, fZap));
3250 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3251 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3252 }
3253
3254 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
3255 AssertRCReturn(rc, rc);
3256
3257 pVCpu->hm.s.vmx.u32EntryCtls = fVal;
3258 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_CTLS);
3259 }
3260 return VINF_SUCCESS;
3261}
3262
3263
3264/**
3265 * Exports the guest state with appropriate VM-exit controls in the VMCS.
3266 *
3267 * @returns VBox status code.
3268 * @param pVCpu The cross context virtual CPU structure.
3269 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3270 * out-of-sync. Make sure to update the required fields
3271 * before using them.
3272 *
3273 * @remarks Requires EFER.
3274 */
3275static int hmR0VmxExportGuestExitCtls(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3276{
3277 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_EXIT_CTLS)
3278 {
3279 PVM pVM = pVCpu->CTX_SUFF(pVM);
3280 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3281 uint32_t fZap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3282
3283 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3284 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3285
3286 /*
3287 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3288 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in
3289 * hmR0VmxExportHostMsrs().
3290 */
3291#if HC_ARCH_BITS == 64
3292 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3293 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3294#else
3295 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3296 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3297 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3298 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3299 {
3300 /* The switcher returns to long mode, EFER is managed by the switcher. */
3301 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3302 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3303 }
3304 else
3305 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3306#endif
3307
3308 /* If the newer VMCS fields for managing EFER exists, use it. */
3309 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3310 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3311 {
3312 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3313 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3314 Log4Func(("VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR and VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n"));
3315 }
3316
3317 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3318 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3319
3320 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3321 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3322 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3323
3324 if ( pVM->hm.s.vmx.fUsePreemptTimer
3325 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3326 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3327
3328 if ((fVal & fZap) != fVal)
3329 {
3330 LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3331 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, fVal, fZap));
3332 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3333 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3334 }
3335
3336 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
3337 AssertRCReturn(rc, rc);
3338
3339 pVCpu->hm.s.vmx.u32ExitCtls = fVal;
3340 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_EXIT_CTLS);
3341 }
3342 return VINF_SUCCESS;
3343}
3344
3345
3346/**
3347 * Sets the TPR threshold in the VMCS.
3348 *
3349 * @returns VBox status code.
3350 * @param pVCpu The cross context virtual CPU structure.
3351 * @param u32TprThreshold The TPR threshold (task-priority class only).
3352 */
3353DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3354{
3355 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3356 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3357 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3358}
3359
3360
3361/**
3362 * Exports the guest APIC TPR state into the VMCS.
3363 *
3364 * @returns VBox status code.
3365 * @param pVCpu The cross context virtual CPU structure.
3366 *
3367 * @remarks No-long-jump zone!!!
3368 */
3369static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu)
3370{
3371 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
3372 {
3373 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3374 && APICIsEnabled(pVCpu))
3375 {
3376 /*
3377 * Setup TPR shadowing.
3378 */
3379 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3380 {
3381 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3382
3383 bool fPendingIntr = false;
3384 uint8_t u8Tpr = 0;
3385 uint8_t u8PendingIntr = 0;
3386 int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3387 AssertRCReturn(rc, rc);
3388
3389 /*
3390 * If there are interrupts pending but masked by the TPR, instruct VT-x to
3391 * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
3392 * priority of the pending interrupt so we can deliver the interrupt. If there
3393 * are no interrupts pending, set threshold to 0 to not cause any
3394 * TPR-below-threshold VM-exits.
3395 */
3396 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3397 uint32_t u32TprThreshold = 0;
3398 if (fPendingIntr)
3399 {
3400 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3401 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3402 const uint8_t u8TprPriority = u8Tpr >> 4;
3403 if (u8PendingPriority <= u8TprPriority)
3404 u32TprThreshold = u8PendingPriority;
3405 }
3406
3407 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3408 AssertRCReturn(rc, rc);
3409 }
3410 }
3411 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
3412 }
3413 return VINF_SUCCESS;
3414}
3415
3416
3417/**
3418 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3419 *
3420 * @returns Guest's interruptibility-state.
3421 * @param pVCpu The cross context virtual CPU structure.
3422 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3423 * out-of-sync. Make sure to update the required fields
3424 * before using them.
3425 *
3426 * @remarks No-long-jump zone!!!
3427 */
3428static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3429{
3430 /*
3431 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3432 */
3433 uint32_t fIntrState = 0;
3434 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3435 {
3436 /* If inhibition is active, RIP & RFLAGS should've been accessed
3437 (i.e. read previously from the VMCS or from ring-3). */
3438#ifdef VBOX_STRICT
3439 uint64_t const fExtrn = ASMAtomicUoReadU64(&pMixedCtx->fExtrn);
3440 AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
3441#endif
3442 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3443 {
3444 if (pMixedCtx->eflags.Bits.u1IF)
3445 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3446 else
3447 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3448 }
3449 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3450 {
3451 /*
3452 * We can clear the inhibit force flag as even if we go back to the recompiler
3453 * without executing guest code in VT-x, the flag's condition to be cleared is
3454 * met and thus the cleared state is correct.
3455 */
3456 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3457 }
3458 }
3459
3460 /*
3461 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3462 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3463 * setting this would block host-NMIs and IRET will not clear the blocking.
3464 *
3465 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3466 */
3467 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3468 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3469 {
3470 fIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3471 }
3472
3473 return fIntrState;
3474}
3475
3476
3477/**
3478 * Exports the guest's interruptibility-state into the guest-state area in the
3479 * VMCS.
3480 *
3481 * @returns VBox status code.
3482 * @param pVCpu The cross context virtual CPU structure.
3483 * @param fIntrState The interruptibility-state to set.
3484 */
3485static int hmR0VmxExportGuestIntrState(PVMCPU pVCpu, uint32_t fIntrState)
3486{
3487 NOREF(pVCpu);
3488 AssertMsg(!(fIntrState & 0xfffffff0), ("%#x\n", fIntrState)); /* Bits 31:4 MBZ. */
3489 Assert((fIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3490 return VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, fIntrState);
3491}
3492
3493
3494/**
3495 * Exports the exception intercepts required for guest execution in the VMCS.
3496 *
3497 * @returns VBox status code.
3498 * @param pVCpu The cross context virtual CPU structure.
3499 *
3500 * @remarks No-long-jump zone!!!
3501 */
3502static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu)
3503{
3504 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
3505 {
3506 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxExportSharedCR0(). */
3507 if (pVCpu->hm.s.fGIMTrapXcptUD)
3508 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3509#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3510 else
3511 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3512#endif
3513
3514 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3515 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3516
3517 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3518 AssertRCReturn(rc, rc);
3519
3520 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3521 Log4Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64\n", pVCpu->hm.s.vmx.u32XcptBitmap));
3522 }
3523 return VINF_SUCCESS;
3524}
3525
3526
3527/**
3528 * Exports the guest's RIP into the guest-state area in the VMCS.
3529 *
3530 * @returns VBox status code.
3531 * @param pVCpu The cross context virtual CPU structure.
3532 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3533 * out-of-sync. Make sure to update the required fields
3534 * before using them.
3535 *
3536 * @remarks No-long-jump zone!!!
3537 */
3538static int hmR0VmxExportGuestRip(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3539{
3540 int rc = VINF_SUCCESS;
3541 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
3542 {
3543 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3544 AssertRCReturn(rc, rc);
3545
3546 /* Update the exit history entry with the correct CS.BASE + RIP or just RIP. */
3547 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
3548 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->cs.u64Base + pMixedCtx->rip, true);
3549 else
3550 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->rip, false);
3551
3552 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3553 Log4Func(("RIP=%#RX64\n", pMixedCtx->rip));
3554 }
3555 return rc;
3556}
3557
3558
3559/**
3560 * Exports the guest's RSP into the guest-state area in the VMCS.
3561 *
3562 * @returns VBox status code.
3563 * @param pVCpu The cross context virtual CPU structure.
3564 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3565 * out-of-sync. Make sure to update the required fields
3566 * before using them.
3567 *
3568 * @remarks No-long-jump zone!!!
3569 */
3570static int hmR0VmxExportGuestRsp(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3571{
3572 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3573 {
3574 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3575 AssertRCReturn(rc, rc);
3576
3577 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3578 }
3579 return VINF_SUCCESS;
3580}
3581
3582
3583/**
3584 * Exports the guest's RFLAGS into the guest-state area in the VMCS.
3585 *
3586 * @returns VBox status code.
3587 * @param pVCpu The cross context virtual CPU structure.
3588 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3589 * out-of-sync. Make sure to update the required fields
3590 * before using them.
3591 *
3592 * @remarks No-long-jump zone!!!
3593 */
3594static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3595{
3596 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3597 {
3598 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3599 Let us assert it as such and use 32-bit VMWRITE. */
3600 Assert(!(pMixedCtx->rflags.u64 >> 32));
3601 X86EFLAGS fEFlags = pMixedCtx->eflags;
3602 Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3603 Assert(!(fEFlags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3604
3605 /*
3606 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3607 * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3608 * can run the real-mode guest code under Virtual 8086 mode.
3609 */
3610 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3611 {
3612 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3613 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3614 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3615 fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3616 fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3617 }
3618
3619 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3620 AssertRCReturn(rc, rc);
3621
3622 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3623 Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
3624 }
3625 return VINF_SUCCESS;
3626}
3627
3628
3629/**
3630 * Exports the guest CR0 control register into the guest-state area in the VMCS.
3631 *
3632 * The guest FPU state is always pre-loaded hence we don't need to bother about
3633 * sharing FPU related CR0 bits between the guest and host.
3634 *
3635 * @returns VBox status code.
3636 * @param pVCpu The cross context virtual CPU structure.
3637 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3638 * out-of-sync. Make sure to update the required fields
3639 * before using them.
3640 *
3641 * @remarks No-long-jump zone!!!
3642 */
3643static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3644{
3645 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
3646 {
3647 PVM pVM = pVCpu->CTX_SUFF(pVM);
3648 Assert(!RT_HI_U32(pMixedCtx->cr0));
3649 uint32_t const uShadowCR0 = pMixedCtx->cr0;
3650 uint32_t uGuestCR0 = pMixedCtx->cr0;
3651
3652 /*
3653 * Setup VT-x's view of the guest CR0.
3654 * Minimize VM-exits due to CR3 changes when we have NestedPaging.
3655 */
3656 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
3657 if (pVM->hm.s.fNestedPaging)
3658 {
3659 if (CPUMIsGuestPagingEnabled(pVCpu))
3660 {
3661 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3662 uProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3663 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3664 }
3665 else
3666 {
3667 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3668 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3669 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3670 }
3671
3672 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3673 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3674 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3675 }
3676 else
3677 {
3678 /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3679 uGuestCR0 |= X86_CR0_WP;
3680 }
3681
3682 /*
3683 * Guest FPU bits.
3684 *
3685 * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3686 * using CR0.TS.
3687 *
3688 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3689 * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3690 */
3691 uGuestCR0 |= X86_CR0_NE;
3692
3693 /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3694 bool const fInterceptMF = !(uShadowCR0 & X86_CR0_NE);
3695
3696 /*
3697 * Update exception intercepts.
3698 */
3699 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3700 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3701 {
3702 Assert(PDMVmmDevHeapIsEnabled(pVM));
3703 Assert(pVM->hm.s.vmx.pRealModeTSS);
3704 uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3705 }
3706 else
3707 {
3708 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3709 uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3710 if (fInterceptMF)
3711 uXcptBitmap |= RT_BIT(X86_XCPT_MF);
3712 }
3713
3714 /* Additional intercepts for debugging, define these yourself explicitly. */
3715#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3716 uXcptBitmap |= 0
3717 | RT_BIT(X86_XCPT_BP)
3718 | RT_BIT(X86_XCPT_DE)
3719 | RT_BIT(X86_XCPT_NM)
3720 | RT_BIT(X86_XCPT_TS)
3721 | RT_BIT(X86_XCPT_UD)
3722 | RT_BIT(X86_XCPT_NP)
3723 | RT_BIT(X86_XCPT_SS)
3724 | RT_BIT(X86_XCPT_GP)
3725 | RT_BIT(X86_XCPT_PF)
3726 | RT_BIT(X86_XCPT_MF)
3727 ;
3728#elif defined(HMVMX_ALWAYS_TRAP_PF)
3729 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
3730#endif
3731 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3732 {
3733 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3734 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3735 }
3736 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3737
3738 /*
3739 * Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW).
3740 */
3741 uint32_t fSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3742 uint32_t fZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3743 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3744 fSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3745 else
3746 Assert((fSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3747
3748 uGuestCR0 |= fSetCR0;
3749 uGuestCR0 &= fZapCR0;
3750 uGuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3751
3752 /*
3753 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3754 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3755 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3756 */
3757 uint32_t uCR0Mask = X86_CR0_PE
3758 | X86_CR0_NE
3759 | (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
3760 | X86_CR0_PG
3761 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3762 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3763 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3764
3765 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3766 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3767 * and @bugref{6944}. */
3768#if 0
3769 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3770 uCr0Mask &= ~X86_CR0_PE;
3771#endif
3772 /* Update the HMCPU's copy of the CR0 mask. */
3773 pVCpu->hm.s.vmx.u32CR0Mask = uCR0Mask;
3774
3775 /*
3776 * Finally, update VMCS fields with the CR0 values.
3777 */
3778 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, uGuestCR0);
3779 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, uShadowCR0);
3780 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, uCR0Mask);
3781 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
3782 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3783 AssertRCReturn(rc, rc);
3784 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
3785
3786 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3787
3788 Log4Func(("uCr0Mask=%#RX32 uShadowCR0=%#RX32 uGuestCR0=%#RX32 (fSetCR0=%#RX32 fZapCR0=%#RX32\n", uCR0Mask, uShadowCR0,
3789 uGuestCR0, fSetCR0, fZapCR0));
3790 }
3791
3792 return VINF_SUCCESS;
3793}
3794
3795
3796/**
3797 * Exports the guest control registers (CR3, CR4) into the guest-state area
3798 * in the VMCS.
3799 *
3800 * @returns VBox strict status code.
3801 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3802 * without unrestricted guest access and the VMMDev is not presently
3803 * mapped (e.g. EFI32).
3804 *
3805 * @param pVCpu The cross context virtual CPU structure.
3806 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3807 * out-of-sync. Make sure to update the required fields
3808 * before using them.
3809 *
3810 * @remarks No-long-jump zone!!!
3811 */
3812static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3813{
3814 int rc = VINF_SUCCESS;
3815 PVM pVM = pVCpu->CTX_SUFF(pVM);
3816
3817 /*
3818 * Guest CR2.
3819 * It's always loaded in the assembler code. Nothing to do here.
3820 */
3821
3822 /*
3823 * Guest CR3.
3824 */
3825 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
3826 {
3827 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3828 if (pVM->hm.s.fNestedPaging)
3829 {
3830 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3831
3832 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3833 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3834 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3835 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3836
3837 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3838 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3839 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3840
3841 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3842 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3843 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3844 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3845 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3846 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3847 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3848
3849 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3850 AssertRCReturn(rc, rc);
3851
3852 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3853 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3854 {
3855 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3856 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3857 {
3858 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3859 AssertRCReturn(rc, rc);
3860 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3861 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3862 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3863 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3864 AssertRCReturn(rc, rc);
3865 }
3866
3867 /*
3868 * The guest's view of its CR3 is unblemished with Nested Paging when the
3869 * guest is using paging or we have unrestricted guest execution to handle
3870 * the guest when it's not using paging.
3871 */
3872 GCPhysGuestCR3 = pMixedCtx->cr3;
3873 }
3874 else
3875 {
3876 /*
3877 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
3878 * thinks it accesses physical memory directly, we use our identity-mapped
3879 * page table to map guest-linear to guest-physical addresses. EPT takes care
3880 * of translating it to host-physical addresses.
3881 */
3882 RTGCPHYS GCPhys;
3883 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3884
3885 /* We obtain it here every time as the guest could have relocated this PCI region. */
3886 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3887 if (RT_SUCCESS(rc))
3888 { /* likely */ }
3889 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3890 {
3891 Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3892 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3893 }
3894 else
3895 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3896
3897 GCPhysGuestCR3 = GCPhys;
3898 }
3899
3900 Log4Func(("uGuestCR3=%#RGp (GstN)\n", GCPhysGuestCR3));
3901 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3902 AssertRCReturn(rc, rc);
3903 }
3904 else
3905 {
3906 /* Non-nested paging case, just use the hypervisor's CR3. */
3907 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3908
3909 Log4Func(("uGuestCR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3910 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3911 AssertRCReturn(rc, rc);
3912 }
3913
3914 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3915 }
3916
3917 /*
3918 * Guest CR4.
3919 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3920 */
3921 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
3922 {
3923 Assert(!RT_HI_U32(pMixedCtx->cr4));
3924 uint32_t uGuestCR4 = pMixedCtx->cr4;
3925
3926 /* The guest's view of its CR4 is unblemished. */
3927 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, uGuestCR4);
3928 AssertRCReturn(rc, rc);
3929 Log4Func(("uShadowCR4=%#RX32\n", uGuestCR4));
3930
3931 /*
3932 * Setup VT-x's view of the guest CR4.
3933 *
3934 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3935 * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3936 * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3937 *
3938 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3939 */
3940 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3941 {
3942 Assert(pVM->hm.s.vmx.pRealModeTSS);
3943 Assert(PDMVmmDevHeapIsEnabled(pVM));
3944 uGuestCR4 &= ~X86_CR4_VME;
3945 }
3946
3947 if (pVM->hm.s.fNestedPaging)
3948 {
3949 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3950 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3951 {
3952 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3953 uGuestCR4 |= X86_CR4_PSE;
3954 /* Our identity mapping is a 32-bit page directory. */
3955 uGuestCR4 &= ~X86_CR4_PAE;
3956 }
3957 /* else use guest CR4.*/
3958 }
3959 else
3960 {
3961 /*
3962 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3963 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3964 */
3965 switch (pVCpu->hm.s.enmShadowMode)
3966 {
3967 case PGMMODE_REAL: /* Real-mode. */
3968 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3969 case PGMMODE_32_BIT: /* 32-bit paging. */
3970 {
3971 uGuestCR4 &= ~X86_CR4_PAE;
3972 break;
3973 }
3974
3975 case PGMMODE_PAE: /* PAE paging. */
3976 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3977 {
3978 uGuestCR4 |= X86_CR4_PAE;
3979 break;
3980 }
3981
3982 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3983 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3984#ifdef VBOX_ENABLE_64_BITS_GUESTS
3985 break;
3986#endif
3987 default:
3988 AssertFailed();
3989 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3990 }
3991 }
3992
3993 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3994 uint64_t fSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3995 uint64_t fZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3996 uGuestCR4 |= fSetCR4;
3997 uGuestCR4 &= fZapCR4;
3998
3999 /* Write VT-x's view of the guest CR4 into the VMCS. */
4000 Log4Func(("uGuestCR4=%#RX32 (fSetCR4=%#RX32 fZapCR4=%#RX32)\n", uGuestCR4, fSetCR4, fZapCR4));
4001 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, uGuestCR4);
4002 AssertRCReturn(rc, rc);
4003
4004 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4005 uint32_t u32CR4Mask = X86_CR4_VME
4006 | X86_CR4_PAE
4007 | X86_CR4_PGE
4008 | X86_CR4_PSE
4009 | X86_CR4_VMXE;
4010 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4011 u32CR4Mask |= X86_CR4_OSXSAVE;
4012 if (pVM->cpum.ro.GuestFeatures.fPcid)
4013 u32CR4Mask |= X86_CR4_PCIDE;
4014 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4015 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4016 AssertRCReturn(rc, rc);
4017
4018 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4019 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4020
4021 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
4022 }
4023 return rc;
4024}
4025
4026
4027/**
4028 * Exports the guest debug registers into the guest-state area in the VMCS.
4029 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4030 *
4031 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4032 *
4033 * @returns VBox status code.
4034 * @param pVCpu The cross context virtual CPU structure.
4035 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4036 * out-of-sync. Make sure to update the required fields
4037 * before using them.
4038 *
4039 * @remarks No-long-jump zone!!!
4040 */
4041static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4042{
4043 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4044
4045#ifdef VBOX_STRICT
4046 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4047 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4048 {
4049 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4050 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4051 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4052 }
4053#endif
4054
4055 bool fSteppingDB = false;
4056 bool fInterceptMovDRx = false;
4057 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
4058 if (pVCpu->hm.s.fSingleInstruction)
4059 {
4060 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4061 PVM pVM = pVCpu->CTX_SUFF(pVM);
4062 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4063 {
4064 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4065 Assert(fSteppingDB == false);
4066 }
4067 else
4068 {
4069 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4070 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_RFLAGS;
4071 pVCpu->hm.s.fClearTrapFlag = true;
4072 fSteppingDB = true;
4073 }
4074 }
4075
4076 uint32_t uGuestDR7;
4077 if ( fSteppingDB
4078 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4079 {
4080 /*
4081 * Use the combined guest and host DRx values found in the hypervisor register set
4082 * because the debugger has breakpoints active or someone is single stepping on the
4083 * host side without a monitor trap flag.
4084 *
4085 * Note! DBGF expects a clean DR6 state before executing guest code.
4086 */
4087#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4088 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4089 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4090 {
4091 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4092 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4093 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4094 }
4095 else
4096#endif
4097 if (!CPUMIsHyperDebugStateActive(pVCpu))
4098 {
4099 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4100 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4101 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4102 }
4103
4104 /* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
4105 uGuestDR7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
4106 pVCpu->hm.s.fUsingHyperDR7 = true;
4107 fInterceptMovDRx = true;
4108 }
4109 else
4110 {
4111 /*
4112 * If the guest has enabled debug registers, we need to load them prior to
4113 * executing guest code so they'll trigger at the right time.
4114 */
4115 if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
4116 {
4117#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4118 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4119 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4120 {
4121 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4122 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4123 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4124 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4125 }
4126 else
4127#endif
4128 if (!CPUMIsGuestDebugStateActive(pVCpu))
4129 {
4130 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4131 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4132 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4133 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4134 }
4135 Assert(!fInterceptMovDRx);
4136 }
4137 /*
4138 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4139 * must intercept #DB in order to maintain a correct DR6 guest value, and
4140 * because we need to intercept it to prevent nested #DBs from hanging the
4141 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4142 */
4143#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4144 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4145 && !CPUMIsGuestDebugStateActive(pVCpu))
4146#else
4147 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4148#endif
4149 {
4150 fInterceptMovDRx = true;
4151 }
4152
4153 /* Update DR7 with the actual guest value. */
4154 uGuestDR7 = pMixedCtx->dr[7];
4155 pVCpu->hm.s.fUsingHyperDR7 = false;
4156 }
4157
4158 if (fInterceptMovDRx)
4159 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4160 else
4161 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4162
4163 /*
4164 * Update the processor-based VM-execution controls for MOV-DRx intercepts and the monitor-trap flag.
4165 */
4166 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
4167 {
4168 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4169 AssertRCReturn(rc2, rc2);
4170 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
4171 }
4172
4173 /*
4174 * Update guest DR7.
4175 */
4176 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, uGuestDR7);
4177 AssertRCReturn(rc, rc);
4178
4179 return VINF_SUCCESS;
4180}
4181
4182
4183#ifdef VBOX_STRICT
4184/**
4185 * Strict function to validate segment registers.
4186 *
4187 * @remarks Will import guest CR0 on strict builds during validation of
4188 * segments.
4189 */
4190static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCCPUMCTX pCtx)
4191{
4192 /*
4193 * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
4194 *
4195 * The reason we check for attribute value 0 in this function and not just the unusable bit is
4196 * because hmR0VmxWriteSegmentReg() only updates the VMCS' copy of the value with the unusable bit
4197 * and doesn't change the guest-context value.
4198 */
4199 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
4200 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4201 && ( !CPUMIsGuestInRealModeEx(pCtx)
4202 && !CPUMIsGuestInV86ModeEx(pCtx)))
4203 {
4204 /* Protected mode checks */
4205 /* CS */
4206 Assert(pCtx->cs.Attr.n.u1Present);
4207 Assert(!(pCtx->cs.Attr.u & 0xf00));
4208 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4209 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4210 || !(pCtx->cs.Attr.n.u1Granularity));
4211 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4212 || (pCtx->cs.Attr.n.u1Granularity));
4213 /* CS cannot be loaded with NULL in protected mode. */
4214 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4215 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4216 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4217 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4218 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4219 else
4220 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4221 /* SS */
4222 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4223 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4224 if ( !(pCtx->cr0 & X86_CR0_PE)
4225 || pCtx->cs.Attr.n.u4Type == 3)
4226 {
4227 Assert(!pCtx->ss.Attr.n.u2Dpl);
4228 }
4229 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4230 {
4231 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4232 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4233 Assert(pCtx->ss.Attr.n.u1Present);
4234 Assert(!(pCtx->ss.Attr.u & 0xf00));
4235 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4236 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4237 || !(pCtx->ss.Attr.n.u1Granularity));
4238 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4239 || (pCtx->ss.Attr.n.u1Granularity));
4240 }
4241 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4242 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4243 {
4244 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4245 Assert(pCtx->ds.Attr.n.u1Present);
4246 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4247 Assert(!(pCtx->ds.Attr.u & 0xf00));
4248 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4249 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4250 || !(pCtx->ds.Attr.n.u1Granularity));
4251 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4252 || (pCtx->ds.Attr.n.u1Granularity));
4253 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4254 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4255 }
4256 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4257 {
4258 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4259 Assert(pCtx->es.Attr.n.u1Present);
4260 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4261 Assert(!(pCtx->es.Attr.u & 0xf00));
4262 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4263 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4264 || !(pCtx->es.Attr.n.u1Granularity));
4265 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4266 || (pCtx->es.Attr.n.u1Granularity));
4267 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4268 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4269 }
4270 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4271 {
4272 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4273 Assert(pCtx->fs.Attr.n.u1Present);
4274 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4275 Assert(!(pCtx->fs.Attr.u & 0xf00));
4276 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4277 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4278 || !(pCtx->fs.Attr.n.u1Granularity));
4279 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4280 || (pCtx->fs.Attr.n.u1Granularity));
4281 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4282 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4283 }
4284 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4285 {
4286 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4287 Assert(pCtx->gs.Attr.n.u1Present);
4288 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4289 Assert(!(pCtx->gs.Attr.u & 0xf00));
4290 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4291 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4292 || !(pCtx->gs.Attr.n.u1Granularity));
4293 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4294 || (pCtx->gs.Attr.n.u1Granularity));
4295 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4296 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4297 }
4298 /* 64-bit capable CPUs. */
4299# if HC_ARCH_BITS == 64
4300 Assert(!(pCtx->cs.u64Base >> 32));
4301 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4302 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4303 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4304# endif
4305 }
4306 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4307 || ( CPUMIsGuestInRealModeEx(pCtx)
4308 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4309 {
4310 /* Real and v86 mode checks. */
4311 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4312 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4313 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4314 {
4315 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4316 }
4317 else
4318 {
4319 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4320 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4321 }
4322
4323 /* CS */
4324 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4325 Assert(pCtx->cs.u32Limit == 0xffff);
4326 Assert(u32CSAttr == 0xf3);
4327 /* SS */
4328 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4329 Assert(pCtx->ss.u32Limit == 0xffff);
4330 Assert(u32SSAttr == 0xf3);
4331 /* DS */
4332 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4333 Assert(pCtx->ds.u32Limit == 0xffff);
4334 Assert(u32DSAttr == 0xf3);
4335 /* ES */
4336 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4337 Assert(pCtx->es.u32Limit == 0xffff);
4338 Assert(u32ESAttr == 0xf3);
4339 /* FS */
4340 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4341 Assert(pCtx->fs.u32Limit == 0xffff);
4342 Assert(u32FSAttr == 0xf3);
4343 /* GS */
4344 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4345 Assert(pCtx->gs.u32Limit == 0xffff);
4346 Assert(u32GSAttr == 0xf3);
4347 /* 64-bit capable CPUs. */
4348# if HC_ARCH_BITS == 64
4349 Assert(!(pCtx->cs.u64Base >> 32));
4350 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4351 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4352 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4353# endif
4354 }
4355}
4356#endif /* VBOX_STRICT */
4357
4358
4359/**
4360 * Writes a guest segment register into the guest-state area in the VMCS.
4361 *
4362 * @returns VBox status code.
4363 * @param pVCpu The cross context virtual CPU structure.
4364 * @param idxSel Index of the selector in the VMCS.
4365 * @param idxLimit Index of the segment limit in the VMCS.
4366 * @param idxBase Index of the segment base in the VMCS.
4367 * @param idxAccess Index of the access rights of the segment in the VMCS.
4368 * @param pSelReg Pointer to the segment selector.
4369 *
4370 * @remarks No-long-jump zone!!!
4371 */
4372static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4373 uint32_t idxAccess, PCCPUMSELREG pSelReg)
4374{
4375 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4376 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4377 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4378 AssertRCReturn(rc, rc);
4379
4380 uint32_t u32Access = pSelReg->Attr.u;
4381 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4382 {
4383 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4384 u32Access = 0xf3;
4385 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4386 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4387 }
4388 else
4389 {
4390 /*
4391 * The way to differentiate between whether this is really a null selector or was just
4392 * a selector loaded with 0 in real-mode is using the segment attributes. A selector
4393 * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4394 * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4395 * NULL selectors loaded in protected-mode have their attribute as 0.
4396 */
4397 if (!u32Access)
4398 u32Access = X86DESCATTR_UNUSABLE;
4399 }
4400
4401 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4402 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4403 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4404
4405 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4406 AssertRCReturn(rc, rc);
4407 return rc;
4408}
4409
4410
4411/**
4412 * Exports the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4413 * into the guest-state area in the VMCS.
4414 *
4415 * @returns VBox status code.
4416 * @param pVCpu The cross context virtual CPU structure.
4417 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4418 * out-of-sync. Make sure to update the required fields
4419 * before using them.
4420 *
4421 * @remarks Will import guest CR0 on strict builds during validation of
4422 * segments.
4423 * @remarks No-long-jump zone!!!
4424 */
4425static int hmR0VmxExportGuestSegmentRegs(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4426{
4427 int rc = VERR_INTERNAL_ERROR_5;
4428 PVM pVM = pVCpu->CTX_SUFF(pVM);
4429
4430 /*
4431 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4432 */
4433 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4434 {
4435 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4436 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4437 {
4438 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4439 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4440 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4441 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4442 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4443 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4444 }
4445
4446#ifdef VBOX_WITH_REM
4447 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4448 {
4449 Assert(pVM->hm.s.vmx.pRealModeTSS);
4450 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4451 if ( pVCpu->hm.s.vmx.fWasInRealMode
4452 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4453 {
4454 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4455 in real-mode (e.g. OpenBSD 4.0) */
4456 REMFlushTBs(pVM);
4457 Log4Func(("Switch to protected mode detected!\n"));
4458 pVCpu->hm.s.vmx.fWasInRealMode = false;
4459 }
4460 }
4461#endif
4462 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4463 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4464 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4465 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4466 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4467 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4468 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4469 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4470 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4471 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4472 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4473 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4474 AssertRCReturn(rc, rc);
4475
4476#ifdef VBOX_STRICT
4477 /* Validate. */
4478 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4479#endif
4480
4481 /* Update the exit history entry with the correct CS.BASE + RIP. */
4482 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4483 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->cs.u64Base + pMixedCtx->rip, true);
4484
4485 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SREG_MASK);
4486 Log4Func(("CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4487 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4488 }
4489
4490 /*
4491 * Guest TR.
4492 */
4493 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
4494 {
4495 /*
4496 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4497 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4498 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4499 */
4500 uint16_t u16Sel = 0;
4501 uint32_t u32Limit = 0;
4502 uint64_t u64Base = 0;
4503 uint32_t u32AccessRights = 0;
4504
4505 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4506 {
4507 u16Sel = pMixedCtx->tr.Sel;
4508 u32Limit = pMixedCtx->tr.u32Limit;
4509 u64Base = pMixedCtx->tr.u64Base;
4510 u32AccessRights = pMixedCtx->tr.Attr.u;
4511 }
4512 else
4513 {
4514 Assert(pVM->hm.s.vmx.pRealModeTSS);
4515 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4516
4517 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4518 RTGCPHYS GCPhys;
4519 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4520 AssertRCReturn(rc, rc);
4521
4522 X86DESCATTR DescAttr;
4523 DescAttr.u = 0;
4524 DescAttr.n.u1Present = 1;
4525 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4526
4527 u16Sel = 0;
4528 u32Limit = HM_VTX_TSS_SIZE;
4529 u64Base = GCPhys; /* in real-mode phys = virt. */
4530 u32AccessRights = DescAttr.u;
4531 }
4532
4533 /* Validate. */
4534 Assert(!(u16Sel & RT_BIT(2)));
4535 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4536 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4537 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4538 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4539 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4540 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4541 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4542 Assert( (u32Limit & 0xfff) == 0xfff
4543 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4544 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4545 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4546
4547 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4548 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4549 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4550 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4551 AssertRCReturn(rc, rc);
4552
4553 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
4554 Log4Func(("TR base=%#RX64\n", pMixedCtx->tr.u64Base));
4555 }
4556
4557 /*
4558 * Guest GDTR.
4559 */
4560 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4561 {
4562 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4563 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4564 AssertRCReturn(rc, rc);
4565
4566 /* Validate. */
4567 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4568
4569 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4570 Log4Func(("GDTR base=%#RX64\n", pMixedCtx->gdtr.pGdt));
4571 }
4572
4573 /*
4574 * Guest LDTR.
4575 */
4576 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4577 {
4578 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4579 uint32_t u32Access = 0;
4580 if (!pMixedCtx->ldtr.Attr.u)
4581 u32Access = X86DESCATTR_UNUSABLE;
4582 else
4583 u32Access = pMixedCtx->ldtr.Attr.u;
4584
4585 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4586 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4587 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4588 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4589 AssertRCReturn(rc, rc);
4590
4591 /* Validate. */
4592 if (!(u32Access & X86DESCATTR_UNUSABLE))
4593 {
4594 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4595 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4596 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4597 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4598 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4599 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4600 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4601 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4602 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4603 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4604 }
4605
4606 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4607 Log4Func(("LDTR base=%#RX64\n", pMixedCtx->ldtr.u64Base));
4608 }
4609
4610 /*
4611 * Guest IDTR.
4612 */
4613 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4614 {
4615 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4616 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4617 AssertRCReturn(rc, rc);
4618
4619 /* Validate. */
4620 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4621
4622 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4623 Log4Func(("IDTR base=%#RX64\n", pMixedCtx->idtr.pIdt));
4624 }
4625
4626 return VINF_SUCCESS;
4627}
4628
4629
4630/**
4631 * Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4632 * areas.
4633 *
4634 * These MSRs will automatically be loaded to the host CPU on every successful
4635 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4636 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4637 * -not- updated here for performance reasons. See hmR0VmxExportHostMsrs().
4638 *
4639 * Also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4640 *
4641 * @returns VBox status code.
4642 * @param pVCpu The cross context virtual CPU structure.
4643 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4644 * out-of-sync. Make sure to update the required fields
4645 * before using them.
4646 *
4647 * @remarks No-long-jump zone!!!
4648 */
4649static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4650{
4651 AssertPtr(pVCpu);
4652 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4653
4654 /*
4655 * MSRs that we use the auto-load/store MSR area in the VMCS.
4656 * For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
4657 */
4658 PVM pVM = pVCpu->CTX_SUFF(pVM);
4659 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4660 {
4661 if (pVM->hm.s.fAllow64BitGuests)
4662 {
4663#if HC_ARCH_BITS == 32
4664 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4665 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4666 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4667 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4668 AssertRCReturn(rc, rc);
4669# ifdef LOG_ENABLED
4670 PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4671 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4672 Log4Func(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4673# endif
4674#endif
4675 }
4676 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4677 }
4678
4679 /*
4680 * Guest Sysenter MSRs.
4681 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4682 * VM-exits on WRMSRs for these MSRs.
4683 */
4684 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4685 {
4686 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4687 {
4688 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs);
4689 AssertRCReturn(rc, rc);
4690 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4691 }
4692
4693 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4694 {
4695 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip);
4696 AssertRCReturn(rc, rc);
4697 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4698 }
4699
4700 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4701 {
4702 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp);
4703 AssertRCReturn(rc, rc);
4704 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4705 }
4706 }
4707
4708 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4709 {
4710 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4711 {
4712 /*
4713 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4714 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4715 */
4716 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4717 {
4718 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4719 AssertRCReturn(rc,rc);
4720 Log4Func(("EFER=%#RX64\n", pMixedCtx->msrEFER));
4721 }
4722 else
4723 {
4724 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4725 NULL /* pfAddedAndUpdated */);
4726 AssertRCReturn(rc, rc);
4727
4728 /* We need to intercept reads too, see @bugref{7386#c16}. */
4729 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4730 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4731 Log4Func(("MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pMixedCtx->msrEFER,
4732 pVCpu->hm.s.vmx.cMsrs));
4733 }
4734 }
4735 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4736 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4737 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4738 }
4739
4740 return VINF_SUCCESS;
4741}
4742
4743
4744#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4745/**
4746 * Check if guest state allows safe use of 32-bit switcher again.
4747 *
4748 * Segment bases and protected mode structures must be 32-bit addressable
4749 * because the 32-bit switcher will ignore high dword when writing these VMCS
4750 * fields. See @bugref{8432} for details.
4751 *
4752 * @returns true if safe, false if must continue to use the 64-bit switcher.
4753 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4754 * out-of-sync. Make sure to update the required fields
4755 * before using them.
4756 *
4757 * @remarks No-long-jump zone!!!
4758 */
4759static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pMixedCtx)
4760{
4761 if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4762 if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4763 if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4764 if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4765 if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4766 if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4767 if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4768 if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4769 if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4770 if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4771
4772 /* All good, bases are 32-bit. */
4773 return true;
4774}
4775#endif
4776
4777
4778/**
4779 * Selects up the appropriate function to run guest code.
4780 *
4781 * @returns VBox status code.
4782 * @param pVCpu The cross context virtual CPU structure.
4783 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4784 * out-of-sync. Make sure to update the required fields
4785 * before using them.
4786 *
4787 * @remarks No-long-jump zone!!!
4788 */
4789static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4790{
4791 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4792 {
4793#ifndef VBOX_ENABLE_64_BITS_GUESTS
4794 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4795#endif
4796 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4797#if HC_ARCH_BITS == 32
4798 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4799 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4800 {
4801#ifdef VBOX_STRICT
4802 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4803 {
4804 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4805 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4806 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4807 | HM_CHANGED_VMX_ENTRY_CTLS
4808 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4809 }
4810#endif
4811 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4812
4813 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4814 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4815 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4816 Log4Func(("Selected 64-bit switcher\n"));
4817 }
4818#else
4819 /* 64-bit host. */
4820 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4821#endif
4822 }
4823 else
4824 {
4825 /* Guest is not in long mode, use the 32-bit handler. */
4826#if HC_ARCH_BITS == 32
4827 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4828 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4829 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4830 {
4831# ifdef VBOX_STRICT
4832 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4833 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4834 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4835 | HM_CHANGED_VMX_ENTRY_CTLS
4836 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4837# endif
4838 }
4839# ifdef VBOX_ENABLE_64_BITS_GUESTS
4840 /*
4841 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
4842 * design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
4843 * switcher flag because now we know the guest is in a sane state where it's safe
4844 * to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4845 * the much faster 32-bit switcher again.
4846 */
4847 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4848 {
4849 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4850 Log4Func(("Selected 32-bit switcher\n"));
4851 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4852 }
4853 else
4854 {
4855 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4856 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4857 || hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
4858 {
4859 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4860 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4861 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
4862 | HM_CHANGED_VMX_ENTRY_CTLS
4863 | HM_CHANGED_VMX_EXIT_CTLS
4864 | HM_CHANGED_HOST_CONTEXT);
4865 Log4Func(("Selected 32-bit switcher (safe)\n"));
4866 }
4867 }
4868# else
4869 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4870# endif
4871#else
4872 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4873#endif
4874 }
4875 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4876 return VINF_SUCCESS;
4877}
4878
4879
4880/**
4881 * Wrapper for running the guest code in VT-x.
4882 *
4883 * @returns VBox status code, no informational status codes.
4884 * @param pVM The cross context VM structure.
4885 * @param pVCpu The cross context virtual CPU structure.
4886 * @param pCtx Pointer to the guest-CPU context.
4887 *
4888 * @remarks No-long-jump zone!!!
4889 */
4890DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4891{
4892 /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4893 pCtx->fExtrn |= HMVMX_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
4894
4895 /*
4896 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4897 * floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
4898 * callee-saved and thus the need for this XMM wrapper.
4899 *
4900 * See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
4901 */
4902 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4903 /** @todo Add stats for resume vs launch. */
4904#ifdef VBOX_WITH_KERNEL_USING_XMM
4905 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4906#else
4907 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4908#endif
4909 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4910 return rc;
4911}
4912
4913
4914/**
4915 * Reports world-switch error and dumps some useful debug info.
4916 *
4917 * @param pVM The cross context VM structure.
4918 * @param pVCpu The cross context virtual CPU structure.
4919 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4920 * @param pCtx Pointer to the guest-CPU context.
4921 * @param pVmxTransient Pointer to the VMX transient structure (only
4922 * exitReason updated).
4923 */
4924static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4925{
4926 Assert(pVM);
4927 Assert(pVCpu);
4928 Assert(pCtx);
4929 Assert(pVmxTransient);
4930 HMVMX_ASSERT_PREEMPT_SAFE();
4931
4932 Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
4933 switch (rcVMRun)
4934 {
4935 case VERR_VMX_INVALID_VMXON_PTR:
4936 AssertFailed();
4937 break;
4938 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4939 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4940 {
4941 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4942 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4943 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4944 AssertRC(rc);
4945
4946 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4947 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4948 Cannot do it here as we may have been long preempted. */
4949
4950#ifdef VBOX_STRICT
4951 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4952 pVmxTransient->uExitReason));
4953 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4954 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4955 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4956 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4957 else
4958 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4959 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4960 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4961
4962 /* VMX control bits. */
4963 uint32_t u32Val;
4964 uint64_t u64Val;
4965 RTHCUINTREG uHCReg;
4966 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4967 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4968 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4969 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4970 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
4971 {
4972 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4973 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4974 }
4975 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4976 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4977 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4978 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4979 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4980 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4981 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4982 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4983 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4984 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4985 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4986 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4987 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4988 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4989 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4990 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4991 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4992 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4993 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4994 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4995 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4996 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4997 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4998 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4999 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5000 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5001 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5002 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5003 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5004 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5005 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5006 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5007 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5008 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5009 if (pVM->hm.s.fNestedPaging)
5010 {
5011 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5012 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5013 }
5014
5015 /* Guest bits. */
5016 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5017 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5018 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5019 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5020 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5021 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5022 if (pVM->hm.s.vmx.fVpid)
5023 {
5024 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5025 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5026 }
5027
5028 /* Host bits. */
5029 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5030 Log4(("Host CR0 %#RHr\n", uHCReg));
5031 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5032 Log4(("Host CR3 %#RHr\n", uHCReg));
5033 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5034 Log4(("Host CR4 %#RHr\n", uHCReg));
5035
5036 RTGDTR HostGdtr;
5037 PCX86DESCHC pDesc;
5038 ASMGetGDTR(&HostGdtr);
5039 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5040 Log4(("Host CS %#08x\n", u32Val));
5041 if (u32Val < HostGdtr.cbGdt)
5042 {
5043 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5044 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5045 }
5046
5047 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5048 Log4(("Host DS %#08x\n", u32Val));
5049 if (u32Val < HostGdtr.cbGdt)
5050 {
5051 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5052 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5053 }
5054
5055 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5056 Log4(("Host ES %#08x\n", u32Val));
5057 if (u32Val < HostGdtr.cbGdt)
5058 {
5059 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5060 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5061 }
5062
5063 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5064 Log4(("Host FS %#08x\n", u32Val));
5065 if (u32Val < HostGdtr.cbGdt)
5066 {
5067 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5068 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5069 }
5070
5071 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5072 Log4(("Host GS %#08x\n", u32Val));
5073 if (u32Val < HostGdtr.cbGdt)
5074 {
5075 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5076 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5077 }
5078
5079 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5080 Log4(("Host SS %#08x\n", u32Val));
5081 if (u32Val < HostGdtr.cbGdt)
5082 {
5083 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5084 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5085 }
5086
5087 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5088 Log4(("Host TR %#08x\n", u32Val));
5089 if (u32Val < HostGdtr.cbGdt)
5090 {
5091 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5092 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5093 }
5094
5095 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5096 Log4(("Host TR Base %#RHv\n", uHCReg));
5097 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5098 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5099 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5100 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5101 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5102 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5103 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5104 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5105 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5106 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5107 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5108 Log4(("Host RSP %#RHv\n", uHCReg));
5109 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5110 Log4(("Host RIP %#RHv\n", uHCReg));
5111# if HC_ARCH_BITS == 64
5112 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5113 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5114 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5115 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5116 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5117 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5118# endif
5119#endif /* VBOX_STRICT */
5120 break;
5121 }
5122
5123 default:
5124 /* Impossible */
5125 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5126 break;
5127 }
5128 NOREF(pVM); NOREF(pCtx);
5129}
5130
5131
5132#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5133#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5134# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5135#endif
5136#ifdef VBOX_STRICT
5137static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5138{
5139 switch (idxField)
5140 {
5141 case VMX_VMCS_GUEST_RIP:
5142 case VMX_VMCS_GUEST_RSP:
5143 case VMX_VMCS_GUEST_SYSENTER_EIP:
5144 case VMX_VMCS_GUEST_SYSENTER_ESP:
5145 case VMX_VMCS_GUEST_GDTR_BASE:
5146 case VMX_VMCS_GUEST_IDTR_BASE:
5147 case VMX_VMCS_GUEST_CS_BASE:
5148 case VMX_VMCS_GUEST_DS_BASE:
5149 case VMX_VMCS_GUEST_ES_BASE:
5150 case VMX_VMCS_GUEST_FS_BASE:
5151 case VMX_VMCS_GUEST_GS_BASE:
5152 case VMX_VMCS_GUEST_SS_BASE:
5153 case VMX_VMCS_GUEST_LDTR_BASE:
5154 case VMX_VMCS_GUEST_TR_BASE:
5155 case VMX_VMCS_GUEST_CR3:
5156 return true;
5157 }
5158 return false;
5159}
5160
5161static bool hmR0VmxIsValidReadField(uint32_t idxField)
5162{
5163 switch (idxField)
5164 {
5165 /* Read-only fields. */
5166 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5167 return true;
5168 }
5169 /* Remaining readable fields should also be writable. */
5170 return hmR0VmxIsValidWriteField(idxField);
5171}
5172#endif /* VBOX_STRICT */
5173
5174
5175/**
5176 * Executes the specified handler in 64-bit mode.
5177 *
5178 * @returns VBox status code (no informational status codes).
5179 * @param pVCpu The cross context virtual CPU structure.
5180 * @param enmOp The operation to perform.
5181 * @param cParams Number of parameters.
5182 * @param paParam Array of 32-bit parameters.
5183 */
5184VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp,
5185 uint32_t cParams, uint32_t *paParam)
5186{
5187 PVM pVM = pVCpu->CTX_SUFF(pVM);
5188 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5189 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5190 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5191 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5192
5193#ifdef VBOX_STRICT
5194 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5195 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5196
5197 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5198 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5199#endif
5200
5201 /* Disable interrupts. */
5202 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5203
5204#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5205 RTCPUID idHostCpu = RTMpCpuId();
5206 CPUMR0SetLApic(pVCpu, idHostCpu);
5207#endif
5208
5209 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5210 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5211
5212 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5213 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5214 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5215
5216 /* Leave VMX Root Mode. */
5217 VMXDisable();
5218
5219 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5220
5221 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5222 CPUMSetHyperEIP(pVCpu, enmOp);
5223 for (int i = (int)cParams - 1; i >= 0; i--)
5224 CPUMPushHyper(pVCpu, paParam[i]);
5225
5226 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5227
5228 /* Call the switcher. */
5229 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5230 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5231
5232 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5233 /* Make sure the VMX instructions don't cause #UD faults. */
5234 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5235
5236 /* Re-enter VMX Root Mode */
5237 int rc2 = VMXEnable(HCPhysCpuPage);
5238 if (RT_FAILURE(rc2))
5239 {
5240 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5241 ASMSetFlags(fOldEFlags);
5242 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5243 return rc2;
5244 }
5245
5246 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5247 AssertRC(rc2);
5248 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5249 Assert(!(ASMGetFlags() & X86_EFL_IF));
5250 ASMSetFlags(fOldEFlags);
5251 return rc;
5252}
5253
5254
5255/**
5256 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5257 * supporting 64-bit guests.
5258 *
5259 * @returns VBox status code.
5260 * @param fResume Whether to VMLAUNCH or VMRESUME.
5261 * @param pCtx Pointer to the guest-CPU context.
5262 * @param pCache Pointer to the VMCS cache.
5263 * @param pVM The cross context VM structure.
5264 * @param pVCpu The cross context virtual CPU structure.
5265 */
5266DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5267{
5268 NOREF(fResume);
5269
5270 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5271 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5272
5273#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5274 pCache->uPos = 1;
5275 pCache->interPD = PGMGetInterPaeCR3(pVM);
5276 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5277#endif
5278
5279#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5280 pCache->TestIn.HCPhysCpuPage = 0;
5281 pCache->TestIn.HCPhysVmcs = 0;
5282 pCache->TestIn.pCache = 0;
5283 pCache->TestOut.HCPhysVmcs = 0;
5284 pCache->TestOut.pCache = 0;
5285 pCache->TestOut.pCtx = 0;
5286 pCache->TestOut.eflags = 0;
5287#else
5288 NOREF(pCache);
5289#endif
5290
5291 uint32_t aParam[10];
5292 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5293 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5294 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5295 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5296 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5297 aParam[5] = 0;
5298 aParam[6] = VM_RC_ADDR(pVM, pVM);
5299 aParam[7] = 0;
5300 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5301 aParam[9] = 0;
5302
5303#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5304 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5305 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5306#endif
5307 int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5308
5309#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5310 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5311 Assert(pCtx->dr[4] == 10);
5312 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5313#endif
5314
5315#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5316 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5317 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5318 pVCpu->hm.s.vmx.HCPhysVmcs));
5319 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5320 pCache->TestOut.HCPhysVmcs));
5321 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5322 pCache->TestOut.pCache));
5323 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5324 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5325 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5326 pCache->TestOut.pCtx));
5327 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5328#endif
5329 NOREF(pCtx);
5330 return rc;
5331}
5332
5333
5334/**
5335 * Initialize the VMCS-Read cache.
5336 *
5337 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5338 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5339 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5340 * (those that have a 32-bit FULL & HIGH part).
5341 *
5342 * @returns VBox status code.
5343 * @param pVM The cross context VM structure.
5344 * @param pVCpu The cross context virtual CPU structure.
5345 */
5346static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5347{
5348#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5349{ \
5350 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5351 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5352 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5353 ++cReadFields; \
5354}
5355
5356 AssertPtr(pVM);
5357 AssertPtr(pVCpu);
5358 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5359 uint32_t cReadFields = 0;
5360
5361 /*
5362 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5363 * and serve to indicate exceptions to the rules.
5364 */
5365
5366 /* Guest-natural selector base fields. */
5367#if 0
5368 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5369 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5370 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5371#endif
5372 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5373 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5374 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5375 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5376 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5377 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5378 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5379 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5380 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5381 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5382 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5383 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5384#if 0
5385 /* Unused natural width guest-state fields. */
5386 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5387 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5388#endif
5389 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5390 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5391
5392 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
5393 these 64-bit fields (using "FULL" and "HIGH" fields). */
5394#if 0
5395 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5396 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5397 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5398 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5399 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5400 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5401 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5402 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5403 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5404#endif
5405
5406 /* Natural width guest-state fields. */
5407 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5408#if 0
5409 /* Currently unused field. */
5410 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5411#endif
5412
5413 if (pVM->hm.s.fNestedPaging)
5414 {
5415 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5416 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5417 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5418 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5419 }
5420 else
5421 {
5422 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5423 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5424 }
5425
5426#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5427 return VINF_SUCCESS;
5428}
5429
5430
5431/**
5432 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5433 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5434 * darwin, running 64-bit guests).
5435 *
5436 * @returns VBox status code.
5437 * @param pVCpu The cross context virtual CPU structure.
5438 * @param idxField The VMCS field encoding.
5439 * @param u64Val 16, 32 or 64-bit value.
5440 */
5441VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5442{
5443 int rc;
5444 switch (idxField)
5445 {
5446 /*
5447 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5448 */
5449 /* 64-bit Control fields. */
5450 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5451 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5452 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5453 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5454 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5455 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5456 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5457 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5458 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5459 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5460 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5461 case VMX_VMCS64_CTRL_EPTP_FULL:
5462 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5463 /* 64-bit Guest-state fields. */
5464 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5465 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5466 case VMX_VMCS64_GUEST_PAT_FULL:
5467 case VMX_VMCS64_GUEST_EFER_FULL:
5468 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5469 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5470 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5471 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5472 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5473 /* 64-bit Host-state fields. */
5474 case VMX_VMCS64_HOST_PAT_FULL:
5475 case VMX_VMCS64_HOST_EFER_FULL:
5476 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5477 {
5478 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5479 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5480 break;
5481 }
5482
5483 /*
5484 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5485 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5486 */
5487 /* Natural-width Guest-state fields. */
5488 case VMX_VMCS_GUEST_CR3:
5489 case VMX_VMCS_GUEST_ES_BASE:
5490 case VMX_VMCS_GUEST_CS_BASE:
5491 case VMX_VMCS_GUEST_SS_BASE:
5492 case VMX_VMCS_GUEST_DS_BASE:
5493 case VMX_VMCS_GUEST_FS_BASE:
5494 case VMX_VMCS_GUEST_GS_BASE:
5495 case VMX_VMCS_GUEST_LDTR_BASE:
5496 case VMX_VMCS_GUEST_TR_BASE:
5497 case VMX_VMCS_GUEST_GDTR_BASE:
5498 case VMX_VMCS_GUEST_IDTR_BASE:
5499 case VMX_VMCS_GUEST_RSP:
5500 case VMX_VMCS_GUEST_RIP:
5501 case VMX_VMCS_GUEST_SYSENTER_ESP:
5502 case VMX_VMCS_GUEST_SYSENTER_EIP:
5503 {
5504 if (!(RT_HI_U32(u64Val)))
5505 {
5506 /* If this field is 64-bit, VT-x will zero out the top bits. */
5507 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5508 }
5509 else
5510 {
5511 /* Assert that only the 32->64 switcher case should ever come here. */
5512 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5513 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5514 }
5515 break;
5516 }
5517
5518 default:
5519 {
5520 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5521 rc = VERR_INVALID_PARAMETER;
5522 break;
5523 }
5524 }
5525 AssertRCReturn(rc, rc);
5526 return rc;
5527}
5528
5529
5530/**
5531 * Queue up a VMWRITE by using the VMCS write cache.
5532 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5533 *
5534 * @param pVCpu The cross context virtual CPU structure.
5535 * @param idxField The VMCS field encoding.
5536 * @param u64Val 16, 32 or 64-bit value.
5537 */
5538VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5539{
5540 AssertPtr(pVCpu);
5541 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5542
5543 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5544 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5545
5546 /* Make sure there are no duplicates. */
5547 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5548 {
5549 if (pCache->Write.aField[i] == idxField)
5550 {
5551 pCache->Write.aFieldVal[i] = u64Val;
5552 return VINF_SUCCESS;
5553 }
5554 }
5555
5556 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5557 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5558 pCache->Write.cValidEntries++;
5559 return VINF_SUCCESS;
5560}
5561#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5562
5563
5564/**
5565 * Sets up the usage of TSC-offsetting and updates the VMCS.
5566 *
5567 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5568 * VMX preemption timer.
5569 *
5570 * @returns VBox status code.
5571 * @param pVM The cross context VM structure.
5572 * @param pVCpu The cross context virtual CPU structure.
5573 *
5574 * @remarks No-long-jump zone!!!
5575 */
5576static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5577{
5578 int rc;
5579 bool fOffsettedTsc;
5580 bool fParavirtTsc;
5581 if (pVM->hm.s.vmx.fUsePreemptTimer)
5582 {
5583 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5584 &fOffsettedTsc, &fParavirtTsc);
5585
5586 /* Make sure the returned values have sane upper and lower boundaries. */
5587 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5588 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5589 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5590 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5591
5592 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5593 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5594 }
5595 else
5596 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5597
5598 /** @todo later optimize this to be done elsewhere and not before every
5599 * VM-entry. */
5600 if (fParavirtTsc)
5601 {
5602 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5603 information before every VM-entry, hence disable it for performance sake. */
5604#if 0
5605 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5606 AssertRC(rc);
5607#endif
5608 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5609 }
5610
5611 if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5612 {
5613 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5614 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5615
5616 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5617 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5618 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5619 }
5620 else
5621 {
5622 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5623 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5624 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5625 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5626 }
5627}
5628
5629
5630/**
5631 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5632 * VM-exit interruption info type.
5633 *
5634 * @returns The IEM exception flags.
5635 * @param uVector The event vector.
5636 * @param uVmxVectorType The VMX event type.
5637 *
5638 * @remarks This function currently only constructs flags required for
5639 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5640 * and CR2 aspects of an exception are not included).
5641 */
5642static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5643{
5644 uint32_t fIemXcptFlags;
5645 switch (uVmxVectorType)
5646 {
5647 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5648 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5649 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5650 break;
5651
5652 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5653 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5654 break;
5655
5656 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5657 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5658 break;
5659
5660 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5661 {
5662 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5663 if (uVector == X86_XCPT_BP)
5664 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5665 else if (uVector == X86_XCPT_OF)
5666 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5667 else
5668 {
5669 fIemXcptFlags = 0;
5670 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5671 }
5672 break;
5673 }
5674
5675 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5676 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5677 break;
5678
5679 default:
5680 fIemXcptFlags = 0;
5681 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5682 break;
5683 }
5684 return fIemXcptFlags;
5685}
5686
5687
5688/**
5689 * Sets an event as a pending event to be injected into the guest.
5690 *
5691 * @param pVCpu The cross context virtual CPU structure.
5692 * @param u32IntInfo The VM-entry interruption-information field.
5693 * @param cbInstr The VM-entry instruction length in bytes (for software
5694 * interrupts, exceptions and privileged software
5695 * exceptions).
5696 * @param u32ErrCode The VM-entry exception error code.
5697 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5698 * page-fault.
5699 *
5700 * @remarks Statistics counter assumes this is a guest event being injected or
5701 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5702 * always incremented.
5703 */
5704DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5705 RTGCUINTPTR GCPtrFaultAddress)
5706{
5707 Assert(!pVCpu->hm.s.Event.fPending);
5708 pVCpu->hm.s.Event.fPending = true;
5709 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5710 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5711 pVCpu->hm.s.Event.cbInstr = cbInstr;
5712 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5713}
5714
5715
5716/**
5717 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5718 *
5719 * @param pVCpu The cross context virtual CPU structure.
5720 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5721 * out-of-sync. Make sure to update the required fields
5722 * before using them.
5723 */
5724DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5725{
5726 NOREF(pMixedCtx);
5727 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5728 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5729 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5730 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5731}
5732
5733
5734/**
5735 * Handle a condition that occurred while delivering an event through the guest
5736 * IDT.
5737 *
5738 * @returns Strict VBox status code (i.e. informational status codes too).
5739 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5740 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5741 * to continue execution of the guest which will delivery the \#DF.
5742 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5743 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5744 *
5745 * @param pVCpu The cross context virtual CPU structure.
5746 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5747 * out-of-sync. Make sure to update the required fields
5748 * before using them.
5749 * @param pVmxTransient Pointer to the VMX transient structure.
5750 *
5751 * @remarks No-long-jump zone!!!
5752 */
5753static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5754{
5755 uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5756
5757 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5758 rc2 |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5759 AssertRCReturn(rc2, rc2);
5760
5761 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5762 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5763 {
5764 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5765 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5766
5767 /*
5768 * If the event was a software interrupt (generated with INT n) or a software exception
5769 * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
5770 * can handle the VM-exit and continue guest execution which will re-execute the
5771 * instruction rather than re-injecting the exception, as that can cause premature
5772 * trips to ring-3 before injection and involve TRPM which currently has no way of
5773 * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
5774 * the problem).
5775 */
5776 IEMXCPTRAISE enmRaise;
5777 IEMXCPTRAISEINFO fRaiseInfo;
5778 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5779 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5780 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5781 {
5782 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5783 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5784 }
5785 else if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5786 {
5787 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5788 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5789 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5790 /** @todo Make AssertMsgReturn as just AssertMsg later. */
5791 AssertMsgReturn(uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT,
5792 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5793 uExitVectorType), VERR_VMX_IPE_5);
5794
5795 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5796
5797 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5798 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
5799 {
5800 pVmxTransient->fVectoringPF = true;
5801 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5802 }
5803 }
5804 else
5805 {
5806 /*
5807 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5808 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5809 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
5810 */
5811 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5812 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5813 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5814 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5815 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5816 }
5817
5818 /*
5819 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5820 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5821 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5822 * subsequent VM-entry would fail.
5823 *
5824 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5825 */
5826 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5827 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5828 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
5829 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
5830 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5831 {
5832 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5833 }
5834
5835 switch (enmRaise)
5836 {
5837 case IEMXCPTRAISE_CURRENT_XCPT:
5838 {
5839 Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
5840 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
5841 Assert(rcStrict == VINF_SUCCESS);
5842 break;
5843 }
5844
5845 case IEMXCPTRAISE_PREV_EVENT:
5846 {
5847 uint32_t u32ErrCode;
5848 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5849 {
5850 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5851 AssertRCReturn(rc2, rc2);
5852 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5853 }
5854 else
5855 u32ErrCode = 0;
5856
5857 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
5858 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5859 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5860 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5861
5862 Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
5863 pVCpu->hm.s.Event.u32ErrCode));
5864 Assert(rcStrict == VINF_SUCCESS);
5865 break;
5866 }
5867
5868 case IEMXCPTRAISE_REEXEC_INSTR:
5869 Assert(rcStrict == VINF_SUCCESS);
5870 break;
5871
5872 case IEMXCPTRAISE_DOUBLE_FAULT:
5873 {
5874 /*
5875 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
5876 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
5877 */
5878 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
5879 {
5880 pVmxTransient->fVectoringDoublePF = true;
5881 Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
5882 pMixedCtx->cr2));
5883 rcStrict = VINF_SUCCESS;
5884 }
5885 else
5886 {
5887 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5888 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5889 Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
5890 uIdtVector, uExitVector));
5891 rcStrict = VINF_HM_DOUBLE_FAULT;
5892 }
5893 break;
5894 }
5895
5896 case IEMXCPTRAISE_TRIPLE_FAULT:
5897 {
5898 Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
5899 rcStrict = VINF_EM_RESET;
5900 break;
5901 }
5902
5903 case IEMXCPTRAISE_CPU_HANG:
5904 {
5905 Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
5906 rcStrict = VERR_EM_GUEST_CPU_HANG;
5907 break;
5908 }
5909
5910 default:
5911 {
5912 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
5913 rcStrict = VERR_VMX_IPE_2;
5914 break;
5915 }
5916 }
5917 }
5918 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5919 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5920 && uExitVector != X86_XCPT_DF
5921 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5922 {
5923 /*
5924 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5925 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5926 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5927 */
5928 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5929 {
5930 Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
5931 VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5932 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5933 }
5934 }
5935
5936 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
5937 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
5938 return rcStrict;
5939}
5940
5941
5942/**
5943 * Imports a guest segment register from the current VMCS into
5944 * the guest-CPU context.
5945 *
5946 * @returns VBox status code.
5947 * @param pVCpu The cross context virtual CPU structure.
5948 * @param idxSel Index of the selector in the VMCS.
5949 * @param idxLimit Index of the segment limit in the VMCS.
5950 * @param idxBase Index of the segment base in the VMCS.
5951 * @param idxAccess Index of the access rights of the segment in the VMCS.
5952 * @param pSelReg Pointer to the segment selector.
5953 *
5954 * @remarks No-long-jump zone!!!
5955 *
5956 * @remarks Never call this function directly!!! Use the
5957 * HMVMX_IMPORT_SREG() macro as that takes care
5958 * of whether to read from the VMCS cache or not.
5959 */
5960static int hmR0VmxImportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5961 PCPUMSELREG pSelReg)
5962{
5963 NOREF(pVCpu);
5964
5965 uint32_t u32Sel;
5966 uint32_t u32Limit;
5967 uint32_t u32Attr;
5968 uint64_t u64Base;
5969 int rc = VMXReadVmcs32(idxSel, &u32Sel);
5970 rc |= VMXReadVmcs32(idxLimit, &u32Limit);
5971 rc |= VMXReadVmcs32(idxAccess, &u32Attr);
5972 rc |= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
5973 AssertRCReturn(rc, rc);
5974
5975 pSelReg->Sel = (uint16_t)u32Sel;
5976 pSelReg->ValidSel = (uint16_t)u32Sel;
5977 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5978 pSelReg->u32Limit = u32Limit;
5979 pSelReg->u64Base = u64Base;
5980 pSelReg->Attr.u = u32Attr;
5981
5982 /*
5983 * If VT-x marks the segment as unusable, most other bits remain undefined:
5984 * - For CS the L, D and G bits have meaning.
5985 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
5986 * - For the remaining data segments no bits are defined.
5987 *
5988 * The present bit and the unusable bit has been observed to be set at the
5989 * same time (the selector was supposed to be invalid as we started executing
5990 * a V8086 interrupt in ring-0).
5991 *
5992 * What should be important for the rest of the VBox code, is that the P bit is
5993 * cleared. Some of the other VBox code recognizes the unusable bit, but
5994 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
5995 * safe side here, we'll strip off P and other bits we don't care about. If
5996 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
5997 *
5998 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5999 */
6000 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6001 {
6002 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6003
6004 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6005 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6006 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6007
6008 Log4Func(("Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Sel, pSelReg->Attr.u));
6009#ifdef DEBUG_bird
6010 AssertMsg((u32Attr & ~X86DESCATTR_P) == pSelReg->Attr.u,
6011 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6012 idxSel, u32Sel, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6013#endif
6014 }
6015 return VINF_SUCCESS;
6016}
6017
6018
6019/**
6020 * Imports the guest RIP from the VMCS back into the guest-CPU context.
6021 *
6022 * @returns VBox status code.
6023 * @param pVCpu The cross context virtual CPU structure.
6024 *
6025 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6026 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6027 * instead!!!
6028 */
6029DECLINLINE(int) hmR0VmxImportGuestRip(PVMCPU pVCpu)
6030{
6031 uint64_t u64Val;
6032 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6033 if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
6034 {
6035 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6036 if (RT_SUCCESS(rc))
6037 {
6038 pCtx->rip = u64Val;
6039 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
6040 }
6041 return rc;
6042 }
6043 return VINF_SUCCESS;
6044}
6045
6046
6047/**
6048 * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
6049 *
6050 * @returns VBox status code.
6051 * @param pVCpu The cross context virtual CPU structure.
6052 *
6053 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6054 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6055 * instead!!!
6056 */
6057DECLINLINE(int) hmR0VmxImportGuestRFlags(PVMCPU pVCpu)
6058{
6059 uint32_t u32Val;
6060 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6061 if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
6062 {
6063 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
6064 if (RT_SUCCESS(rc))
6065 {
6066 pCtx->eflags.u32 = u32Val;
6067
6068 /* Restore eflags for real-on-v86-mode hack. */
6069 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6070 {
6071 pCtx->eflags.Bits.u1VM = 0;
6072 pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6073 }
6074 }
6075 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
6076 return rc;
6077 }
6078 return VINF_SUCCESS;
6079}
6080
6081
6082/**
6083 * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
6084 * context.
6085 *
6086 * @returns VBox status code.
6087 * @param pVCpu The cross context virtual CPU structure.
6088 *
6089 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6090 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6091 * instead!!!
6092 */
6093DECLINLINE(int) hmR0VmxImportGuestIntrState(PVMCPU pVCpu)
6094{
6095 uint32_t u32Val;
6096 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6097 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32Val);
6098 if (RT_SUCCESS(rc))
6099 {
6100 /*
6101 * We additionally have a requirement to import RIP, RFLAGS depending on whether we
6102 * might need them in hmR0VmxEvaluatePendingEvent().
6103 */
6104 if (!u32Val)
6105 {
6106 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6107 {
6108 rc = hmR0VmxImportGuestRip(pVCpu);
6109 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6110 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6111 }
6112
6113 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6114 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6115 }
6116 else
6117 {
6118 rc = hmR0VmxImportGuestRip(pVCpu);
6119 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6120
6121 if (u32Val & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6122 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6123 {
6124 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
6125 }
6126 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6127 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6128
6129 if (u32Val & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6130 {
6131 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6132 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6133 }
6134 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6135 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6136 }
6137 }
6138 return rc;
6139}
6140
6141
6142/**
6143 * Worker for VMXR0ImportStateOnDemand.
6144 *
6145 * @returns VBox status code.
6146 * @param pVCpu The cross context virtual CPU structure.
6147 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6148 */
6149static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
6150{
6151#define VMXLOCAL_BREAK_RC(a_rc) \
6152 if (RT_FAILURE(a_rc)) \
6153 break
6154
6155 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
6156
6157 int rc = VINF_SUCCESS;
6158 PVM pVM = pVCpu->CTX_SUFF(pVM);
6159 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6160 uint64_t u64Val;
6161 uint32_t u32Val;
6162
6163 Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
6164
6165 /*
6166 * We disable interrupts to make the updating of the state and in particular
6167 * the fExtrn modification atomic wrt to preemption hooks.
6168 */
6169 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
6170
6171 fWhat &= pCtx->fExtrn;
6172 if (fWhat)
6173 {
6174 do
6175 {
6176 if (fWhat & CPUMCTX_EXTRN_RIP)
6177 {
6178 rc = hmR0VmxImportGuestRip(pVCpu);
6179 VMXLOCAL_BREAK_RC(rc);
6180 }
6181
6182 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
6183 {
6184 rc = hmR0VmxImportGuestRFlags(pVCpu);
6185 VMXLOCAL_BREAK_RC(rc);
6186 }
6187
6188 if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
6189 {
6190 rc = hmR0VmxImportGuestIntrState(pVCpu);
6191 VMXLOCAL_BREAK_RC(rc);
6192 }
6193
6194 if (fWhat & CPUMCTX_EXTRN_RSP)
6195 {
6196 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6197 VMXLOCAL_BREAK_RC(rc);
6198 pCtx->rsp = u64Val;
6199 }
6200
6201 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
6202 {
6203 if (fWhat & CPUMCTX_EXTRN_CS)
6204 {
6205 rc = HMVMX_IMPORT_SREG(CS, &pCtx->cs);
6206 VMXLOCAL_BREAK_RC(rc);
6207 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6208 pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6209 }
6210 if (fWhat & CPUMCTX_EXTRN_SS)
6211 {
6212 rc = HMVMX_IMPORT_SREG(SS, &pCtx->ss);
6213 VMXLOCAL_BREAK_RC(rc);
6214 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6215 pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6216 }
6217 if (fWhat & CPUMCTX_EXTRN_DS)
6218 {
6219 rc = HMVMX_IMPORT_SREG(DS, &pCtx->ds);
6220 VMXLOCAL_BREAK_RC(rc);
6221 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6222 pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6223 }
6224 if (fWhat & CPUMCTX_EXTRN_ES)
6225 {
6226 rc = HMVMX_IMPORT_SREG(ES, &pCtx->es);
6227 VMXLOCAL_BREAK_RC(rc);
6228 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6229 pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6230 }
6231 if (fWhat & CPUMCTX_EXTRN_FS)
6232 {
6233 rc = HMVMX_IMPORT_SREG(FS, &pCtx->fs);
6234 VMXLOCAL_BREAK_RC(rc);
6235 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6236 pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6237 }
6238 if (fWhat & CPUMCTX_EXTRN_GS)
6239 {
6240 rc = HMVMX_IMPORT_SREG(GS, &pCtx->gs);
6241 VMXLOCAL_BREAK_RC(rc);
6242 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6243 pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6244 }
6245 }
6246
6247 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
6248 {
6249 if (fWhat & CPUMCTX_EXTRN_LDTR)
6250 {
6251 rc = HMVMX_IMPORT_SREG(LDTR, &pCtx->ldtr);
6252 VMXLOCAL_BREAK_RC(rc);
6253 }
6254
6255 if (fWhat & CPUMCTX_EXTRN_GDTR)
6256 {
6257 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6258 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
6259 VMXLOCAL_BREAK_RC(rc);
6260 pCtx->gdtr.pGdt = u64Val;
6261 pCtx->gdtr.cbGdt = u32Val;
6262 }
6263
6264 /* Guest IDTR. */
6265 if (fWhat & CPUMCTX_EXTRN_IDTR)
6266 {
6267 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6268 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
6269 VMXLOCAL_BREAK_RC(rc);
6270 pCtx->idtr.pIdt = u64Val;
6271 pCtx->idtr.cbIdt = u32Val;
6272 }
6273
6274 /* Guest TR. */
6275 if (fWhat & CPUMCTX_EXTRN_TR)
6276 {
6277 /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
6278 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6279 {
6280 rc = HMVMX_IMPORT_SREG(TR, &pCtx->tr);
6281 VMXLOCAL_BREAK_RC(rc);
6282 }
6283 }
6284 }
6285
6286 if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
6287 {
6288 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
6289 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
6290 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
6291 pCtx->SysEnter.cs = u32Val;
6292 VMXLOCAL_BREAK_RC(rc);
6293 }
6294
6295#if HC_ARCH_BITS == 64
6296 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
6297 {
6298 if ( pVM->hm.s.fAllow64BitGuests
6299 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6300 pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
6301 }
6302
6303 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
6304 {
6305 if ( pVM->hm.s.fAllow64BitGuests
6306 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6307 {
6308 pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
6309 pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
6310 pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
6311 }
6312 }
6313#endif
6314
6315 if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
6316#if HC_ARCH_BITS == 32
6317 || (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS))
6318#endif
6319 )
6320 {
6321 PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6322 uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
6323 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6324 {
6325 switch (pMsr->u32Msr)
6326 {
6327#if HC_ARCH_BITS == 32
6328 case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
6329 case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
6330 case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
6331 case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6332#endif
6333 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6334 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6335 default:
6336 {
6337 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
6338 cMsrs));
6339 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6340 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6341 }
6342 }
6343 }
6344 }
6345
6346 if (fWhat & CPUMCTX_EXTRN_DR7)
6347 {
6348 if (!pVCpu->hm.s.fUsingHyperDR7)
6349 {
6350 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6351 rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
6352 VMXLOCAL_BREAK_RC(rc);
6353 pCtx->dr[7] = u32Val;
6354 }
6355 }
6356
6357 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
6358 {
6359 uint32_t u32Shadow;
6360 /* CR0 required for saving CR3 below, see CPUMIsGuestPagingEnabledEx(). */
6361 if (fWhat & CPUMCTX_EXTRN_CR0)
6362 {
6363 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
6364 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
6365 VMXLOCAL_BREAK_RC(rc);
6366 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32CR0Mask)
6367 | (u32Shadow & pVCpu->hm.s.vmx.u32CR0Mask);
6368 VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
6369 CPUMSetGuestCR0(pVCpu, u32Val);
6370 VMMRZCallRing3Enable(pVCpu);
6371 }
6372
6373 /* CR4 required for saving CR3 below, see CPUMIsGuestInPAEModeEx(). */
6374 if (fWhat & CPUMCTX_EXTRN_CR4)
6375 {
6376 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
6377 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
6378 VMXLOCAL_BREAK_RC(rc);
6379 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32CR4Mask)
6380 | (u32Shadow & pVCpu->hm.s.vmx.u32CR4Mask);
6381 CPUMSetGuestCR4(pVCpu, u32Val);
6382 }
6383
6384 if (fWhat & CPUMCTX_EXTRN_CR3)
6385 {
6386 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6387 || ( pVM->hm.s.fNestedPaging
6388 && CPUMIsGuestPagingEnabledEx(pCtx))) /* PG bit changes are always intercepted, so it's up to date. */
6389 {
6390 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6391 if (pCtx->cr3 != u64Val)
6392 {
6393 CPUMSetGuestCR3(pVCpu, u64Val);
6394 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6395 }
6396
6397 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6398 if (CPUMIsGuestInPAEModeEx(pCtx))
6399 {
6400 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6401 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6402 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6403 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6404 VMXLOCAL_BREAK_RC(rc);
6405 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6406 }
6407 }
6408 }
6409 }
6410 } while (0);
6411
6412 if (RT_SUCCESS(rc))
6413 {
6414 /* Update fExtrn. */
6415 pCtx->fExtrn &= ~fWhat;
6416
6417 /* If everything has been imported, clear the HM keeper bit. */
6418 if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
6419 {
6420 pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
6421 Assert(!pCtx->fExtrn);
6422 }
6423 }
6424 }
6425 else
6426 AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
6427
6428 ASMSetFlags(fEFlags);
6429
6430 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
6431
6432 /*
6433 * Honor any pending CR3 updates.
6434 *
6435 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6436 * -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6437 * -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
6438 *
6439 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6440 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6441 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6442 * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
6443 *
6444 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6445 */
6446 if (VMMRZCallRing3IsEnabled(pVCpu))
6447 {
6448 VMMR0LogFlushDisable(pVCpu);
6449
6450 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6451 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6452
6453 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6454 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6455
6456 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6457 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6458
6459 VMMR0LogFlushEnable(pVCpu);
6460 }
6461
6462 return VINF_SUCCESS;
6463#undef VMXLOCAL_BREAK_RC
6464}
6465
6466
6467/**
6468 * Saves the guest state from the VMCS into the guest-CPU context.
6469 *
6470 * @returns VBox status code.
6471 * @param pVCpu The cross context virtual CPU structure.
6472 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6473 */
6474VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
6475{
6476 return hmR0VmxImportGuestState(pVCpu, fWhat);
6477}
6478
6479
6480/**
6481 * Check per-VM and per-VCPU force flag actions that require us to go back to
6482 * ring-3 for one reason or another.
6483 *
6484 * @returns Strict VBox status code (i.e. informational status codes too)
6485 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6486 * ring-3.
6487 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6488 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6489 * interrupts)
6490 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6491 * all EMTs to be in ring-3.
6492 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6493 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6494 * to the EM loop.
6495 *
6496 * @param pVM The cross context VM structure.
6497 * @param pVCpu The cross context virtual CPU structure.
6498 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6499 * out-of-sync. Make sure to update the required fields
6500 * before using them.
6501 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6502 */
6503static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6504{
6505 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6506
6507 /*
6508 * Anything pending? Should be more likely than not if we're doing a good job.
6509 */
6510 if ( !fStepping
6511 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6512 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6513 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6514 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6515 return VINF_SUCCESS;
6516
6517#if 0
6518 /* We need the control registers now, make sure the guest-CPU context is updated. */
6519 int rc3 = hmR0VmxImportGuestStatae(pVCpu, CPUMCTX_EXTRN_CR0);
6520 AssertRCReturn(rc3, rc3);
6521
6522 /** @todo r=ramshankar: VMCPU_FF_HM_UPDATE_CR3 and VMCPU_FF_HM_UPDATE_PAE_PDPES
6523 * are not part of VMCPU_FF_HP_R0_PRE_HM_MASK. Hence, the two if
6524 * statements below won't ever be entered. Consider removing it or
6525 * determine if it is necessary to add these flags to VMCPU_FF_HP_R0_PRE_HM_MASK. */
6526 /* Pending HM CR3 sync. */
6527 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6528 {
6529 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6530 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6531 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6532 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6533 }
6534
6535 /* Pending HM PAE PDPEs. */
6536 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6537 {
6538 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6539 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6540 }
6541#endif
6542
6543 /* Pending PGM C3 sync. */
6544 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6545 {
6546 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
6547 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6548 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6549 if (rcStrict2 != VINF_SUCCESS)
6550 {
6551 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6552 Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6553 return rcStrict2;
6554 }
6555 }
6556
6557 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6558 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6559 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6560 {
6561 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6562 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6563 Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6564 return rc2;
6565 }
6566
6567 /* Pending VM request packets, such as hardware interrupts. */
6568 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6569 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6570 {
6571 Log4Func(("Pending VM request forcing us back to ring-3\n"));
6572 return VINF_EM_PENDING_REQUEST;
6573 }
6574
6575 /* Pending PGM pool flushes. */
6576 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6577 {
6578 Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
6579 return VINF_PGM_POOL_FLUSH_PENDING;
6580 }
6581
6582 /* Pending DMA requests. */
6583 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6584 {
6585 Log4Func(("Pending DMA request forcing us back to ring-3\n"));
6586 return VINF_EM_RAW_TO_R3;
6587 }
6588
6589 return VINF_SUCCESS;
6590}
6591
6592
6593/**
6594 * Converts any TRPM trap into a pending HM event. This is typically used when
6595 * entering from ring-3 (not longjmp returns).
6596 *
6597 * @param pVCpu The cross context virtual CPU structure.
6598 */
6599static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6600{
6601 Assert(TRPMHasTrap(pVCpu));
6602 Assert(!pVCpu->hm.s.Event.fPending);
6603
6604 uint8_t uVector;
6605 TRPMEVENT enmTrpmEvent;
6606 RTGCUINT uErrCode;
6607 RTGCUINTPTR GCPtrFaultAddress;
6608 uint8_t cbInstr;
6609
6610 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6611 AssertRC(rc);
6612
6613 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6614 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6615 if (enmTrpmEvent == TRPM_TRAP)
6616 {
6617 switch (uVector)
6618 {
6619 case X86_XCPT_NMI:
6620 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6621 break;
6622
6623 case X86_XCPT_BP:
6624 case X86_XCPT_OF:
6625 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6626 break;
6627
6628 case X86_XCPT_PF:
6629 case X86_XCPT_DF:
6630 case X86_XCPT_TS:
6631 case X86_XCPT_NP:
6632 case X86_XCPT_SS:
6633 case X86_XCPT_GP:
6634 case X86_XCPT_AC:
6635 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6636 RT_FALL_THRU();
6637 default:
6638 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6639 break;
6640 }
6641 }
6642 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6643 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6644 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6645 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6646 else
6647 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6648
6649 rc = TRPMResetTrap(pVCpu);
6650 AssertRC(rc);
6651 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6652 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6653
6654 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6655}
6656
6657
6658/**
6659 * Converts the pending HM event into a TRPM trap.
6660 *
6661 * @param pVCpu The cross context virtual CPU structure.
6662 */
6663static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6664{
6665 Assert(pVCpu->hm.s.Event.fPending);
6666
6667 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6668 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6669 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6670 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6671
6672 /* If a trap was already pending, we did something wrong! */
6673 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6674
6675 TRPMEVENT enmTrapType;
6676 switch (uVectorType)
6677 {
6678 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6679 enmTrapType = TRPM_HARDWARE_INT;
6680 break;
6681
6682 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6683 enmTrapType = TRPM_SOFTWARE_INT;
6684 break;
6685
6686 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6687 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6688 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6689 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6690 enmTrapType = TRPM_TRAP;
6691 break;
6692
6693 default:
6694 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6695 enmTrapType = TRPM_32BIT_HACK;
6696 break;
6697 }
6698
6699 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6700
6701 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6702 AssertRC(rc);
6703
6704 if (fErrorCodeValid)
6705 TRPMSetErrorCode(pVCpu, uErrorCode);
6706
6707 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6708 && uVector == X86_XCPT_PF)
6709 {
6710 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6711 }
6712 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6713 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6714 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6715 {
6716 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6717 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6718 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6719 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6720 }
6721
6722 /* Clear any pending events from the VMCS. */
6723 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
6724 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
6725
6726 /* We're now done converting the pending event. */
6727 pVCpu->hm.s.Event.fPending = false;
6728}
6729
6730
6731/**
6732 * Does the necessary state syncing before returning to ring-3 for any reason
6733 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6734 *
6735 * @returns VBox status code.
6736 * @param pVCpu The cross context virtual CPU structure.
6737 * @param fImportState Whether to import the guest state from the VMCS back
6738 * to the guest-CPU context.
6739 *
6740 * @remarks No-long-jmp zone!!!
6741 */
6742static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
6743{
6744 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6745 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6746
6747 RTCPUID idCpu = RTMpCpuId();
6748 Log4Func(("HostCpuId=%u\n", idCpu));
6749
6750 /*
6751 * !!! IMPORTANT !!!
6752 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
6753 */
6754
6755 /* Save the guest state if necessary. */
6756 if (fImportState)
6757 {
6758 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
6759 AssertRCReturn(rc, rc);
6760 }
6761
6762 /* Restore host FPU state if necessary. We will resync on next R0 reentry. */
6763 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
6764 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6765
6766 /* Restore host debug registers if necessary. We will resync on next R0 reentry. */
6767#ifdef VBOX_STRICT
6768 if (CPUMIsHyperDebugStateActive(pVCpu))
6769 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6770#endif
6771 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6772 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6773 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6774
6775#if HC_ARCH_BITS == 64
6776 /* Restore host-state bits that VT-x only restores partially. */
6777 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6778 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6779 {
6780 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6781 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6782 }
6783 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6784#endif
6785
6786 /* Restore the lazy host MSRs as we're leaving VT-x context. */
6787 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
6788 {
6789 /* We shouldn't restore the host MSRs without saving the guest MSRs first. */
6790 if (!fImportState)
6791 {
6792 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE
6793 | CPUMCTX_EXTRN_SYSCALL_MSRS);
6794 AssertRCReturn(rc, rc);
6795 }
6796 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6797 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
6798 }
6799 else
6800 pVCpu->hm.s.vmx.fLazyMsrs = 0;
6801
6802 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6803 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6804
6805 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6806 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
6807 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
6808 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6809 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6810 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6811 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6812 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6813 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6814
6815 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6816
6817 /** @todo This partially defeats the purpose of having preemption hooks.
6818 * The problem is, deregistering the hooks should be moved to a place that
6819 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6820 * context.
6821 */
6822 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6823 {
6824 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6825 AssertRCReturn(rc, rc);
6826
6827 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6828 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6829 }
6830 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6831 NOREF(idCpu);
6832
6833 return VINF_SUCCESS;
6834}
6835
6836
6837/**
6838 * Leaves the VT-x session.
6839 *
6840 * @returns VBox status code.
6841 * @param pVCpu The cross context virtual CPU structure.
6842 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6843 * out-of-sync. Make sure to update the required fields
6844 * before using them.
6845 *
6846 * @remarks No-long-jmp zone!!!
6847 */
6848static int hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6849{
6850 HM_DISABLE_PREEMPT();
6851 HMVMX_ASSERT_CPU_SAFE();
6852 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6853 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6854
6855 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6856 and done this from the VMXR0ThreadCtxCallback(). */
6857 if (!pVCpu->hm.s.fLeaveDone)
6858 {
6859 int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
6860 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
6861 pVCpu->hm.s.fLeaveDone = true;
6862 }
6863 Assert(!pMixedCtx->fExtrn); NOREF(pMixedCtx);
6864
6865 /*
6866 * !!! IMPORTANT !!!
6867 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6868 */
6869
6870 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6871 /** @todo Deregistering here means we need to VMCLEAR always
6872 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6873 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
6874 VMMR0ThreadCtxHookDisable(pVCpu);
6875
6876 /* Leave HM context. This takes care of local init (term). */
6877 int rc = HMR0LeaveCpu(pVCpu);
6878
6879 HM_RESTORE_PREEMPT();
6880 return rc;
6881}
6882
6883
6884/**
6885 * Does the necessary state syncing before doing a longjmp to ring-3.
6886 *
6887 * @returns VBox status code.
6888 * @param pVCpu The cross context virtual CPU structure.
6889 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6890 * out-of-sync. Make sure to update the required fields
6891 * before using them.
6892 *
6893 * @remarks No-long-jmp zone!!!
6894 */
6895DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6896{
6897 return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
6898}
6899
6900
6901/**
6902 * Take necessary actions before going back to ring-3.
6903 *
6904 * An action requires us to go back to ring-3. This function does the necessary
6905 * steps before we can safely return to ring-3. This is not the same as longjmps
6906 * to ring-3, this is voluntary and prepares the guest so it may continue
6907 * executing outside HM (recompiler/IEM).
6908 *
6909 * @returns VBox status code.
6910 * @param pVM The cross context VM structure.
6911 * @param pVCpu The cross context virtual CPU structure.
6912 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6913 * out-of-sync. Make sure to update the required fields
6914 * before using them.
6915 * @param rcExit The reason for exiting to ring-3. Can be
6916 * VINF_VMM_UNKNOWN_RING3_CALL.
6917 */
6918static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
6919{
6920 Assert(pVM);
6921 Assert(pVCpu);
6922 Assert(pMixedCtx);
6923 HMVMX_ASSERT_PREEMPT_SAFE();
6924
6925 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6926 {
6927 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6928 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6929 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6930 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6931 }
6932
6933 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6934 VMMRZCallRing3Disable(pVCpu);
6935 Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
6936
6937 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6938 if (pVCpu->hm.s.Event.fPending)
6939 {
6940 hmR0VmxPendingEventToTrpmTrap(pVCpu);
6941 Assert(!pVCpu->hm.s.Event.fPending);
6942 }
6943
6944 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
6945 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
6946
6947 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
6948 and if we're injecting an event we should have a TRPM trap pending. */
6949 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6950#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
6951 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6952#endif
6953
6954 /* Save guest state and restore host state bits. */
6955 int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
6956 AssertRCReturn(rc, rc);
6957 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6958 /* Thread-context hooks are unregistered at this point!!! */
6959
6960 /* Sync recompiler state. */
6961 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6962 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6963 | CPUM_CHANGED_LDTR
6964 | CPUM_CHANGED_GDTR
6965 | CPUM_CHANGED_IDTR
6966 | CPUM_CHANGED_TR
6967 | CPUM_CHANGED_HIDDEN_SEL_REGS);
6968 if ( pVM->hm.s.fNestedPaging
6969 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
6970 {
6971 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6972 }
6973
6974 Assert(!pVCpu->hm.s.fClearTrapFlag);
6975
6976 /* Update the exit-to-ring 3 reason. */
6977 pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
6978
6979 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6980 if (rcExit != VINF_EM_RAW_INTERRUPT)
6981 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
6982
6983 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6984
6985 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6986 VMMRZCallRing3RemoveNotification(pVCpu);
6987 VMMRZCallRing3Enable(pVCpu);
6988
6989 return rc;
6990}
6991
6992
6993/**
6994 * VMMRZCallRing3() callback wrapper which saves the guest state before we
6995 * longjump to ring-3 and possibly get preempted.
6996 *
6997 * @returns VBox status code.
6998 * @param pVCpu The cross context virtual CPU structure.
6999 * @param enmOperation The operation causing the ring-3 longjump.
7000 * @param pvUser Opaque pointer to the guest-CPU context. The data
7001 * may be out-of-sync. Make sure to update the required
7002 * fields before using them.
7003 */
7004static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7005{
7006 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7007 {
7008 /*
7009 * !!! IMPORTANT !!!
7010 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7011 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7012 */
7013 VMMRZCallRing3RemoveNotification(pVCpu);
7014 VMMRZCallRing3Disable(pVCpu);
7015 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7016 RTThreadPreemptDisable(&PreemptState);
7017
7018 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7019 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7020
7021#if HC_ARCH_BITS == 64
7022 /* Restore host-state bits that VT-x only restores partially. */
7023 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7024 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7025 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7026 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7027#endif
7028
7029 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7030 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7031 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7032
7033 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7034 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7035 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7036 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7037 {
7038 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7039 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7040 }
7041
7042 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7043 VMMR0ThreadCtxHookDisable(pVCpu);
7044 HMR0LeaveCpu(pVCpu);
7045 RTThreadPreemptRestore(&PreemptState);
7046 return VINF_SUCCESS;
7047 }
7048
7049 Assert(pVCpu);
7050 Assert(pvUser);
7051 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7052 HMVMX_ASSERT_PREEMPT_SAFE();
7053
7054 VMMRZCallRing3Disable(pVCpu);
7055 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7056
7057 Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
7058
7059 int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7060 AssertRCReturn(rc, rc);
7061
7062 VMMRZCallRing3Enable(pVCpu);
7063 return VINF_SUCCESS;
7064}
7065
7066
7067/**
7068 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7069 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7070 *
7071 * @param pVCpu The cross context virtual CPU structure.
7072 */
7073DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7074{
7075 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7076 {
7077 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7078 {
7079 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7080 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7081 AssertRC(rc);
7082 Log4Func(("Setup interrupt-window exiting\n"));
7083 }
7084 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7085}
7086
7087
7088/**
7089 * Clears the interrupt-window exiting control in the VMCS.
7090 *
7091 * @param pVCpu The cross context virtual CPU structure.
7092 */
7093DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7094{
7095 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7096 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7097 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7098 AssertRC(rc);
7099 Log4Func(("Cleared interrupt-window exiting\n"));
7100}
7101
7102
7103/**
7104 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7105 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7106 *
7107 * @param pVCpu The cross context virtual CPU structure.
7108 */
7109DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7110{
7111 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7112 {
7113 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7114 {
7115 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7116 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7117 AssertRC(rc);
7118 Log4Func(("Setup NMI-window exiting\n"));
7119 }
7120 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7121}
7122
7123
7124/**
7125 * Clears the NMI-window exiting control in the VMCS.
7126 *
7127 * @param pVCpu The cross context virtual CPU structure.
7128 */
7129DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7130{
7131 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7132 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7133 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7134 AssertRC(rc);
7135 Log4Func(("Cleared NMI-window exiting\n"));
7136}
7137
7138
7139/**
7140 * Evaluates the event to be delivered to the guest and sets it as the pending
7141 * event.
7142 *
7143 * @returns The VT-x guest-interruptibility state.
7144 * @param pVCpu The cross context virtual CPU structure.
7145 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7146 * out-of-sync. Make sure to update the required fields
7147 * before using them.
7148 */
7149static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7150{
7151 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7152 uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7153 bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7154 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7155 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7156
7157 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7158 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7159 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7160 Assert(!TRPMHasTrap(pVCpu));
7161
7162 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7163 APICUpdatePendingInterrupts(pVCpu);
7164
7165 /*
7166 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7167 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7168 */
7169 /** @todo SMI. SMIs take priority over NMIs. */
7170 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7171 {
7172 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7173 if ( !pVCpu->hm.s.Event.fPending
7174 && !fBlockNmi
7175 && !fBlockSti
7176 && !fBlockMovSS)
7177 {
7178 Log4Func(("Pending NMI\n"));
7179 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7180 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7181
7182 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7183 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7184 }
7185 else
7186 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7187 }
7188 /*
7189 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7190 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7191 */
7192 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7193 && !pVCpu->hm.s.fSingleInstruction)
7194 {
7195 Assert(!DBGFIsStepping(pVCpu));
7196 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7197 AssertRCReturn(rc, 0);
7198 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7199 if ( !pVCpu->hm.s.Event.fPending
7200 && !fBlockInt
7201 && !fBlockSti
7202 && !fBlockMovSS)
7203 {
7204 uint8_t u8Interrupt;
7205 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7206 if (RT_SUCCESS(rc))
7207 {
7208 Log4Func(("Pending external interrupt u8Interrupt=%#x\n", u8Interrupt));
7209 uint32_t u32IntInfo = u8Interrupt
7210 | VMX_EXIT_INTERRUPTION_INFO_VALID
7211 | (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7212
7213 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7214 }
7215 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7216 {
7217 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7218 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7219 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7220
7221 /*
7222 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7223 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7224 * need to re-set this force-flag here.
7225 */
7226 }
7227 else
7228 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7229 }
7230 else
7231 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7232 }
7233
7234 return fIntrState;
7235}
7236
7237
7238/**
7239 * Sets a pending-debug exception to be delivered to the guest if the guest is
7240 * single-stepping in the VMCS.
7241 *
7242 * @param pVCpu The cross context virtual CPU structure.
7243 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7244 * out-of-sync. Make sure to update the required fields
7245 * before using them.
7246 */
7247DECLINLINE(int) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7248{
7249 RT_NOREF(pVCpu);
7250 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS)); NOREF(pMixedCtx);
7251 return VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7252}
7253
7254
7255/**
7256 * Injects any pending events into the guest if the guest is in a state to
7257 * receive them.
7258 *
7259 * @returns Strict VBox status code (i.e. informational status codes too).
7260 * @param pVCpu The cross context virtual CPU structure.
7261 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7262 * out-of-sync. Make sure to update the required fields
7263 * before using them.
7264 * @param fIntrState The VT-x guest-interruptibility state.
7265 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7266 * return VINF_EM_DBG_STEPPED if the event was
7267 * dispatched directly.
7268 */
7269static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t fIntrState, bool fStepping)
7270{
7271 HMVMX_ASSERT_PREEMPT_SAFE();
7272 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7273
7274 bool fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7275 bool fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7276
7277 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7278 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7279 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7280 Assert(!TRPMHasTrap(pVCpu));
7281
7282 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7283 if (pVCpu->hm.s.Event.fPending)
7284 {
7285 /*
7286 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7287 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7288 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7289 *
7290 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7291 */
7292 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7293#ifdef VBOX_STRICT
7294 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7295 {
7296 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7297 Assert(!fBlockInt);
7298 Assert(!fBlockSti);
7299 Assert(!fBlockMovSS);
7300 }
7301 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7302 {
7303 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7304 Assert(!fBlockSti);
7305 Assert(!fBlockMovSS);
7306 Assert(!fBlockNmi);
7307 }
7308#endif
7309 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7310 (uint8_t)uIntType));
7311 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7312 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping,
7313 &fIntrState);
7314 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7315
7316 /* Update the interruptibility-state as it could have been changed by
7317 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7318 fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7319 fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7320
7321 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7322 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7323 else
7324 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7325 }
7326
7327 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7328 if ( fBlockSti
7329 || fBlockMovSS)
7330 {
7331 if (!pVCpu->hm.s.fSingleInstruction)
7332 {
7333 /*
7334 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7335 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7336 * See Intel spec. 27.3.4 "Saving Non-Register State".
7337 */
7338 Assert(!DBGFIsStepping(pVCpu));
7339 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7340 AssertRCReturn(rc, rc);
7341 if (pMixedCtx->eflags.Bits.u1TF)
7342 {
7343 int rc2 = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
7344 AssertRCReturn(rc2, rc2);
7345 }
7346 }
7347 else if (pMixedCtx->eflags.Bits.u1TF)
7348 {
7349 /*
7350 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7351 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7352 */
7353 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7354 fIntrState = 0;
7355 }
7356 }
7357
7358 /*
7359 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7360 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7361 */
7362 int rc3 = hmR0VmxExportGuestIntrState(pVCpu, fIntrState);
7363 AssertRCReturn(rc3, rc3);
7364
7365 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7366 NOREF(fBlockMovSS); NOREF(fBlockSti);
7367 return rcStrict;
7368}
7369
7370
7371/**
7372 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7373 *
7374 * @param pVCpu The cross context virtual CPU structure.
7375 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7376 * out-of-sync. Make sure to update the required fields
7377 * before using them.
7378 */
7379DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7380{
7381 NOREF(pMixedCtx);
7382 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7383 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7384}
7385
7386
7387/**
7388 * Injects a double-fault (\#DF) exception into the VM.
7389 *
7390 * @returns Strict VBox status code (i.e. informational status codes too).
7391 * @param pVCpu The cross context virtual CPU structure.
7392 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7393 * out-of-sync. Make sure to update the required fields
7394 * before using them.
7395 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7396 * and should return VINF_EM_DBG_STEPPED if the event
7397 * is injected directly (register modified by us, not
7398 * by hardware on VM-entry).
7399 * @param pfIntrState Pointer to the current guest interruptibility-state.
7400 * This interruptibility-state will be updated if
7401 * necessary. This cannot not be NULL.
7402 */
7403DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCCPUMCTX pMixedCtx, bool fStepping, uint32_t *pfIntrState)
7404{
7405 NOREF(pMixedCtx);
7406 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7407 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7408 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7409 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */, fStepping,
7410 pfIntrState);
7411}
7412
7413
7414/**
7415 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7416 *
7417 * @param pVCpu The cross context virtual CPU structure.
7418 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7419 * out-of-sync. Make sure to update the required fields
7420 * before using them.
7421 */
7422DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7423{
7424 NOREF(pMixedCtx);
7425 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7426 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7427 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7428}
7429
7430
7431/**
7432 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7433 *
7434 * @param pVCpu The cross context virtual CPU structure.
7435 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7436 * out-of-sync. Make sure to update the required fields
7437 * before using them.
7438 * @param cbInstr The value of RIP that is to be pushed on the guest
7439 * stack.
7440 */
7441DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7442{
7443 NOREF(pMixedCtx);
7444 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7445 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7446 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7447}
7448
7449
7450/**
7451 * Injects a general-protection (\#GP) fault into the VM.
7452 *
7453 * @returns Strict VBox status code (i.e. informational status codes too).
7454 * @param pVCpu The cross context virtual CPU structure.
7455 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7456 * out-of-sync. Make sure to update the required fields
7457 * before using them.
7458 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7459 * mode, i.e. in real-mode it's not valid).
7460 * @param u32ErrorCode The error code associated with the \#GP.
7461 * @param fStepping Whether we're running in
7462 * hmR0VmxRunGuestCodeStep() and should return
7463 * VINF_EM_DBG_STEPPED if the event is injected
7464 * directly (register modified by us, not by
7465 * hardware on VM-entry).
7466 * @param pfIntrState Pointer to the current guest interruptibility-state.
7467 * This interruptibility-state will be updated if
7468 * necessary. This cannot not be NULL.
7469 */
7470DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7471 bool fStepping, uint32_t *pfIntrState)
7472{
7473 NOREF(pMixedCtx);
7474 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7475 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7476 if (fErrorCodeValid)
7477 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7478 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */, fStepping,
7479 pfIntrState);
7480}
7481
7482
7483#if 0 /* unused */
7484/**
7485 * Sets a general-protection (\#GP) exception as pending-for-injection into the
7486 * VM.
7487 *
7488 * @param pVCpu The cross context virtual CPU structure.
7489 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7490 * out-of-sync. Make sure to update the required fields
7491 * before using them.
7492 * @param u32ErrorCode The error code associated with the \#GP.
7493 */
7494DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7495{
7496 NOREF(pMixedCtx);
7497 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7498 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7499 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7500 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7501}
7502#endif /* unused */
7503
7504
7505/**
7506 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7507 *
7508 * @param pVCpu The cross context virtual CPU structure.
7509 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7510 * out-of-sync. Make sure to update the required fields
7511 * before using them.
7512 * @param uVector The software interrupt vector number.
7513 * @param cbInstr The value of RIP that is to be pushed on the guest
7514 * stack.
7515 */
7516DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7517{
7518 NOREF(pMixedCtx);
7519 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7520 if ( uVector == X86_XCPT_BP
7521 || uVector == X86_XCPT_OF)
7522 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7523 else
7524 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7525 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7526}
7527
7528
7529/**
7530 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7531 * stack.
7532 *
7533 * @returns Strict VBox status code (i.e. informational status codes too).
7534 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7535 * @param pVM The cross context VM structure.
7536 * @param pMixedCtx Pointer to the guest-CPU context.
7537 * @param uValue The value to push to the guest stack.
7538 */
7539DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7540{
7541 /*
7542 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7543 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7544 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7545 */
7546 if (pMixedCtx->sp == 1)
7547 return VINF_EM_RESET;
7548 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7549 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7550 AssertRC(rc);
7551 return rc;
7552}
7553
7554
7555/**
7556 * Injects an event into the guest upon VM-entry by updating the relevant fields
7557 * in the VM-entry area in the VMCS.
7558 *
7559 * @returns Strict VBox status code (i.e. informational status codes too).
7560 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7561 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7562 *
7563 * @param pVCpu The cross context virtual CPU structure.
7564 * @param u64IntInfo The VM-entry interruption-information field.
7565 * @param cbInstr The VM-entry instruction length in bytes (for
7566 * software interrupts, exceptions and privileged
7567 * software exceptions).
7568 * @param u32ErrCode The VM-entry exception error code.
7569 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7570 * @param pfIntrState Pointer to the current guest interruptibility-state.
7571 * This interruptibility-state will be updated if
7572 * necessary. This cannot not be NULL.
7573 * @param fStepping Whether we're running in
7574 * hmR0VmxRunGuestCodeStep() and should return
7575 * VINF_EM_DBG_STEPPED if the event is injected
7576 * directly (register modified by us, not by
7577 * hardware on VM-entry).
7578 *
7579 * @remarks Requires CR0!
7580 */
7581static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7582 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState)
7583{
7584 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7585 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7586 Assert(pfIntrState);
7587
7588 PCPUMCTX pMixedCtx = &pVCpu->cpum.GstCtx;
7589 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7590 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7591 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7592
7593#ifdef VBOX_STRICT
7594 /*
7595 * Validate the error-code-valid bit for hardware exceptions.
7596 * No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7597 */
7598 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7599 && !CPUMIsGuestInRealModeEx(pMixedCtx))
7600 {
7601 switch (uVector)
7602 {
7603 case X86_XCPT_PF:
7604 case X86_XCPT_DF:
7605 case X86_XCPT_TS:
7606 case X86_XCPT_NP:
7607 case X86_XCPT_SS:
7608 case X86_XCPT_GP:
7609 case X86_XCPT_AC:
7610 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7611 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7612 RT_FALL_THRU();
7613 default:
7614 break;
7615 }
7616 }
7617#endif
7618
7619 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7620 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7621 || !(*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7622
7623 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7624
7625 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
7626 {
7627 /*
7628 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7629 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7630 */
7631 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7632 }
7633 else
7634 {
7635 /* We require CR0 to check if the guest is in real-mode. */
7636 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
7637 AssertRCReturn(rc, rc);
7638
7639 /*
7640 * Hardware interrupts & exceptions cannot be delivered through the software interrupt
7641 * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
7642 * interrupt handler in the (real-mode) guest.
7643 *
7644 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
7645 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7646 */
7647 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7648 {
7649 PVM pVM = pVCpu->CTX_SUFF(pVM);
7650 Assert(PDMVmmDevHeapIsEnabled(pVM));
7651 Assert(pVM->hm.s.vmx.pRealModeTSS);
7652
7653 /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
7654 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK
7655 | CPUMCTX_EXTRN_TABLE_MASK
7656 | CPUMCTX_EXTRN_RIP
7657 | CPUMCTX_EXTRN_RSP
7658 | CPUMCTX_EXTRN_RFLAGS);
7659 AssertRCReturn(rc, rc);
7660
7661 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7662 size_t const cbIdtEntry = sizeof(X86IDTR16);
7663 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7664 {
7665 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7666 if (uVector == X86_XCPT_DF)
7667 return VINF_EM_RESET;
7668
7669 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7670 if (uVector == X86_XCPT_GP)
7671 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, pfIntrState);
7672
7673 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7674 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7675 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
7676 fStepping, pfIntrState);
7677 }
7678
7679 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7680 uint16_t uGuestIp = pMixedCtx->ip;
7681 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7682 {
7683 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7684 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7685 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7686 }
7687 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7688 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7689
7690 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7691 X86IDTR16 IdtEntry;
7692 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7693 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7694 AssertRCReturn(rc, rc);
7695
7696 /* Construct the stack frame for the interrupt/exception handler. */
7697 VBOXSTRICTRC rcStrict;
7698 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7699 if (rcStrict == VINF_SUCCESS)
7700 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7701 if (rcStrict == VINF_SUCCESS)
7702 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7703
7704 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7705 if (rcStrict == VINF_SUCCESS)
7706 {
7707 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7708 pMixedCtx->rip = IdtEntry.offSel;
7709 pMixedCtx->cs.Sel = IdtEntry.uSel;
7710 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7711 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7712 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7713 && uVector == X86_XCPT_PF)
7714 pMixedCtx->cr2 = GCPtrFaultAddress;
7715
7716 /* If any other guest-state bits are changed here, make sure to update
7717 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7718 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS
7719 | HM_CHANGED_GUEST_CR2
7720 | HM_CHANGED_GUEST_RIP
7721 | HM_CHANGED_GUEST_RFLAGS
7722 | HM_CHANGED_GUEST_RSP);
7723
7724 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7725 if (*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7726 {
7727 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7728 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7729 Log4Func(("Clearing inhibition due to STI\n"));
7730 *pfIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7731 }
7732 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7733 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
7734
7735 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7736 it, if we are returning to ring-3 before executing guest code. */
7737 pVCpu->hm.s.Event.fPending = false;
7738
7739 /* Make hmR0VmxPreRunGuest() return if we're stepping since we've changed cs:rip. */
7740 if (fStepping)
7741 rcStrict = VINF_EM_DBG_STEPPED;
7742 }
7743 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7744 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7745 return rcStrict;
7746 }
7747 }
7748
7749 /* Validate. */
7750 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7751 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7752 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7753
7754 /* Inject. */
7755 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7756 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7757 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7758 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7759 AssertRCReturn(rc, rc);
7760
7761 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7762 && uVector == X86_XCPT_PF)
7763 pMixedCtx->cr2 = GCPtrFaultAddress;
7764
7765 Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr,
7766 pMixedCtx->cr2));
7767
7768 return VINF_SUCCESS;
7769}
7770
7771
7772/**
7773 * Clears the interrupt-window exiting control in the VMCS and if necessary
7774 * clears the current event in the VMCS as well.
7775 *
7776 * @returns VBox status code.
7777 * @param pVCpu The cross context virtual CPU structure.
7778 *
7779 * @remarks Use this function only to clear events that have not yet been
7780 * delivered to the guest but are injected in the VMCS!
7781 * @remarks No-long-jump zone!!!
7782 */
7783static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
7784{
7785 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7786 {
7787 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7788 Log4Func(("Cleared interrupt widow\n"));
7789 }
7790
7791 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
7792 {
7793 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
7794 Log4Func(("Cleared interrupt widow\n"));
7795 }
7796}
7797
7798
7799/**
7800 * Enters the VT-x session.
7801 *
7802 * @returns VBox status code.
7803 * @param pVM The cross context VM structure.
7804 * @param pVCpu The cross context virtual CPU structure.
7805 * @param pCpu Pointer to the CPU info struct.
7806 */
7807VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7808{
7809 AssertPtr(pVM);
7810 AssertPtr(pVCpu);
7811 Assert(pVM->hm.s.vmx.fSupported);
7812 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7813 NOREF(pCpu); NOREF(pVM);
7814
7815 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7816 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7817 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7818
7819#ifdef VBOX_STRICT
7820 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
7821 RTCCUINTREG uHostCR4 = ASMGetCR4();
7822 if (!(uHostCR4 & X86_CR4_VMXE))
7823 {
7824 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7825 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7826 }
7827#endif
7828
7829 /*
7830 * Load the VCPU's VMCS as the current (and active) one.
7831 */
7832 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7833 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7834 if (RT_FAILURE(rc))
7835 return rc;
7836
7837 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7838 pVCpu->hm.s.fLeaveDone = false;
7839 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7840
7841 return VINF_SUCCESS;
7842}
7843
7844
7845/**
7846 * The thread-context callback (only on platforms which support it).
7847 *
7848 * @param enmEvent The thread-context event.
7849 * @param pVCpu The cross context virtual CPU structure.
7850 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7851 * @thread EMT(pVCpu)
7852 */
7853VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7854{
7855 NOREF(fGlobalInit);
7856
7857 switch (enmEvent)
7858 {
7859 case RTTHREADCTXEVENT_OUT:
7860 {
7861 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7862 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7863 VMCPU_ASSERT_EMT(pVCpu);
7864
7865 /* No longjmps (logger flushes, locks) in this fragile context. */
7866 VMMRZCallRing3Disable(pVCpu);
7867 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7868
7869 /*
7870 * Restore host-state (FPU, debug etc.)
7871 */
7872 if (!pVCpu->hm.s.fLeaveDone)
7873 {
7874 /*
7875 * Do -not- import the guest-state here as we might already be in the middle of importing
7876 * it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
7877 */
7878 hmR0VmxLeave(pVCpu, false /* fImportState */);
7879 pVCpu->hm.s.fLeaveDone = true;
7880 }
7881
7882 /* Leave HM context, takes care of local init (term). */
7883 int rc = HMR0LeaveCpu(pVCpu);
7884 AssertRC(rc); NOREF(rc);
7885
7886 /* Restore longjmp state. */
7887 VMMRZCallRing3Enable(pVCpu);
7888 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
7889 break;
7890 }
7891
7892 case RTTHREADCTXEVENT_IN:
7893 {
7894 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7895 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7896 VMCPU_ASSERT_EMT(pVCpu);
7897
7898 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7899 VMMRZCallRing3Disable(pVCpu);
7900 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7901
7902 /* Initialize the bare minimum state required for HM. This takes care of
7903 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7904 int rc = HMR0EnterCpu(pVCpu);
7905 AssertRC(rc);
7906 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7907 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7908
7909 /* Load the active VMCS as the current one. */
7910 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7911 {
7912 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7913 AssertRC(rc); NOREF(rc);
7914 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7915 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7916 }
7917 pVCpu->hm.s.fLeaveDone = false;
7918
7919 /* Restore longjmp state. */
7920 VMMRZCallRing3Enable(pVCpu);
7921 break;
7922 }
7923
7924 default:
7925 break;
7926 }
7927}
7928
7929
7930/**
7931 * Exports the host state into the VMCS host-state area.
7932 * Sets up the VM-exit MSR-load area.
7933 *
7934 * The CPU state will be loaded from these fields on every successful VM-exit.
7935 *
7936 * @returns VBox status code.
7937 * @param pVCpu The cross context virtual CPU structure.
7938 *
7939 * @remarks No-long-jump zone!!!
7940 */
7941static int hmR0VmxExportHostState(PVMCPU pVCpu)
7942{
7943 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7944
7945 int rc = VINF_SUCCESS;
7946 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
7947 {
7948 rc = hmR0VmxExportHostControlRegs();
7949 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7950
7951 rc = hmR0VmxExportHostSegmentRegs(pVCpu);
7952 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7953
7954 rc = hmR0VmxExportHostMsrs(pVCpu);
7955 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7956
7957 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
7958 }
7959 return rc;
7960}
7961
7962
7963/**
7964 * Saves the host state in the VMCS host-state.
7965 *
7966 * @returns VBox status code.
7967 * @param pVM The cross context VM structure.
7968 * @param pVCpu The cross context virtual CPU structure.
7969 *
7970 * @remarks No-long-jump zone!!!
7971 */
7972VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
7973{
7974 AssertPtr(pVCpu);
7975 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7976
7977 /*
7978 * Export the host state here while entering HM context.
7979 * When thread-context hooks are used, we might get preempted and have to re-save the host
7980 * state but most of the time we won't be, so do it here before we disable interrupts.
7981 */
7982 return hmR0VmxExportHostState(pVCpu);
7983}
7984
7985
7986/**
7987 * Exports the guest state into the VMCS guest-state area.
7988 *
7989 * The will typically be done before VM-entry when the guest-CPU state and the
7990 * VMCS state may potentially be out of sync.
7991 *
7992 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
7993 * VM-entry controls.
7994 * Sets up the appropriate VMX non-root function to execute guest code based on
7995 * the guest CPU mode.
7996 *
7997 * @returns VBox strict status code.
7998 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
7999 * without unrestricted guest access and the VMMDev is not presently
8000 * mapped (e.g. EFI32).
8001 *
8002 * @param pVM The cross context VM structure.
8003 * @param pVCpu The cross context virtual CPU structure.
8004 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8005 * out-of-sync. Make sure to update the required fields
8006 * before using them.
8007 *
8008 * @remarks No-long-jump zone!!!
8009 */
8010static VBOXSTRICTRC hmR0VmxExportGuestState(PVM pVM, PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
8011{
8012 AssertPtr(pVM);
8013 AssertPtr(pVCpu);
8014 AssertPtr(pMixedCtx);
8015 HMVMX_ASSERT_PREEMPT_SAFE();
8016
8017 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8018
8019 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
8020
8021 /* Determine real-on-v86 mode. */
8022 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8023 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8024 && CPUMIsGuestInRealModeEx(pMixedCtx))
8025 {
8026 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8027 }
8028
8029 /*
8030 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8031 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8032 */
8033 int rc = hmR0VmxSelectVMRunHandler(pVCpu, pMixedCtx);
8034 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8035
8036 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8037 rc = hmR0VmxExportGuestEntryCtls(pVCpu, pMixedCtx);
8038 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8039
8040 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8041 rc = hmR0VmxExportGuestExitCtls(pVCpu, pMixedCtx);
8042 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8043
8044 rc = hmR0VmxExportGuestCR0(pVCpu, pMixedCtx);
8045 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8046
8047 VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pMixedCtx);
8048 if (rcStrict == VINF_SUCCESS)
8049 { /* likely */ }
8050 else
8051 {
8052 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8053 return rcStrict;
8054 }
8055
8056 rc = hmR0VmxExportGuestSegmentRegs(pVCpu, pMixedCtx);
8057 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8058
8059 /* This needs to be done after hmR0VmxExportGuestEntryCtls() and hmR0VmxExportGuestExitCtls() as it
8060 may alter controls if we determine we don't have to swap EFER after all. */
8061 rc = hmR0VmxExportGuestMsrs(pVCpu, pMixedCtx);
8062 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8063
8064 rc = hmR0VmxExportGuestApicTpr(pVCpu);
8065 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8066
8067 /* This needs to be done after hmR0VmxExportGuestCR0() as it may alter intercepted exceptions. */
8068 rc = hmR0VmxExportGuestXcptIntercepts(pVCpu);
8069 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8070
8071 /* Exporting RFLAGS here is fine, even though RFLAGS.TF might depend on guest debug state which is
8072 not exported here. It is re-evaluated and updated if necessary in hmR0VmxExportSharedState(). */
8073 rc = hmR0VmxExportGuestRip(pVCpu, pMixedCtx);
8074 rc |= hmR0VmxExportGuestRsp(pVCpu, pMixedCtx);
8075 rc |= hmR0VmxExportGuestRflags(pVCpu, pMixedCtx);
8076 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8077
8078 /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
8079 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
8080 | HM_CHANGED_GUEST_CR2
8081 | (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
8082 | HM_CHANGED_GUEST_X87
8083 | HM_CHANGED_GUEST_SSE_AVX
8084 | HM_CHANGED_GUEST_OTHER_XSAVE
8085 | HM_CHANGED_GUEST_XCRx
8086 | HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
8087 | HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
8088 | HM_CHANGED_GUEST_TSC_AUX
8089 | HM_CHANGED_GUEST_OTHER_MSRS
8090 | HM_CHANGED_GUEST_HWVIRT
8091 | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
8092
8093 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
8094 return rc;
8095}
8096
8097
8098/**
8099 * Exports the state shared between the host and guest into the VMCS.
8100 *
8101 * @param pVM The cross context VM structure.
8102 * @param pVCpu The cross context virtual CPU structure.
8103 * @param pCtx Pointer to the guest-CPU context.
8104 *
8105 * @remarks No-long-jump zone!!!
8106 */
8107static void hmR0VmxExportSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8108{
8109 NOREF(pVM);
8110
8111 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8112 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8113
8114 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
8115 {
8116 int rc = hmR0VmxExportSharedDebugState(pVCpu, pCtx);
8117 AssertRC(rc);
8118 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
8119
8120 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8121 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
8122 {
8123 rc = hmR0VmxExportGuestRflags(pVCpu, pCtx);
8124 AssertRC(rc);
8125 }
8126 }
8127
8128 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
8129 {
8130 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8131 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
8132 }
8133
8134 AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
8135 ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8136}
8137
8138
8139/**
8140 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8141 *
8142 * @returns Strict VBox status code (i.e. informational status codes too).
8143 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8144 * without unrestricted guest access and the VMMDev is not presently
8145 * mapped (e.g. EFI32).
8146 *
8147 * @param pVM The cross context VM structure.
8148 * @param pVCpu The cross context virtual CPU structure.
8149 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8150 * out-of-sync. Make sure to update the required fields
8151 * before using them.
8152 *
8153 * @remarks No-long-jump zone!!!
8154 */
8155static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
8156{
8157 HMVMX_ASSERT_PREEMPT_SAFE();
8158 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8159 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8160
8161#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8162 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_ALL_GUEST;
8163#endif
8164
8165 /*
8166 * For many exits it's only RIP that changes and hence try to export it first
8167 * without going through a lot of change flag checks.
8168 */
8169 VBOXSTRICTRC rcStrict;
8170 uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8171 if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
8172 {
8173 rcStrict = hmR0VmxExportGuestRip(pVCpu, pMixedCtx);
8174 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8175 { /* likely */}
8176 else
8177 AssertMsgFailedReturn(("hmR0VmxExportGuestRip failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8178 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
8179 }
8180 else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8181 {
8182 rcStrict = hmR0VmxExportGuestState(pVM, pVCpu, pMixedCtx);
8183 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8184 { /* likely */}
8185 else
8186 {
8187 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("hmR0VmxExportGuestState failed! rc=%Rrc\n",
8188 VBOXSTRICTRC_VAL(rcStrict)));
8189 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8190 return rcStrict;
8191 }
8192 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
8193 }
8194 else
8195 rcStrict = VINF_SUCCESS;
8196
8197#ifdef VBOX_STRICT
8198 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8199 fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8200 AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
8201 ("fCtxChanged=%#RX64\n", fCtxChanged));
8202#endif
8203 return rcStrict;
8204}
8205
8206
8207/**
8208 * Does the preparations before executing guest code in VT-x.
8209 *
8210 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8211 * recompiler/IEM. We must be cautious what we do here regarding committing
8212 * guest-state information into the VMCS assuming we assuredly execute the
8213 * guest in VT-x mode.
8214 *
8215 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8216 * the common-state (TRPM/forceflags), we must undo those changes so that the
8217 * recompiler/IEM can (and should) use them when it resumes guest execution.
8218 * Otherwise such operations must be done when we can no longer exit to ring-3.
8219 *
8220 * @returns Strict VBox status code (i.e. informational status codes too).
8221 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8222 * have been disabled.
8223 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8224 * double-fault into the guest.
8225 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8226 * dispatched directly.
8227 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8228 *
8229 * @param pVM The cross context VM structure.
8230 * @param pVCpu The cross context virtual CPU structure.
8231 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8232 * out-of-sync. Make sure to update the required fields
8233 * before using them.
8234 * @param pVmxTransient Pointer to the VMX transient structure.
8235 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8236 * us ignore some of the reasons for returning to
8237 * ring-3, and return VINF_EM_DBG_STEPPED if event
8238 * dispatching took place.
8239 */
8240static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8241{
8242 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8243
8244#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8245 PGMRZDynMapFlushAutoSet(pVCpu);
8246#endif
8247
8248 /* Check force flag actions that might require us to go back to ring-3. */
8249 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8250 if (rcStrict == VINF_SUCCESS)
8251 { /* FFs doesn't get set all the time. */ }
8252 else
8253 return rcStrict;
8254
8255 /*
8256 * Setup the virtualized-APIC accesses.
8257 *
8258 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8259 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8260 *
8261 * This is the reason we do it here and not in hmR0VmxExportGuestState().
8262 */
8263 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8264 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8265 && PDMHasApic(pVM))
8266 {
8267 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8268 Assert(u64MsrApicBase);
8269 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8270
8271 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8272
8273 /* Unalias any existing mapping. */
8274 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8275 AssertRCReturn(rc, rc);
8276
8277 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8278 Log4Func(("Mapped HC APIC-access page at %#RGp\n", GCPhysApicBase));
8279 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8280 AssertRCReturn(rc, rc);
8281
8282 /* Update the per-VCPU cache of the APIC base MSR. */
8283 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8284 }
8285
8286 if (TRPMHasTrap(pVCpu))
8287 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8288 uint32_t fIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8289
8290 /*
8291 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8292 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8293 */
8294 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, fIntrState, fStepping);
8295 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8296 { /* likely */ }
8297 else
8298 {
8299 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8300 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8301 return rcStrict;
8302 }
8303
8304 /*
8305 * No longjmps to ring-3 from this point on!!!
8306 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8307 * This also disables flushing of the R0-logger instance (if any).
8308 */
8309 VMMRZCallRing3Disable(pVCpu);
8310
8311 /*
8312 * Export the guest state bits.
8313 *
8314 * We cannot perform longjmps while loading the guest state because we do not preserve the
8315 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8316 * CPU migration.
8317 *
8318 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8319 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8320 * Hence, loading of the guest state needs to be done -after- injection of events.
8321 */
8322 rcStrict = hmR0VmxExportGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8323 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8324 { /* likely */ }
8325 else
8326 {
8327 VMMRZCallRing3Enable(pVCpu);
8328 return rcStrict;
8329 }
8330
8331 /*
8332 * We disable interrupts so that we don't miss any interrupts that would flag
8333 * preemption (IPI/timers etc.) when thread-context hooks aren't used and we've
8334 * been running with preemption disabled for a while. Since this is purly to aid
8335 * the RTThreadPreemptIsPending code, it doesn't matter that it may temporarily
8336 * reenable and disable interrupt on NT.
8337 *
8338 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8339 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8340 *
8341 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8342 * executing guest code.
8343 */
8344 pVmxTransient->fEFlags = ASMIntDisableFlags();
8345
8346 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8347 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8348 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8349 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8350 {
8351 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8352 {
8353 pVCpu->hm.s.Event.fPending = false;
8354
8355 /*
8356 * We've injected any pending events. This is really the point of no return (to ring-3).
8357 *
8358 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
8359 * returns from this function, so don't enable them here.
8360 */
8361 return VINF_SUCCESS;
8362 }
8363
8364 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8365 rcStrict = VINF_EM_RAW_INTERRUPT;
8366 }
8367 else
8368 {
8369 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8370 rcStrict = VINF_EM_RAW_TO_R3;
8371 }
8372
8373 ASMSetFlags(pVmxTransient->fEFlags);
8374 VMMRZCallRing3Enable(pVCpu);
8375
8376 return rcStrict;
8377}
8378
8379
8380/**
8381 * Prepares to run guest code in VT-x and we've committed to doing so. This
8382 * means there is no backing out to ring-3 or anywhere else at this
8383 * point.
8384 *
8385 * @param pVM The cross context VM structure.
8386 * @param pVCpu The cross context virtual CPU structure.
8387 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8388 * out-of-sync. Make sure to update the required fields
8389 * before using them.
8390 * @param pVmxTransient Pointer to the VMX transient structure.
8391 *
8392 * @remarks Called with preemption disabled.
8393 * @remarks No-long-jump zone!!!
8394 */
8395static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8396{
8397 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8398 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8399 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8400
8401 /*
8402 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8403 */
8404 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8405 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8406
8407 if (!CPUMIsGuestFPUStateActive(pVCpu))
8408 {
8409 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8410 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8411 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT;
8412 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8413 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
8414 }
8415
8416 /*
8417 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8418 */
8419 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8420 && pVCpu->hm.s.vmx.cMsrs > 0)
8421 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8422
8423 /*
8424 * Re-save the host state bits as we may've been preempted (only happens when
8425 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8426 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8427 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8428 * See @bugref{8432}.
8429 */
8430 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8431 {
8432 int rc = hmR0VmxExportHostState(pVCpu);
8433 AssertRC(rc);
8434 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
8435 }
8436 Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
8437
8438 /*
8439 * Export the state shared between host and guest (FPU, debug, lazy MSRs).
8440 */
8441 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
8442 hmR0VmxExportSharedState(pVM, pVCpu, pMixedCtx);
8443 AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8444
8445 /* Store status of the shared guest-host state at the time of VM-entry. */
8446#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8447 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8448 {
8449 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8450 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8451 }
8452 else
8453#endif
8454 {
8455 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8456 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8457 }
8458
8459 /*
8460 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8461 */
8462 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8463 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8464
8465 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8466 RTCPUID idCurrentCpu = pCpu->idCpu;
8467 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8468 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8469 {
8470 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8471 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8472 }
8473
8474 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8475 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8476 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8477 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8478
8479 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8480
8481 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8482 to start executing. */
8483
8484 /*
8485 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8486 */
8487 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8488 {
8489 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8490 {
8491 bool fMsrUpdated;
8492 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
8493 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8494 &fMsrUpdated);
8495 AssertRC(rc2);
8496 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8497 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8498 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8499 }
8500 else
8501 {
8502 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8503 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8504 }
8505 }
8506
8507 if (pVM->cpum.ro.GuestFeatures.fIbrs)
8508 {
8509 bool fMsrUpdated;
8510 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_OTHER_MSRS);
8511 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
8512 &fMsrUpdated);
8513 AssertRC(rc2);
8514 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8515 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8516 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8517 }
8518
8519#ifdef VBOX_STRICT
8520 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8521 hmR0VmxCheckHostEferMsr(pVCpu);
8522 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8523#endif
8524#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8525 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8526 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8527 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8528#endif
8529}
8530
8531
8532/**
8533 * Performs some essential restoration of state after running guest code in
8534 * VT-x.
8535 *
8536 * @param pVM The cross context VM structure.
8537 * @param pVCpu The cross context virtual CPU structure.
8538 * @param pVmxTransient Pointer to the VMX transient structure.
8539 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8540 *
8541 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8542 *
8543 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8544 * unconditionally when it is safe to do so.
8545 */
8546static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8547{
8548 uint64_t const uHostTsc = ASMReadTSC();
8549 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8550
8551 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8552 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8553 pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8554 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8555 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8556 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8557
8558 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8559 TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TSCOffset);
8560
8561 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8562 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8563 Assert(!ASMIntAreEnabled());
8564 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8565
8566#if HC_ARCH_BITS == 64
8567 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8568#endif
8569#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8570 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8571 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8572 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8573#else
8574 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8575#endif
8576#ifdef VBOX_STRICT
8577 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8578#endif
8579 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8580
8581 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8582 uint32_t uExitReason;
8583 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8584 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8585 AssertRC(rc);
8586 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8587 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8588
8589 if (rcVMRun == VINF_SUCCESS)
8590 {
8591 /*
8592 * Update the VM-exit history array here even if the VM-entry failed due to:
8593 * - Invalid guest state.
8594 * - MSR loading.
8595 * - Machine-check event.
8596 *
8597 * In any of the above cases we will still have a "valid" VM-exit reason
8598 * despite @a fVMEntryFailed being false.
8599 *
8600 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8601 *
8602 * Note! We don't have CS or RIP at this point. Will probably address that later
8603 * by amending the history entry added here.
8604 */
8605 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
8606 UINT64_MAX, uHostTsc);
8607
8608 if (!pVmxTransient->fVMEntryFailed)
8609 {
8610 VMMRZCallRing3Enable(pVCpu);
8611
8612 /*
8613 * Import the guest-interruptibility state always as we need it while evaluating
8614 * injecting events on re-entry.
8615 *
8616 * We don't import CR0 (when Unrestricted guest execution is unavailable) despite
8617 * checking for real-mode while exporting the state because all bits that cause
8618 * mode changes wrt CR0 are intercepted.
8619 */
8620 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
8621 AssertRC(rc);
8622
8623#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8624 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
8625 AssertRC(rc);
8626#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8627 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_RFLAGS);
8628 AssertRC(rc);
8629#endif
8630
8631 /*
8632 * Sync the TPR shadow with our APIC state.
8633 */
8634 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8635 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8636 {
8637 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8638 AssertRC(rc);
8639 ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8640 }
8641
8642 return;
8643 }
8644 }
8645 else
8646 {
8647 Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
8648 }
8649
8650 VMMRZCallRing3Enable(pVCpu);
8651}
8652
8653
8654/**
8655 * Runs the guest code using VT-x the normal way.
8656 *
8657 * @returns VBox status code.
8658 * @param pVM The cross context VM structure.
8659 * @param pVCpu The cross context virtual CPU structure.
8660 * @param pCtx Pointer to the guest-CPU context.
8661 *
8662 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8663 */
8664static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8665{
8666 VMXTRANSIENT VmxTransient;
8667 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8668 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8669 uint32_t cLoops = 0;
8670
8671 for (;; cLoops++)
8672 {
8673 Assert(!HMR0SuspendPending());
8674 HMVMX_ASSERT_CPU_SAFE();
8675
8676 /* Preparatory work for running guest code, this may force us to return
8677 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8678 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8679 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8680 if (rcStrict != VINF_SUCCESS)
8681 break;
8682
8683 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8684 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8685 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8686
8687 /* Restore any residual host-state and save any bits shared between host
8688 and guest into the guest-CPU state. Re-enables interrupts! */
8689 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
8690
8691 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8692 if (RT_SUCCESS(rcRun))
8693 { /* very likely */ }
8694 else
8695 {
8696 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8697 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
8698 return rcRun;
8699 }
8700
8701 /* Profile the VM-exit. */
8702 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8703 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8704 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8705 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8706 HMVMX_START_EXIT_DISPATCH_PROF();
8707
8708 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8709
8710 /* Handle the VM-exit. */
8711#ifdef HMVMX_USE_FUNCTION_TABLE
8712 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8713#else
8714 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8715#endif
8716 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8717 if (rcStrict == VINF_SUCCESS)
8718 {
8719 if (cLoops <= pVM->hm.s.cMaxResumeLoops)
8720 continue; /* likely */
8721 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8722 rcStrict = VINF_EM_RAW_INTERRUPT;
8723 }
8724 break;
8725 }
8726
8727 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8728 return rcStrict;
8729}
8730
8731
8732
8733/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8734 * probes.
8735 *
8736 * The following few functions and associated structure contains the bloat
8737 * necessary for providing detailed debug events and dtrace probes as well as
8738 * reliable host side single stepping. This works on the principle of
8739 * "subclassing" the normal execution loop and workers. We replace the loop
8740 * method completely and override selected helpers to add necessary adjustments
8741 * to their core operation.
8742 *
8743 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8744 * any performance for debug and analysis features.
8745 *
8746 * @{
8747 */
8748
8749/**
8750 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
8751 * the debug run loop.
8752 */
8753typedef struct VMXRUNDBGSTATE
8754{
8755 /** The RIP we started executing at. This is for detecting that we stepped. */
8756 uint64_t uRipStart;
8757 /** The CS we started executing with. */
8758 uint16_t uCsStart;
8759
8760 /** Whether we've actually modified the 1st execution control field. */
8761 bool fModifiedProcCtls : 1;
8762 /** Whether we've actually modified the 2nd execution control field. */
8763 bool fModifiedProcCtls2 : 1;
8764 /** Whether we've actually modified the exception bitmap. */
8765 bool fModifiedXcptBitmap : 1;
8766
8767 /** We desire the modified the CR0 mask to be cleared. */
8768 bool fClearCr0Mask : 1;
8769 /** We desire the modified the CR4 mask to be cleared. */
8770 bool fClearCr4Mask : 1;
8771 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
8772 uint32_t fCpe1Extra;
8773 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
8774 uint32_t fCpe1Unwanted;
8775 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
8776 uint32_t fCpe2Extra;
8777 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
8778 uint32_t bmXcptExtra;
8779 /** The sequence number of the Dtrace provider settings the state was
8780 * configured against. */
8781 uint32_t uDtraceSettingsSeqNo;
8782 /** VM-exits to check (one bit per VM-exit). */
8783 uint32_t bmExitsToCheck[3];
8784
8785 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
8786 uint32_t fProcCtlsInitial;
8787 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
8788 uint32_t fProcCtls2Initial;
8789 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
8790 uint32_t bmXcptInitial;
8791} VMXRUNDBGSTATE;
8792AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
8793typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
8794
8795
8796/**
8797 * Initializes the VMXRUNDBGSTATE structure.
8798 *
8799 * @param pVCpu The cross context virtual CPU structure of the
8800 * calling EMT.
8801 * @param pCtx The CPU register context to go with @a pVCpu.
8802 * @param pDbgState The structure to initialize.
8803 */
8804static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
8805{
8806 pDbgState->uRipStart = pCtx->rip;
8807 pDbgState->uCsStart = pCtx->cs.Sel;
8808
8809 pDbgState->fModifiedProcCtls = false;
8810 pDbgState->fModifiedProcCtls2 = false;
8811 pDbgState->fModifiedXcptBitmap = false;
8812 pDbgState->fClearCr0Mask = false;
8813 pDbgState->fClearCr4Mask = false;
8814 pDbgState->fCpe1Extra = 0;
8815 pDbgState->fCpe1Unwanted = 0;
8816 pDbgState->fCpe2Extra = 0;
8817 pDbgState->bmXcptExtra = 0;
8818 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
8819 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
8820 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
8821}
8822
8823
8824/**
8825 * Updates the VMSC fields with changes requested by @a pDbgState.
8826 *
8827 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
8828 * immediately before executing guest code, i.e. when interrupts are disabled.
8829 * We don't check status codes here as we cannot easily assert or return in the
8830 * latter case.
8831 *
8832 * @param pVCpu The cross context virtual CPU structure.
8833 * @param pDbgState The debug state.
8834 */
8835static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
8836{
8837 /*
8838 * Ensure desired flags in VMCS control fields are set.
8839 * (Ignoring write failure here, as we're committed and it's just debug extras.)
8840 *
8841 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
8842 * there should be no stale data in pCtx at this point.
8843 */
8844 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
8845 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
8846 {
8847 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
8848 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
8849 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8850 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
8851 pDbgState->fModifiedProcCtls = true;
8852 }
8853
8854 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
8855 {
8856 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
8857 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
8858 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
8859 pDbgState->fModifiedProcCtls2 = true;
8860 }
8861
8862 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
8863 {
8864 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
8865 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8866 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
8867 pDbgState->fModifiedXcptBitmap = true;
8868 }
8869
8870 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
8871 {
8872 pVCpu->hm.s.vmx.u32CR0Mask = 0;
8873 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
8874 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
8875 }
8876
8877 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
8878 {
8879 pVCpu->hm.s.vmx.u32CR4Mask = 0;
8880 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
8881 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
8882 }
8883}
8884
8885
8886static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
8887{
8888 /*
8889 * Restore VM-exit control settings as we may not reenter this function the
8890 * next time around.
8891 */
8892 /* We reload the initial value, trigger what we can of recalculations the
8893 next time around. From the looks of things, that's all that's required atm. */
8894 if (pDbgState->fModifiedProcCtls)
8895 {
8896 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
8897 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
8898 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
8899 AssertRCReturn(rc2, rc2);
8900 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
8901 }
8902
8903 /* We're currently the only ones messing with this one, so just restore the
8904 cached value and reload the field. */
8905 if ( pDbgState->fModifiedProcCtls2
8906 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
8907 {
8908 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
8909 AssertRCReturn(rc2, rc2);
8910 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
8911 }
8912
8913 /* If we've modified the exception bitmap, we restore it and trigger
8914 reloading and partial recalculation the next time around. */
8915 if (pDbgState->fModifiedXcptBitmap)
8916 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
8917
8918 return rcStrict;
8919}
8920
8921
8922/**
8923 * Configures VM-exit controls for current DBGF and DTrace settings.
8924 *
8925 * This updates @a pDbgState and the VMCS execution control fields to reflect
8926 * the necessary VM-exits demanded by DBGF and DTrace.
8927 *
8928 * @param pVCpu The cross context virtual CPU structure.
8929 * @param pDbgState The debug state.
8930 * @param pVmxTransient Pointer to the VMX transient structure. May update
8931 * fUpdateTscOffsettingAndPreemptTimer.
8932 */
8933static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
8934{
8935 /*
8936 * Take down the dtrace serial number so we can spot changes.
8937 */
8938 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
8939 ASMCompilerBarrier();
8940
8941 /*
8942 * We'll rebuild most of the middle block of data members (holding the
8943 * current settings) as we go along here, so start by clearing it all.
8944 */
8945 pDbgState->bmXcptExtra = 0;
8946 pDbgState->fCpe1Extra = 0;
8947 pDbgState->fCpe1Unwanted = 0;
8948 pDbgState->fCpe2Extra = 0;
8949 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
8950 pDbgState->bmExitsToCheck[i] = 0;
8951
8952 /*
8953 * Software interrupts (INT XXh) - no idea how to trigger these...
8954 */
8955 PVM pVM = pVCpu->CTX_SUFF(pVM);
8956 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
8957 || VBOXVMM_INT_SOFTWARE_ENABLED())
8958 {
8959 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
8960 }
8961
8962 /*
8963 * INT3 breakpoints - triggered by #BP exceptions.
8964 */
8965 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
8966 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
8967
8968 /*
8969 * Exception bitmap and XCPT events+probes.
8970 */
8971 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
8972 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
8973 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
8974
8975 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
8976 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
8977 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
8978 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
8979 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
8980 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
8981 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
8982 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
8983 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
8984 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
8985 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
8986 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
8987 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
8988 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
8989 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
8990 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
8991 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
8992 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
8993
8994 if (pDbgState->bmXcptExtra)
8995 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
8996
8997 /*
8998 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
8999 *
9000 * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
9001 * So, when adding/changing/removing please don't forget to update it.
9002 *
9003 * Some of the macros are picking up local variables to save horizontal space,
9004 * (being able to see it in a table is the lesser evil here).
9005 */
9006#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9007 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9008 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9009#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9010 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9011 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9012 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9013 } else do { } while (0)
9014#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9015 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9016 { \
9017 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9018 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9019 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9020 } else do { } while (0)
9021#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9022 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9023 { \
9024 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9025 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9026 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9027 } else do { } while (0)
9028#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9029 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9030 { \
9031 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9032 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9033 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9034 } else do { } while (0)
9035
9036 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9037 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9038 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9039 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9040 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9041
9042 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9043 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9044 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9045 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9046 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9047 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9048 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9049 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9050 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9051 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9052 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9053 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9054 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9055 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9056 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9057 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9058 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9059 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9060 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9061 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9062 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9063 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9064 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9065 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9066 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9067 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9068 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9069 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9070 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9071 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9072 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9073 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9074 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9075 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9076 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9077 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9078
9079 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9080 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9081 {
9082 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
9083 | CPUMCTX_EXTRN_CR4
9084 | CPUMCTX_EXTRN_APIC_TPR);
9085 AssertRC(rc);
9086
9087#if 0 /** @todo fix me */
9088 pDbgState->fClearCr0Mask = true;
9089 pDbgState->fClearCr4Mask = true;
9090#endif
9091 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9092 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9093 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9094 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9095 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9096 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9097 require clearing here and in the loop if we start using it. */
9098 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9099 }
9100 else
9101 {
9102 if (pDbgState->fClearCr0Mask)
9103 {
9104 pDbgState->fClearCr0Mask = false;
9105 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
9106 }
9107 if (pDbgState->fClearCr4Mask)
9108 {
9109 pDbgState->fClearCr4Mask = false;
9110 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
9111 }
9112 }
9113 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9114 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9115
9116 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9117 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9118 {
9119 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9120 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9121 }
9122 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9123 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9124
9125 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9126 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9127 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9128 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9129 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9130 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9131 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9132 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9133#if 0 /** @todo too slow, fix handler. */
9134 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9135#endif
9136 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9137
9138 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9139 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9140 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9141 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9142 {
9143 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9144 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9145 }
9146 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9147 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9148 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9149 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9150
9151 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9152 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9153 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9154 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9155 {
9156 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9157 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9158 }
9159 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9160 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9161 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9162 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9163
9164 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9165 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9166 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9167 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9168 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9169 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9170 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9171 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9172 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9173 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9174 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9175 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9176 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9177 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9178 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9179 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9180 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9181 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9182 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9183 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9184 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9185 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9186
9187#undef IS_EITHER_ENABLED
9188#undef SET_ONLY_XBM_IF_EITHER_EN
9189#undef SET_CPE1_XBM_IF_EITHER_EN
9190#undef SET_CPEU_XBM_IF_EITHER_EN
9191#undef SET_CPE2_XBM_IF_EITHER_EN
9192
9193 /*
9194 * Sanitize the control stuff.
9195 */
9196 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9197 if (pDbgState->fCpe2Extra)
9198 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9199 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9200 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9201 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9202 {
9203 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9204 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9205 }
9206
9207 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9208 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9209 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9210 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9211}
9212
9213
9214/**
9215 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9216 * appropriate.
9217 *
9218 * The caller has checked the VM-exit against the
9219 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9220 * already, so we don't have to do that either.
9221 *
9222 * @returns Strict VBox status code (i.e. informational status codes too).
9223 * @param pVM The cross context VM structure.
9224 * @param pVCpu The cross context virtual CPU structure.
9225 * @param pMixedCtx Pointer to the guest-CPU context.
9226 * @param pVmxTransient Pointer to the VMX-transient structure.
9227 * @param uExitReason The VM-exit reason.
9228 *
9229 * @remarks The name of this function is displayed by dtrace, so keep it short
9230 * and to the point. No longer than 33 chars long, please.
9231 */
9232static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9233 PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9234{
9235 /*
9236 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9237 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9238 *
9239 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9240 * does. Must add/change/remove both places. Same ordering, please.
9241 *
9242 * Added/removed events must also be reflected in the next section
9243 * where we dispatch dtrace events.
9244 */
9245 bool fDtrace1 = false;
9246 bool fDtrace2 = false;
9247 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9248 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9249 uint32_t uEventArg = 0;
9250#define SET_EXIT(a_EventSubName) \
9251 do { \
9252 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9253 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9254 } while (0)
9255#define SET_BOTH(a_EventSubName) \
9256 do { \
9257 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9258 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9259 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9260 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9261 } while (0)
9262 switch (uExitReason)
9263 {
9264 case VMX_EXIT_MTF:
9265 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9266
9267 case VMX_EXIT_XCPT_OR_NMI:
9268 {
9269 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9270 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9271 {
9272 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9273 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9274 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9275 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9276 {
9277 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9278 {
9279 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9280 uEventArg = pVmxTransient->uExitIntErrorCode;
9281 }
9282 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9283 switch (enmEvent1)
9284 {
9285 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9286 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9287 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9288 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9289 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9290 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9291 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9292 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9293 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9294 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9295 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9296 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9297 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9298 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9299 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9300 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9301 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9302 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9303 default: break;
9304 }
9305 }
9306 else
9307 AssertFailed();
9308 break;
9309
9310 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9311 uEventArg = idxVector;
9312 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9313 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9314 break;
9315 }
9316 break;
9317 }
9318
9319 case VMX_EXIT_TRIPLE_FAULT:
9320 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9321 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9322 break;
9323 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9324 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9325 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9326 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9327 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9328
9329 /* Instruction specific VM-exits: */
9330 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9331 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9332 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9333 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9334 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9335 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9336 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9337 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9338 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9339 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9340 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9341 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9342 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9343 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9344 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9345 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9346 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9347 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9348 case VMX_EXIT_MOV_CRX:
9349 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9350 if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
9351 SET_BOTH(CRX_READ);
9352 else
9353 SET_BOTH(CRX_WRITE);
9354 uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQualification);
9355 break;
9356 case VMX_EXIT_MOV_DRX:
9357 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9358 if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification)
9359 == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
9360 SET_BOTH(DRX_READ);
9361 else
9362 SET_BOTH(DRX_WRITE);
9363 uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification);
9364 break;
9365 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9366 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9367 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9368 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9369 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9370 case VMX_EXIT_XDTR_ACCESS:
9371 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9372 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9373 {
9374 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9375 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9376 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9377 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9378 }
9379 break;
9380
9381 case VMX_EXIT_TR_ACCESS:
9382 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9383 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9384 {
9385 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9386 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9387 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9388 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9389 }
9390 break;
9391
9392 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9393 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9394 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9395 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9396 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9397 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9398 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9399 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9400 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9401 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9402 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9403
9404 /* Events that aren't relevant at this point. */
9405 case VMX_EXIT_EXT_INT:
9406 case VMX_EXIT_INT_WINDOW:
9407 case VMX_EXIT_NMI_WINDOW:
9408 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9409 case VMX_EXIT_PREEMPT_TIMER:
9410 case VMX_EXIT_IO_INSTR:
9411 break;
9412
9413 /* Errors and unexpected events. */
9414 case VMX_EXIT_INIT_SIGNAL:
9415 case VMX_EXIT_SIPI:
9416 case VMX_EXIT_IO_SMI:
9417 case VMX_EXIT_SMI:
9418 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9419 case VMX_EXIT_ERR_MSR_LOAD:
9420 case VMX_EXIT_ERR_MACHINE_CHECK:
9421 break;
9422
9423 default:
9424 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9425 break;
9426 }
9427#undef SET_BOTH
9428#undef SET_EXIT
9429
9430 /*
9431 * Dtrace tracepoints go first. We do them here at once so we don't
9432 * have to copy the guest state saving and stuff a few dozen times.
9433 * Down side is that we've got to repeat the switch, though this time
9434 * we use enmEvent since the probes are a subset of what DBGF does.
9435 */
9436 if (fDtrace1 || fDtrace2)
9437 {
9438 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9439 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9440 switch (enmEvent1)
9441 {
9442 /** @todo consider which extra parameters would be helpful for each probe. */
9443 case DBGFEVENT_END: break;
9444 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9445 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9446 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9447 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9448 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9449 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9450 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9451 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9452 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9453 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9454 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9455 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9456 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9457 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9458 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9459 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9460 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9461 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9462 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9463 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9464 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9465 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9466 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9467 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9468 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9469 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9470 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9471 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9472 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9473 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9474 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9475 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9476 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9477 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9478 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9479 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9480 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9481 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9482 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9483 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9484 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9485 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9486 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9487 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9488 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9489 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9490 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9491 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9492 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9493 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9494 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9495 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9496 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9497 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9498 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9499 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9500 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9501 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9502 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9503 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9504 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9505 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9506 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9507 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9508 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9509 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9510 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9511 }
9512 switch (enmEvent2)
9513 {
9514 /** @todo consider which extra parameters would be helpful for each probe. */
9515 case DBGFEVENT_END: break;
9516 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9517 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9518 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9519 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9520 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9521 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9522 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9523 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9524 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9525 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9526 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9527 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9528 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9529 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9530 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9531 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9532 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9533 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9534 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9535 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9536 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9537 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9538 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9539 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9540 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9541 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9542 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9543 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9544 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9545 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9546 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9547 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9548 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9549 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9550 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9551 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9552 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9553 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9554 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9555 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9556 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9557 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9558 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9559 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9560 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9561 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9562 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9563 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9564 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9565 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9566 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9567 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9568 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9569 }
9570 }
9571
9572 /*
9573 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9574 * the DBGF call will do a full check).
9575 *
9576 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9577 * Note! If we have to events, we prioritize the first, i.e. the instruction
9578 * one, in order to avoid event nesting.
9579 */
9580 if ( enmEvent1 != DBGFEVENT_END
9581 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9582 {
9583 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9584 if (rcStrict != VINF_SUCCESS)
9585 return rcStrict;
9586 }
9587 else if ( enmEvent2 != DBGFEVENT_END
9588 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9589 {
9590 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9591 if (rcStrict != VINF_SUCCESS)
9592 return rcStrict;
9593 }
9594
9595 return VINF_SUCCESS;
9596}
9597
9598
9599/**
9600 * Single-stepping VM-exit filtering.
9601 *
9602 * This is preprocessing the VM-exits and deciding whether we've gotten far
9603 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9604 * handling is performed.
9605 *
9606 * @returns Strict VBox status code (i.e. informational status codes too).
9607 * @param pVM The cross context VM structure.
9608 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9609 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9610 * out-of-sync. Make sure to update the required
9611 * fields before using them.
9612 * @param pVmxTransient Pointer to the VMX-transient structure.
9613 * @param uExitReason The VM-exit reason.
9614 * @param pDbgState The debug state.
9615 */
9616DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9617 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9618{
9619 /*
9620 * Expensive (saves context) generic dtrace VM-exit probe.
9621 */
9622 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9623 { /* more likely */ }
9624 else
9625 {
9626 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9627 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9628 AssertRC(rc);
9629 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9630 }
9631
9632 /*
9633 * Check for host NMI, just to get that out of the way.
9634 */
9635 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9636 { /* normally likely */ }
9637 else
9638 {
9639 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9640 AssertRCReturn(rc2, rc2);
9641 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9642 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9643 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9644 }
9645
9646 /*
9647 * Check for single stepping event if we're stepping.
9648 */
9649 if (pVCpu->hm.s.fSingleInstruction)
9650 {
9651 switch (uExitReason)
9652 {
9653 case VMX_EXIT_MTF:
9654 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9655
9656 /* Various events: */
9657 case VMX_EXIT_XCPT_OR_NMI:
9658 case VMX_EXIT_EXT_INT:
9659 case VMX_EXIT_TRIPLE_FAULT:
9660 case VMX_EXIT_INT_WINDOW:
9661 case VMX_EXIT_NMI_WINDOW:
9662 case VMX_EXIT_TASK_SWITCH:
9663 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9664 case VMX_EXIT_APIC_ACCESS:
9665 case VMX_EXIT_EPT_VIOLATION:
9666 case VMX_EXIT_EPT_MISCONFIG:
9667 case VMX_EXIT_PREEMPT_TIMER:
9668
9669 /* Instruction specific VM-exits: */
9670 case VMX_EXIT_CPUID:
9671 case VMX_EXIT_GETSEC:
9672 case VMX_EXIT_HLT:
9673 case VMX_EXIT_INVD:
9674 case VMX_EXIT_INVLPG:
9675 case VMX_EXIT_RDPMC:
9676 case VMX_EXIT_RDTSC:
9677 case VMX_EXIT_RSM:
9678 case VMX_EXIT_VMCALL:
9679 case VMX_EXIT_VMCLEAR:
9680 case VMX_EXIT_VMLAUNCH:
9681 case VMX_EXIT_VMPTRLD:
9682 case VMX_EXIT_VMPTRST:
9683 case VMX_EXIT_VMREAD:
9684 case VMX_EXIT_VMRESUME:
9685 case VMX_EXIT_VMWRITE:
9686 case VMX_EXIT_VMXOFF:
9687 case VMX_EXIT_VMXON:
9688 case VMX_EXIT_MOV_CRX:
9689 case VMX_EXIT_MOV_DRX:
9690 case VMX_EXIT_IO_INSTR:
9691 case VMX_EXIT_RDMSR:
9692 case VMX_EXIT_WRMSR:
9693 case VMX_EXIT_MWAIT:
9694 case VMX_EXIT_MONITOR:
9695 case VMX_EXIT_PAUSE:
9696 case VMX_EXIT_XDTR_ACCESS:
9697 case VMX_EXIT_TR_ACCESS:
9698 case VMX_EXIT_INVEPT:
9699 case VMX_EXIT_RDTSCP:
9700 case VMX_EXIT_INVVPID:
9701 case VMX_EXIT_WBINVD:
9702 case VMX_EXIT_XSETBV:
9703 case VMX_EXIT_RDRAND:
9704 case VMX_EXIT_INVPCID:
9705 case VMX_EXIT_VMFUNC:
9706 case VMX_EXIT_RDSEED:
9707 case VMX_EXIT_XSAVES:
9708 case VMX_EXIT_XRSTORS:
9709 {
9710 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
9711 | CPUMCTX_EXTRN_CS);
9712 AssertRCReturn(rc, rc);
9713 if ( pMixedCtx->rip != pDbgState->uRipStart
9714 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
9715 return VINF_EM_DBG_STEPPED;
9716 break;
9717 }
9718
9719 /* Errors and unexpected events: */
9720 case VMX_EXIT_INIT_SIGNAL:
9721 case VMX_EXIT_SIPI:
9722 case VMX_EXIT_IO_SMI:
9723 case VMX_EXIT_SMI:
9724 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9725 case VMX_EXIT_ERR_MSR_LOAD:
9726 case VMX_EXIT_ERR_MACHINE_CHECK:
9727 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9728 break;
9729
9730 default:
9731 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9732 break;
9733 }
9734 }
9735
9736 /*
9737 * Check for debugger event breakpoints and dtrace probes.
9738 */
9739 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9740 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9741 {
9742 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9743 if (rcStrict != VINF_SUCCESS)
9744 return rcStrict;
9745 }
9746
9747 /*
9748 * Normal processing.
9749 */
9750#ifdef HMVMX_USE_FUNCTION_TABLE
9751 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9752#else
9753 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9754#endif
9755}
9756
9757
9758/**
9759 * Single steps guest code using VT-x.
9760 *
9761 * @returns Strict VBox status code (i.e. informational status codes too).
9762 * @param pVM The cross context VM structure.
9763 * @param pVCpu The cross context virtual CPU structure.
9764 * @param pCtx Pointer to the guest-CPU context.
9765 *
9766 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
9767 */
9768static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9769{
9770 VMXTRANSIENT VmxTransient;
9771 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9772
9773 /* Set HMCPU indicators. */
9774 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
9775 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
9776 pVCpu->hm.s.fDebugWantRdTscExit = false;
9777 pVCpu->hm.s.fUsingDebugLoop = true;
9778
9779 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
9780 VMXRUNDBGSTATE DbgState;
9781 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
9782 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9783
9784 /*
9785 * The loop.
9786 */
9787 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9788 for (uint32_t cLoops = 0; ; cLoops++)
9789 {
9790 Assert(!HMR0SuspendPending());
9791 HMVMX_ASSERT_CPU_SAFE();
9792 bool fStepping = pVCpu->hm.s.fSingleInstruction;
9793
9794 /*
9795 * Preparatory work for running guest code, this may force us to return
9796 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
9797 */
9798 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9799 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
9800 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
9801 if (rcStrict != VINF_SUCCESS)
9802 break;
9803
9804 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9805 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
9806
9807 /*
9808 * Now we can run the guest code.
9809 */
9810 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9811
9812 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9813
9814 /*
9815 * Restore any residual host-state and save any bits shared between host
9816 * and guest into the guest-CPU state. Re-enables interrupts!
9817 */
9818 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
9819
9820 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9821 if (RT_SUCCESS(rcRun))
9822 { /* very likely */ }
9823 else
9824 {
9825 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9826 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9827 return rcRun;
9828 }
9829
9830 /* Profile the VM-exit. */
9831 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9832 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9833 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9834 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9835 HMVMX_START_EXIT_DISPATCH_PROF();
9836
9837 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9838
9839 /*
9840 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
9841 */
9842 rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
9843 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9844 if (rcStrict != VINF_SUCCESS)
9845 break;
9846 if (cLoops > pVM->hm.s.cMaxResumeLoops)
9847 {
9848 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9849 rcStrict = VINF_EM_RAW_INTERRUPT;
9850 break;
9851 }
9852
9853 /*
9854 * Stepping: Did the RIP change, if so, consider it a single step.
9855 * Otherwise, make sure one of the TFs gets set.
9856 */
9857 if (fStepping)
9858 {
9859 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
9860 | CPUMCTX_EXTRN_CS);
9861 AssertRC(rc);
9862 if ( pCtx->rip != DbgState.uRipStart
9863 || pCtx->cs.Sel != DbgState.uCsStart)
9864 {
9865 rcStrict = VINF_EM_DBG_STEPPED;
9866 break;
9867 }
9868 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
9869 }
9870
9871 /*
9872 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
9873 */
9874 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
9875 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9876 }
9877
9878 /*
9879 * Clear the X86_EFL_TF if necessary.
9880 */
9881 if (pVCpu->hm.s.fClearTrapFlag)
9882 {
9883 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
9884 AssertRC(rc);
9885 pVCpu->hm.s.fClearTrapFlag = false;
9886 pCtx->eflags.Bits.u1TF = 0;
9887 }
9888 /** @todo there seems to be issues with the resume flag when the monitor trap
9889 * flag is pending without being used. Seen early in bios init when
9890 * accessing APIC page in protected mode. */
9891
9892 /*
9893 * Restore VM-exit control settings as we may not reenter this function the
9894 * next time around.
9895 */
9896 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
9897
9898 /* Restore HMCPU indicators. */
9899 pVCpu->hm.s.fUsingDebugLoop = false;
9900 pVCpu->hm.s.fDebugWantRdTscExit = false;
9901 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
9902
9903 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9904 return rcStrict;
9905}
9906
9907
9908/** @} */
9909
9910
9911/**
9912 * Checks if any expensive dtrace probes are enabled and we should go to the
9913 * debug loop.
9914 *
9915 * @returns true if we should use debug loop, false if not.
9916 */
9917static bool hmR0VmxAnyExpensiveProbesEnabled(void)
9918{
9919 /* It's probably faster to OR the raw 32-bit counter variables together.
9920 Since the variables are in an array and the probes are next to one
9921 another (more or less), we have good locality. So, better read
9922 eight-nine cache lines ever time and only have one conditional, than
9923 128+ conditionals, right? */
9924 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
9925 | VBOXVMM_XCPT_DE_ENABLED_RAW()
9926 | VBOXVMM_XCPT_DB_ENABLED_RAW()
9927 | VBOXVMM_XCPT_BP_ENABLED_RAW()
9928 | VBOXVMM_XCPT_OF_ENABLED_RAW()
9929 | VBOXVMM_XCPT_BR_ENABLED_RAW()
9930 | VBOXVMM_XCPT_UD_ENABLED_RAW()
9931 | VBOXVMM_XCPT_NM_ENABLED_RAW()
9932 | VBOXVMM_XCPT_DF_ENABLED_RAW()
9933 | VBOXVMM_XCPT_TS_ENABLED_RAW()
9934 | VBOXVMM_XCPT_NP_ENABLED_RAW()
9935 | VBOXVMM_XCPT_SS_ENABLED_RAW()
9936 | VBOXVMM_XCPT_GP_ENABLED_RAW()
9937 | VBOXVMM_XCPT_PF_ENABLED_RAW()
9938 | VBOXVMM_XCPT_MF_ENABLED_RAW()
9939 | VBOXVMM_XCPT_AC_ENABLED_RAW()
9940 | VBOXVMM_XCPT_XF_ENABLED_RAW()
9941 | VBOXVMM_XCPT_VE_ENABLED_RAW()
9942 | VBOXVMM_XCPT_SX_ENABLED_RAW()
9943 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
9944 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
9945 ) != 0
9946 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
9947 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
9948 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
9949 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
9950 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
9951 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
9952 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
9953 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
9954 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
9955 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
9956 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
9957 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
9958 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
9959 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
9960 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
9961 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
9962 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
9963 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
9964 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
9965 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
9966 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
9967 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
9968 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
9969 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
9970 | VBOXVMM_INSTR_STR_ENABLED_RAW()
9971 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
9972 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
9973 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
9974 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
9975 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
9976 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
9977 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
9978 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
9979 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
9980 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
9981 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
9982 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
9983 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
9984 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
9985 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
9986 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
9987 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
9988 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
9989 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
9990 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
9991 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
9992 ) != 0
9993 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
9994 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
9995 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
9996 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
9997 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
9998 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
9999 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10000 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10001 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10002 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10003 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10004 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10005 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10006 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10007 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10008 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10009 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10010 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10011 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10012 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10013 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10014 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10015 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10016 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10017 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10018 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10019 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10020 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10021 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10022 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10023 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10024 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10025 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10026 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10027 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10028 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10029 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10030 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10031 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10032 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10033 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10034 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10035 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10036 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10037 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10038 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10039 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10040 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10041 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10042 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10043 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10044 ) != 0;
10045}
10046
10047
10048/**
10049 * Runs the guest code using VT-x.
10050 *
10051 * @returns Strict VBox status code (i.e. informational status codes too).
10052 * @param pVM The cross context VM structure.
10053 * @param pVCpu The cross context virtual CPU structure.
10054 * @param pCtx Pointer to the guest-CPU context.
10055 */
10056VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10057{
10058 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10059 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10060 HMVMX_ASSERT_PREEMPT_SAFE();
10061
10062 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10063
10064 VBOXSTRICTRC rcStrict;
10065 if ( !pVCpu->hm.s.fUseDebugLoop
10066 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10067 && !DBGFIsStepping(pVCpu)
10068 && !pVM->dbgf.ro.cEnabledInt3Breakpoints)
10069 rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10070 else
10071 rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10072
10073 if (rcStrict == VERR_EM_INTERPRETER)
10074 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10075 else if (rcStrict == VINF_EM_RESET)
10076 rcStrict = VINF_EM_TRIPLE_FAULT;
10077
10078 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10079 if (RT_FAILURE(rc2))
10080 {
10081 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10082 rcStrict = rc2;
10083 }
10084 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10085 return rcStrict;
10086}
10087
10088
10089#ifndef HMVMX_USE_FUNCTION_TABLE
10090DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10091{
10092#ifdef DEBUG_ramshankar
10093#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
10094 do { \
10095 if (a_fSave != 0) \
10096 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
10097 VBOXSTRICTRC rcStrict = a_CallExpr; \
10098 if (a_fSave != 0) \
10099 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
10100 return rcStrict; \
10101 } while (0)
10102#else
10103# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
10104#endif
10105 switch (rcReason)
10106 {
10107 case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10108 case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10109 case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10110 case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10111 case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10112 case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10113 case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10114 case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10115 case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10116 case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10117 case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10118 case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10119 case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10120 case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10121 case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10122 case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10123 case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10124 case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10125 case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10126 case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10127 case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10128 case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10129 case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10130 case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10131 case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10132 case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10133 case VMX_EXIT_XDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10134 case VMX_EXIT_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10135 case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10136 case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10137 case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10138 case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10139 case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10140 case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10141
10142 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10143 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10144 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10145 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10146 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10147 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10148 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10149 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10150 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10151
10152 case VMX_EXIT_VMCLEAR:
10153 case VMX_EXIT_VMLAUNCH:
10154 case VMX_EXIT_VMPTRLD:
10155 case VMX_EXIT_VMPTRST:
10156 case VMX_EXIT_VMREAD:
10157 case VMX_EXIT_VMRESUME:
10158 case VMX_EXIT_VMWRITE:
10159 case VMX_EXIT_VMXOFF:
10160 case VMX_EXIT_VMXON:
10161 case VMX_EXIT_INVEPT:
10162 case VMX_EXIT_INVVPID:
10163 case VMX_EXIT_VMFUNC:
10164 case VMX_EXIT_XSAVES:
10165 case VMX_EXIT_XRSTORS:
10166 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10167
10168 case VMX_EXIT_ENCLS:
10169 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10170 case VMX_EXIT_PML_FULL:
10171 default:
10172 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10173 }
10174#undef VMEXIT_CALL_RET
10175}
10176#endif /* !HMVMX_USE_FUNCTION_TABLE */
10177
10178
10179#ifdef VBOX_STRICT
10180/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10181# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10182 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10183
10184# define HMVMX_ASSERT_PREEMPT_CPUID() \
10185 do { \
10186 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10187 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10188 } while (0)
10189
10190# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10191 do { \
10192 AssertPtr(pVCpu); \
10193 AssertPtr(pMixedCtx); \
10194 AssertPtr(pVmxTransient); \
10195 Assert(pVmxTransient->fVMEntryFailed == false); \
10196 Assert(ASMIntAreEnabled()); \
10197 HMVMX_ASSERT_PREEMPT_SAFE(); \
10198 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10199 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10200 HMVMX_ASSERT_PREEMPT_SAFE(); \
10201 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10202 HMVMX_ASSERT_PREEMPT_CPUID(); \
10203 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10204 } while (0)
10205
10206# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10207 do { \
10208 Log4Func(("\n")); \
10209 } while (0)
10210#else /* nonstrict builds: */
10211# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10212 do { \
10213 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10214 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10215 } while (0)
10216# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10217#endif
10218
10219
10220/**
10221 * Advances the guest RIP by the specified number of bytes.
10222 *
10223 * @param pVCpu The cross context virtual CPU structure.
10224 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10225 * out-of-sync. Make sure to update the required fields
10226 * before using them.
10227 * @param cbInstr Number of bytes to advance the RIP by.
10228 *
10229 * @remarks No-long-jump zone!!!
10230 */
10231DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10232{
10233 /* Advance the RIP. */
10234 pMixedCtx->rip += cbInstr;
10235 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
10236
10237 /* Update interrupt inhibition. */
10238 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10239 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10240 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10241}
10242
10243
10244/**
10245 * Advances the guest RIP after reading it from the VMCS.
10246 *
10247 * @returns VBox status code, no informational status codes.
10248 * @param pVCpu The cross context virtual CPU structure.
10249 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10250 * out-of-sync. Make sure to update the required fields
10251 * before using them.
10252 * @param pVmxTransient Pointer to the VMX transient structure.
10253 *
10254 * @remarks No-long-jump zone!!!
10255 */
10256static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10257{
10258 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10259 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
10260 | CPUMCTX_EXTRN_RFLAGS);
10261 AssertRCReturn(rc, rc);
10262
10263 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10264
10265 /*
10266 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10267 * pending debug exception field as it takes care of priority of events.
10268 *
10269 * See Intel spec. 32.2.1 "Debug Exceptions".
10270 */
10271 if ( !pVCpu->hm.s.fSingleInstruction
10272 && pMixedCtx->eflags.Bits.u1TF)
10273 {
10274 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
10275 AssertRCReturn(rc, rc);
10276 }
10277
10278 return VINF_SUCCESS;
10279}
10280
10281
10282/**
10283 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10284 * and update error record fields accordingly.
10285 *
10286 * @return VMX_IGS_* return codes.
10287 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10288 * wrong with the guest state.
10289 *
10290 * @param pVM The cross context VM structure.
10291 * @param pVCpu The cross context virtual CPU structure.
10292 * @param pCtx Pointer to the guest-CPU state.
10293 *
10294 * @remarks This function assumes our cache of the VMCS controls
10295 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10296 */
10297static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10298{
10299#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10300#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10301 uError = (err); \
10302 break; \
10303 } else do { } while (0)
10304
10305 int rc;
10306 uint32_t uError = VMX_IGS_ERROR;
10307 uint32_t u32Val;
10308 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10309
10310 do
10311 {
10312 /*
10313 * CR0.
10314 */
10315 uint32_t fSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10316 uint32_t fZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10317 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10318 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10319 if (fUnrestrictedGuest)
10320 fSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
10321
10322 uint32_t uGuestCR0;
10323 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uGuestCR0);
10324 AssertRCBreak(rc);
10325 HMVMX_CHECK_BREAK((uGuestCR0 & fSetCR0) == fSetCR0, VMX_IGS_CR0_FIXED1);
10326 HMVMX_CHECK_BREAK(!(uGuestCR0 & ~fZapCR0), VMX_IGS_CR0_FIXED0);
10327 if ( !fUnrestrictedGuest
10328 && (uGuestCR0 & X86_CR0_PG)
10329 && !(uGuestCR0 & X86_CR0_PE))
10330 {
10331 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10332 }
10333
10334 /*
10335 * CR4.
10336 */
10337 uint64_t fSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10338 uint64_t fZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10339
10340 uint32_t uGuestCR4;
10341 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uGuestCR4);
10342 AssertRCBreak(rc);
10343 HMVMX_CHECK_BREAK((uGuestCR4 & fSetCR4) == fSetCR4, VMX_IGS_CR4_FIXED1);
10344 HMVMX_CHECK_BREAK(!(uGuestCR4 & ~fZapCR4), VMX_IGS_CR4_FIXED0);
10345
10346 /*
10347 * IA32_DEBUGCTL MSR.
10348 */
10349 uint64_t u64Val;
10350 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10351 AssertRCBreak(rc);
10352 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10353 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10354 {
10355 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10356 }
10357 uint64_t u64DebugCtlMsr = u64Val;
10358
10359#ifdef VBOX_STRICT
10360 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10361 AssertRCBreak(rc);
10362 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10363#endif
10364 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10365
10366 /*
10367 * RIP and RFLAGS.
10368 */
10369 uint32_t u32Eflags;
10370#if HC_ARCH_BITS == 64
10371 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10372 AssertRCBreak(rc);
10373 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10374 if ( !fLongModeGuest
10375 || !pCtx->cs.Attr.n.u1Long)
10376 {
10377 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10378 }
10379 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10380 * must be identical if the "IA-32e mode guest" VM-entry
10381 * control is 1 and CS.L is 1. No check applies if the
10382 * CPU supports 64 linear-address bits. */
10383
10384 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10385 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10386 AssertRCBreak(rc);
10387 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10388 VMX_IGS_RFLAGS_RESERVED);
10389 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10390 u32Eflags = u64Val;
10391#else
10392 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10393 AssertRCBreak(rc);
10394 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10395 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10396#endif
10397
10398 if ( fLongModeGuest
10399 || ( fUnrestrictedGuest
10400 && !(uGuestCR0 & X86_CR0_PE)))
10401 {
10402 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10403 }
10404
10405 uint32_t u32EntryInfo;
10406 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10407 AssertRCBreak(rc);
10408 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10409 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10410 {
10411 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10412 }
10413
10414 /*
10415 * 64-bit checks.
10416 */
10417#if HC_ARCH_BITS == 64
10418 if (fLongModeGuest)
10419 {
10420 HMVMX_CHECK_BREAK(uGuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10421 HMVMX_CHECK_BREAK(uGuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10422 }
10423
10424 if ( !fLongModeGuest
10425 && (uGuestCR4 & X86_CR4_PCIDE))
10426 {
10427 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10428 }
10429
10430 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10431 * 51:32 beyond the processor's physical-address width are 0. */
10432
10433 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10434 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10435 {
10436 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10437 }
10438
10439 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10440 AssertRCBreak(rc);
10441 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10442
10443 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10444 AssertRCBreak(rc);
10445 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10446#endif
10447
10448 /*
10449 * PERF_GLOBAL MSR.
10450 */
10451 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10452 {
10453 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10454 AssertRCBreak(rc);
10455 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10456 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10457 }
10458
10459 /*
10460 * PAT MSR.
10461 */
10462 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10463 {
10464 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10465 AssertRCBreak(rc);
10466 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10467 for (unsigned i = 0; i < 8; i++)
10468 {
10469 uint8_t u8Val = (u64Val & 0xff);
10470 if ( u8Val != 0 /* UC */
10471 && u8Val != 1 /* WC */
10472 && u8Val != 4 /* WT */
10473 && u8Val != 5 /* WP */
10474 && u8Val != 6 /* WB */
10475 && u8Val != 7 /* UC- */)
10476 {
10477 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10478 }
10479 u64Val >>= 8;
10480 }
10481 }
10482
10483 /*
10484 * EFER MSR.
10485 */
10486 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10487 {
10488 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10489 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10490 AssertRCBreak(rc);
10491 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10492 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10493 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10494 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10495 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10496 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10497 || !(uGuestCR0 & X86_CR0_PG)
10498 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10499 VMX_IGS_EFER_LMA_LME_MISMATCH);
10500 }
10501
10502 /*
10503 * Segment registers.
10504 */
10505 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10506 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10507 if (!(u32Eflags & X86_EFL_VM))
10508 {
10509 /* CS */
10510 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10511 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10512 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10513 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10514 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10515 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10516 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10517 /* CS cannot be loaded with NULL in protected mode. */
10518 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10519 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10520 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10521 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10522 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10523 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10524 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10525 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10526 else
10527 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10528
10529 /* SS */
10530 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10531 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10532 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10533 if ( !(pCtx->cr0 & X86_CR0_PE)
10534 || pCtx->cs.Attr.n.u4Type == 3)
10535 {
10536 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10537 }
10538 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10539 {
10540 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10541 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10542 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10543 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10544 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10545 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10546 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10547 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10548 }
10549
10550 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
10551 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10552 {
10553 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10554 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10555 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10556 || pCtx->ds.Attr.n.u4Type > 11
10557 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10558 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10559 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10560 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10561 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10562 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10563 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10564 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10565 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10566 }
10567 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10568 {
10569 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10570 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10571 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10572 || pCtx->es.Attr.n.u4Type > 11
10573 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10574 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10575 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10576 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10577 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10578 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10579 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10580 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10581 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10582 }
10583 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10584 {
10585 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10586 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10587 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10588 || pCtx->fs.Attr.n.u4Type > 11
10589 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10590 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10591 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10592 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10593 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10594 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10595 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10596 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10597 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10598 }
10599 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10600 {
10601 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10602 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10603 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10604 || pCtx->gs.Attr.n.u4Type > 11
10605 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10606 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10607 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10608 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10609 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10610 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10611 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10612 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10613 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10614 }
10615 /* 64-bit capable CPUs. */
10616#if HC_ARCH_BITS == 64
10617 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10618 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10619 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10620 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10621 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10622 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10623 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10624 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10625 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10626 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10627 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10628#endif
10629 }
10630 else
10631 {
10632 /* V86 mode checks. */
10633 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10634 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10635 {
10636 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10637 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10638 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10639 }
10640 else
10641 {
10642 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10643 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10644 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10645 }
10646
10647 /* CS */
10648 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10649 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10650 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10651 /* SS */
10652 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10653 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10654 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10655 /* DS */
10656 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10657 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10658 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10659 /* ES */
10660 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10661 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10662 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10663 /* FS */
10664 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10665 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10666 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10667 /* GS */
10668 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10669 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10670 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10671 /* 64-bit capable CPUs. */
10672#if HC_ARCH_BITS == 64
10673 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10674 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10675 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10676 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10677 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10678 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10679 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10680 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10681 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10682 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10683 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10684#endif
10685 }
10686
10687 /*
10688 * TR.
10689 */
10690 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10691 /* 64-bit capable CPUs. */
10692#if HC_ARCH_BITS == 64
10693 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10694#endif
10695 if (fLongModeGuest)
10696 {
10697 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10698 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10699 }
10700 else
10701 {
10702 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10703 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10704 VMX_IGS_TR_ATTR_TYPE_INVALID);
10705 }
10706 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10707 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10708 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10709 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10710 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10711 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10712 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10713 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10714
10715 /*
10716 * GDTR and IDTR.
10717 */
10718#if HC_ARCH_BITS == 64
10719 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10720 AssertRCBreak(rc);
10721 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10722
10723 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10724 AssertRCBreak(rc);
10725 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10726#endif
10727
10728 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10729 AssertRCBreak(rc);
10730 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10731
10732 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10733 AssertRCBreak(rc);
10734 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10735
10736 /*
10737 * Guest Non-Register State.
10738 */
10739 /* Activity State. */
10740 uint32_t u32ActivityState;
10741 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10742 AssertRCBreak(rc);
10743 HMVMX_CHECK_BREAK( !u32ActivityState
10744 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
10745 VMX_IGS_ACTIVITY_STATE_INVALID);
10746 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10747 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10748 uint32_t u32IntrState;
10749 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
10750 AssertRCBreak(rc);
10751 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
10752 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10753 {
10754 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10755 }
10756
10757 /** @todo Activity state and injecting interrupts. Left as a todo since we
10758 * currently don't use activity states but ACTIVE. */
10759
10760 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10761 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10762
10763 /* Guest interruptibility-state. */
10764 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10765 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10766 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
10767 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10768 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10769 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10770 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10771 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10772 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10773 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
10774 {
10775 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10776 {
10777 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10778 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10779 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10780 }
10781 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10782 {
10783 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10784 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10785 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10786 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10787 }
10788 }
10789 /** @todo Assumes the processor is not in SMM. */
10790 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10791 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10792 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10793 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10794 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10795 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
10796 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10797 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10798 {
10799 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
10800 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10801 }
10802
10803 /* Pending debug exceptions. */
10804#if HC_ARCH_BITS == 64
10805 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
10806 AssertRCBreak(rc);
10807 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10808 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10809 u32Val = u64Val; /* For pending debug exceptions checks below. */
10810#else
10811 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
10812 AssertRCBreak(rc);
10813 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10814 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10815#endif
10816
10817 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10818 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
10819 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10820 {
10821 if ( (u32Eflags & X86_EFL_TF)
10822 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10823 {
10824 /* Bit 14 is PendingDebug.BS. */
10825 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10826 }
10827 if ( !(u32Eflags & X86_EFL_TF)
10828 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10829 {
10830 /* Bit 14 is PendingDebug.BS. */
10831 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10832 }
10833 }
10834
10835 /* VMCS link pointer. */
10836 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10837 AssertRCBreak(rc);
10838 if (u64Val != UINT64_C(0xffffffffffffffff))
10839 {
10840 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10841 /** @todo Bits beyond the processor's physical-address width MBZ. */
10842 /** @todo 32-bit located in memory referenced by value of this field (as a
10843 * physical address) must contain the processor's VMCS revision ID. */
10844 /** @todo SMM checks. */
10845 }
10846
10847 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10848 * not using Nested Paging? */
10849 if ( pVM->hm.s.fNestedPaging
10850 && !fLongModeGuest
10851 && CPUMIsGuestInPAEModeEx(pCtx))
10852 {
10853 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10854 AssertRCBreak(rc);
10855 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10856
10857 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10858 AssertRCBreak(rc);
10859 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10860
10861 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10862 AssertRCBreak(rc);
10863 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10864
10865 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10866 AssertRCBreak(rc);
10867 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10868 }
10869
10870 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10871 if (uError == VMX_IGS_ERROR)
10872 uError = VMX_IGS_REASON_NOT_FOUND;
10873 } while (0);
10874
10875 pVCpu->hm.s.u32HMError = uError;
10876 return uError;
10877
10878#undef HMVMX_ERROR_BREAK
10879#undef HMVMX_CHECK_BREAK
10880}
10881
10882/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10883/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
10884/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10885
10886/** @name VM-exit handlers.
10887 * @{
10888 */
10889
10890/**
10891 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
10892 */
10893HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10894{
10895 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10896 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
10897 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
10898 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
10899 return VINF_SUCCESS;
10900 return VINF_EM_RAW_INTERRUPT;
10901}
10902
10903
10904/**
10905 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
10906 */
10907HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10908{
10909 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10910 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
10911
10912 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10913 AssertRCReturn(rc, rc);
10914
10915 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10916 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
10917 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
10918 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
10919
10920 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10921 {
10922 /*
10923 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
10924 * injected it ourselves and anything we inject is not going to cause a VM-exit directly
10925 * for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
10926 *
10927 * [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
10928 * [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
10929 */
10930 VMXDispatchHostNmi();
10931 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
10932 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10933 return VINF_SUCCESS;
10934 }
10935
10936 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10937 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10938 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
10939 { /* likely */ }
10940 else
10941 {
10942 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
10943 rcStrictRc1 = VINF_SUCCESS;
10944 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10945 return rcStrictRc1;
10946 }
10947
10948 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
10949 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
10950 switch (uIntType)
10951 {
10952 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
10953 Assert(uVector == X86_XCPT_DB);
10954 RT_FALL_THRU();
10955 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
10956 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
10957 RT_FALL_THRU();
10958 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
10959 {
10960 /*
10961 * If there's any exception caused as a result of event injection, the resulting
10962 * secondary/final execption will be pending, we shall continue guest execution
10963 * after injecting the event. The page-fault case is complicated and we manually
10964 * handle any currently pending event in hmR0VmxExitXcptPF.
10965 */
10966 if (!pVCpu->hm.s.Event.fPending)
10967 { /* likely */ }
10968 else if (uVector != X86_XCPT_PF)
10969 {
10970 rc = VINF_SUCCESS;
10971 break;
10972 }
10973
10974 switch (uVector)
10975 {
10976 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
10977 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
10978 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
10979 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
10980 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
10981 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
10982
10983 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10984 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10985 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
10986 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10987 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
10988 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10989 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
10990 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10991 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
10992 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10993 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
10994 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10995 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
10996 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10997 default:
10998 {
10999 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11000 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11001 {
11002 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11003 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11004 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11005
11006 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
11007 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11008 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11009 AssertRCReturn(rc, rc);
11010 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11011 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11012 0 /* GCPtrFaultAddress */);
11013 }
11014 else
11015 {
11016 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11017 pVCpu->hm.s.u32HMError = uVector;
11018 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11019 }
11020 break;
11021 }
11022 }
11023 break;
11024 }
11025
11026 default:
11027 {
11028 pVCpu->hm.s.u32HMError = uExitIntInfo;
11029 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11030 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11031 break;
11032 }
11033 }
11034 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11035 return rc;
11036}
11037
11038
11039/**
11040 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11041 */
11042HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11043{
11044 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11045
11046 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11047 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11048
11049 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11050 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11051 return VINF_SUCCESS;
11052}
11053
11054
11055/**
11056 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11057 */
11058HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11059{
11060 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11061 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11062 {
11063 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11064 HMVMX_RETURN_UNEXPECTED_EXIT();
11065 }
11066
11067 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11068
11069 /*
11070 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11071 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11072 */
11073 uint32_t fIntrState = 0;
11074 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11075 AssertRCReturn(rc, rc);
11076
11077 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11078 if ( fBlockSti
11079 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11080 {
11081 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11082 }
11083
11084 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11085 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11086
11087 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11088 return VINF_SUCCESS;
11089}
11090
11091
11092/**
11093 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11094 */
11095HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11096{
11097 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11098 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11099}
11100
11101
11102/**
11103 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11104 */
11105HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11106{
11107 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11108 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11109}
11110
11111
11112/**
11113 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11114 */
11115HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11116{
11117 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11118 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
11119
11120 /*
11121 * Get the state we need and update the exit history entry.
11122 */
11123 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11124 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
11125 | CPUMCTX_EXTRN_CS);
11126 AssertRCReturn(rc, rc);
11127
11128 VBOXSTRICTRC rcStrict;
11129 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11130 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
11131 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11132 if (!pExitRec)
11133 {
11134 /*
11135 * Regular CPUID instruction execution.
11136 */
11137 PVM pVM = pVCpu->CTX_SUFF(pVM);
11138 rcStrict = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11139 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11140 {
11141 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11142 Assert(pVmxTransient->cbInstr == 2);
11143 }
11144 else
11145 {
11146 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11147 rcStrict = VERR_EM_INTERPRETER;
11148 }
11149 }
11150 else
11151 {
11152 /*
11153 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
11154 */
11155 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
11156 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11157 AssertRCReturn(rc2, rc2);
11158
11159 Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11160 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11161
11162 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11163 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
11164
11165 Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11166 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11167 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11168 }
11169 return VBOXSTRICTRC_TODO(rcStrict);
11170}
11171
11172
11173/**
11174 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11175 */
11176HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11177{
11178 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11179 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4);
11180 AssertRCReturn(rc, rc);
11181
11182 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11183 return VINF_EM_RAW_EMULATE_INSTR;
11184
11185 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11186 HMVMX_RETURN_UNEXPECTED_EXIT();
11187}
11188
11189
11190/**
11191 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11192 */
11193HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11194{
11195 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11196 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11197 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11198 AssertRCReturn(rc, rc);
11199
11200 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11201 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11202 {
11203 /* If we get a spurious VM-exit when offsetting is enabled,
11204 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11205 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11206 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11207 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
11208 | HM_CHANGED_GUEST_RFLAGS);
11209 }
11210 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11211 {
11212 rcStrict = VINF_SUCCESS;
11213 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11214 }
11215 return rcStrict;
11216}
11217
11218
11219/**
11220 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11221 */
11222HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11223{
11224 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11225 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
11226 | CPUMCTX_EXTRN_TSC_AUX);
11227 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11228 AssertRCReturn(rc, rc);
11229
11230 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11231 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11232 {
11233 /* If we get a spurious VM-exit when offsetting is enabled,
11234 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11235 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11236 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11237 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
11238 | HM_CHANGED_GUEST_RFLAGS);
11239 }
11240 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11241 {
11242 rcStrict = VINF_SUCCESS;
11243 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11244 }
11245 return rcStrict;
11246}
11247
11248
11249/**
11250 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11251 */
11252HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11253{
11254 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11255 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4
11256 | CPUMCTX_EXTRN_CR0
11257 | CPUMCTX_EXTRN_RFLAGS
11258 | CPUMCTX_EXTRN_SS);
11259 AssertRCReturn(rc, rc);
11260
11261 PVM pVM = pVCpu->CTX_SUFF(pVM);
11262 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11263 if (RT_LIKELY(rc == VINF_SUCCESS))
11264 {
11265 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11266 Assert(pVmxTransient->cbInstr == 2);
11267 }
11268 else
11269 {
11270 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11271 rc = VERR_EM_INTERPRETER;
11272 }
11273 return rc;
11274}
11275
11276
11277/**
11278 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11279 */
11280HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11281{
11282 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11283
11284 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11285 if (EMAreHypercallInstructionsEnabled(pVCpu))
11286 {
11287 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
11288 | CPUMCTX_EXTRN_RFLAGS
11289 | CPUMCTX_EXTRN_CR0
11290 | CPUMCTX_EXTRN_SS
11291 | CPUMCTX_EXTRN_CS
11292 | CPUMCTX_EXTRN_EFER);
11293 AssertRCReturn(rc, rc);
11294
11295 /* Perform the hypercall. */
11296 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11297 if (rcStrict == VINF_SUCCESS)
11298 {
11299 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11300 AssertRCReturn(rc, rc);
11301 }
11302 else
11303 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11304 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11305 || RT_FAILURE(rcStrict));
11306
11307 /* If the hypercall changes anything other than guest's general-purpose registers,
11308 we would need to reload the guest changed bits here before VM-entry. */
11309 }
11310 else
11311 Log4Func(("Hypercalls not enabled\n"));
11312
11313 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11314 if (RT_FAILURE(rcStrict))
11315 {
11316 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11317 rcStrict = VINF_SUCCESS;
11318 }
11319
11320 return rcStrict;
11321}
11322
11323
11324/**
11325 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11326 */
11327HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11328{
11329 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11330 PVM pVM = pVCpu->CTX_SUFF(pVM);
11331 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11332
11333 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11334 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK);
11335 AssertRCReturn(rc, rc);
11336
11337 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11338 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11339 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11340 else
11341 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11342 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11343 return rcStrict;
11344}
11345
11346
11347/**
11348 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11349 */
11350HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11351{
11352 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11353 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11354 | CPUMCTX_EXTRN_RFLAGS
11355 | CPUMCTX_EXTRN_SS);
11356 AssertRCReturn(rc, rc);
11357
11358 PVM pVM = pVCpu->CTX_SUFF(pVM);
11359 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11360 if (RT_LIKELY(rc == VINF_SUCCESS))
11361 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11362 else
11363 {
11364 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11365 rc = VERR_EM_INTERPRETER;
11366 }
11367 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11368 return rc;
11369}
11370
11371
11372/**
11373 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11374 */
11375HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11376{
11377 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11378 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11379 | CPUMCTX_EXTRN_RFLAGS
11380 | CPUMCTX_EXTRN_SS);
11381 AssertRCReturn(rc, rc);
11382
11383 PVM pVM = pVCpu->CTX_SUFF(pVM);
11384 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11385 rc = VBOXSTRICTRC_VAL(rc2);
11386 if (RT_LIKELY( rc == VINF_SUCCESS
11387 || rc == VINF_EM_HALT))
11388 {
11389 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11390 AssertRCReturn(rc3, rc3);
11391
11392 if ( rc == VINF_EM_HALT
11393 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11394 rc = VINF_SUCCESS;
11395 }
11396 else
11397 {
11398 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11399 rc = VERR_EM_INTERPRETER;
11400 }
11401 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11402 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11403 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11404 return rc;
11405}
11406
11407
11408/**
11409 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11410 */
11411HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11412{
11413 /*
11414 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
11415 * mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
11416 * treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
11417 * VMX root operation. If we get here, something funny is going on.
11418 *
11419 * See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
11420 */
11421 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11422 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11423 HMVMX_RETURN_UNEXPECTED_EXIT();
11424}
11425
11426
11427/**
11428 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11429 */
11430HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11431{
11432 /*
11433 * This can only happen if we support dual-monitor treatment of SMI, which can be activated
11434 * by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
11435 * this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
11436 * an SMI. If we get here, something funny is going on.
11437 *
11438 * See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11439 * See Intel spec. 25.3 "Other Causes of VM-Exits"
11440 */
11441 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11442 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11443 HMVMX_RETURN_UNEXPECTED_EXIT();
11444}
11445
11446
11447/**
11448 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11449 */
11450HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11451{
11452 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11453 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11454 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11455 HMVMX_RETURN_UNEXPECTED_EXIT();
11456}
11457
11458
11459/**
11460 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11461 */
11462HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11463{
11464 /*
11465 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
11466 * We don't make use of it as our guests don't have direct access to the host LAPIC.
11467 * See Intel spec. 25.3 "Other Causes of VM-exits".
11468 */
11469 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11470 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11471 HMVMX_RETURN_UNEXPECTED_EXIT();
11472}
11473
11474
11475/**
11476 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11477 * VM-exit.
11478 */
11479HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11480{
11481 /*
11482 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11483 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11484 *
11485 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11486 * See Intel spec. "23.8 Restrictions on VMX operation".
11487 */
11488 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11489 return VINF_SUCCESS;
11490}
11491
11492
11493/**
11494 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11495 * VM-exit.
11496 */
11497HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11498{
11499 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11500 return VINF_EM_RESET;
11501}
11502
11503
11504/**
11505 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11506 */
11507HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11508{
11509 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11510 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11511
11512 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11513 AssertRCReturn(rc, rc);
11514
11515 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11516 rc = VINF_SUCCESS;
11517 else
11518 rc = VINF_EM_HALT;
11519
11520 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11521 if (rc != VINF_SUCCESS)
11522 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11523 return rc;
11524}
11525
11526
11527/**
11528 * VM-exit handler for instructions that result in a \#UD exception delivered to
11529 * the guest.
11530 */
11531HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11532{
11533 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11534 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11535 return VINF_SUCCESS;
11536}
11537
11538
11539/**
11540 * VM-exit handler for expiry of the VMX preemption timer.
11541 */
11542HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11543{
11544 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11545
11546 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11547 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11548
11549 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11550 PVM pVM = pVCpu->CTX_SUFF(pVM);
11551 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11552 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11553 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11554}
11555
11556
11557/**
11558 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11559 */
11560HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11561{
11562 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11563
11564 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11565 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
11566 | CPUMCTX_EXTRN_CR4);
11567 AssertRCReturn(rc, rc);
11568
11569 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11570 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11571 : HM_CHANGED_XCPT_RAISED_MASK);
11572
11573 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11574
11575 return rcStrict;
11576}
11577
11578
11579/**
11580 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11581 */
11582HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11583{
11584 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11585 /** @todo Use VM-exit instruction information. */
11586 return VERR_EM_INTERPRETER;
11587}
11588
11589
11590/**
11591 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11592 * Error VM-exit.
11593 */
11594HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11595{
11596 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11597 rc |= hmR0VmxCheckVmcsCtls(pVCpu);
11598 AssertRCReturn(rc, rc);
11599
11600 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11601 NOREF(uInvalidReason);
11602
11603#ifdef VBOX_STRICT
11604 uint32_t fIntrState;
11605 RTHCUINTREG uHCReg;
11606 uint64_t u64Val;
11607 uint32_t u32Val;
11608
11609 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11610 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11611 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11612 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11613 AssertRCReturn(rc, rc);
11614
11615 Log4(("uInvalidReason %u\n", uInvalidReason));
11616 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11617 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11618 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11619 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", fIntrState));
11620
11621 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11622 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11623 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11624 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11625 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11626 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11627 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11628 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11629 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11630 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11631 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11632 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11633#else
11634 NOREF(pVmxTransient);
11635#endif
11636
11637 hmR0DumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11638 return VERR_VMX_INVALID_GUEST_STATE;
11639}
11640
11641
11642/**
11643 * VM-exit handler for VM-entry failure due to an MSR-load
11644 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11645 */
11646HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11647{
11648 NOREF(pVmxTransient);
11649 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11650 HMVMX_RETURN_UNEXPECTED_EXIT();
11651}
11652
11653
11654/**
11655 * VM-exit handler for VM-entry failure due to a machine-check event
11656 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11657 */
11658HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11659{
11660 NOREF(pVmxTransient);
11661 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11662 HMVMX_RETURN_UNEXPECTED_EXIT();
11663}
11664
11665
11666/**
11667 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11668 * theory.
11669 */
11670HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11671{
11672 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
11673 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
11674 return VERR_VMX_UNDEFINED_EXIT_CODE;
11675}
11676
11677
11678/**
11679 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11680 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11681 * Conditional VM-exit.
11682 */
11683HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11684{
11685 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11686
11687 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
11688 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11689 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
11690 return VERR_EM_INTERPRETER;
11691 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11692 HMVMX_RETURN_UNEXPECTED_EXIT();
11693}
11694
11695
11696/**
11697 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11698 */
11699HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11700{
11701 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11702
11703 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
11704 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
11705 return VERR_EM_INTERPRETER;
11706 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11707 HMVMX_RETURN_UNEXPECTED_EXIT();
11708}
11709
11710
11711/**
11712 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11713 */
11714HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11715{
11716 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11717
11718 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
11719 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11720 | CPUMCTX_EXTRN_RFLAGS
11721 | CPUMCTX_EXTRN_SS);
11722 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11723 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
11724 AssertRCReturn(rc, rc);
11725 Log4Func(("ecx=%#RX32\n", pMixedCtx->ecx));
11726
11727#ifdef VBOX_STRICT
11728 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
11729 {
11730 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
11731 && pMixedCtx->ecx != MSR_K6_EFER)
11732 {
11733 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11734 pMixedCtx->ecx));
11735 HMVMX_RETURN_UNEXPECTED_EXIT();
11736 }
11737 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11738 {
11739 VMXMSREXITREAD enmRead;
11740 VMXMSREXITWRITE enmWrite;
11741 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
11742 AssertRCReturn(rc2, rc2);
11743 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11744 {
11745 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11746 HMVMX_RETURN_UNEXPECTED_EXIT();
11747 }
11748 }
11749 }
11750#endif
11751
11752 PVM pVM = pVCpu->CTX_SUFF(pVM);
11753 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11754 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
11755 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
11756 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11757 if (RT_SUCCESS(rc))
11758 {
11759 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11760 Assert(pVmxTransient->cbInstr == 2);
11761 }
11762 return rc;
11763}
11764
11765
11766/**
11767 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11768 */
11769HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11770{
11771 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11772 PVM pVM = pVCpu->CTX_SUFF(pVM);
11773 int rc = VINF_SUCCESS;
11774
11775 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
11776 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11777 | CPUMCTX_EXTRN_RFLAGS
11778 | CPUMCTX_EXTRN_SS);
11779 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11780 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
11781 AssertRCReturn(rc, rc);
11782 Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
11783
11784 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11785 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
11786 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
11787
11788 if (RT_SUCCESS(rc))
11789 {
11790 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11791
11792 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
11793 if ( pMixedCtx->ecx == MSR_IA32_APICBASE
11794 || ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
11795 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
11796 {
11797 /*
11798 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
11799 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
11800 * EMInterpretWrmsr() changes it.
11801 */
11802 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11803 }
11804 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
11805 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11806 else if (pMixedCtx->ecx == MSR_K6_EFER)
11807 {
11808 /*
11809 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
11810 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
11811 * the other bits as well, SCE and NXE. See @bugref{7368}.
11812 */
11813 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
11814 | HM_CHANGED_VMX_ENTRY_CTLS
11815 | HM_CHANGED_VMX_EXIT_CTLS);
11816 }
11817
11818 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
11819 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11820 {
11821 switch (pMixedCtx->ecx)
11822 {
11823 case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
11824 case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
11825 case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
11826 case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
11827 case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
11828 case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
11829 default:
11830 {
11831 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11832 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
11833 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11834 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
11835 break;
11836 }
11837 }
11838 }
11839#ifdef VBOX_STRICT
11840 else
11841 {
11842 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
11843 switch (pMixedCtx->ecx)
11844 {
11845 case MSR_IA32_SYSENTER_CS:
11846 case MSR_IA32_SYSENTER_EIP:
11847 case MSR_IA32_SYSENTER_ESP:
11848 case MSR_K8_FS_BASE:
11849 case MSR_K8_GS_BASE:
11850 {
11851 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
11852 HMVMX_RETURN_UNEXPECTED_EXIT();
11853 }
11854
11855 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
11856 default:
11857 {
11858 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11859 {
11860 /* EFER writes are always intercepted, see hmR0VmxExportGuestMsrs(). */
11861 if (pMixedCtx->ecx != MSR_K6_EFER)
11862 {
11863 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11864 pMixedCtx->ecx));
11865 HMVMX_RETURN_UNEXPECTED_EXIT();
11866 }
11867 }
11868
11869 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11870 {
11871 VMXMSREXITREAD enmRead;
11872 VMXMSREXITWRITE enmWrite;
11873 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
11874 AssertRCReturn(rc2, rc2);
11875 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
11876 {
11877 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11878 HMVMX_RETURN_UNEXPECTED_EXIT();
11879 }
11880 }
11881 break;
11882 }
11883 }
11884 }
11885#endif /* VBOX_STRICT */
11886 }
11887 return rc;
11888}
11889
11890
11891/**
11892 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
11893 */
11894HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11895{
11896 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11897 /** @todo The guest has likely hit a contended spinlock. We might want to
11898 * poke a schedule different guest VCPU. */
11899 return VINF_EM_RAW_INTERRUPT;
11900}
11901
11902
11903/**
11904 * VM-exit handler for when the TPR value is lowered below the specified
11905 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
11906 */
11907HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11908{
11909 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11910 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
11911
11912 /*
11913 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
11914 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
11915 */
11916 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
11917 return VINF_SUCCESS;
11918}
11919
11920
11921/**
11922 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
11923 * VM-exit.
11924 *
11925 * @retval VINF_SUCCESS when guest execution can continue.
11926 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
11927 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
11928 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
11929 * interpreter.
11930 */
11931HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11932{
11933 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11934 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
11935
11936 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11937 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11938 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11939 AssertRCReturn(rc, rc);
11940
11941 VBOXSTRICTRC rcStrict;
11942 PVM pVM = pVCpu->CTX_SUFF(pVM);
11943 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
11944 uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQualification);
11945 switch (uAccessType)
11946 {
11947 case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
11948 {
11949 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
11950 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
11951 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification));
11952 AssertMsg( rcStrict == VINF_SUCCESS
11953 || rcStrict == VINF_IEM_RAISED_XCPT
11954 || rcStrict == VINF_PGM_CHANGE_MODE
11955 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11956
11957 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
11958 {
11959 case 0:
11960 {
11961 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11962 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
11963 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
11964 break;
11965 }
11966
11967 case 2:
11968 {
11969 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
11970 /* Nothing to do here, CR2 it's not part of the VMCS. */
11971 break;
11972 }
11973
11974 case 3:
11975 {
11976 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
11977 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
11978 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR3);
11979 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
11980 break;
11981 }
11982
11983 case 4:
11984 {
11985 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
11986 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11987 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
11988 pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
11989 break;
11990 }
11991
11992 case 8:
11993 {
11994 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
11995 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
11996 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11997 break;
11998 }
11999 default:
12000 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification)));
12001 break;
12002 }
12003 break;
12004 }
12005
12006 case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
12007 {
12008 Assert( !pVM->hm.s.fNestedPaging
12009 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12010 || pVCpu->hm.s.fUsingDebugLoop
12011 || VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 3);
12012 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12013 Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 8
12014 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12015
12016 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12017 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification),
12018 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification));
12019 AssertMsg( rcStrict == VINF_SUCCESS
12020 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12021#ifdef VBOX_WITH_STATISTICS
12022 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
12023 {
12024 case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
12025 case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
12026 case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
12027 case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
12028 case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
12029 }
12030#endif
12031 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
12032 VBOXSTRICTRC_VAL(rcStrict)));
12033 if (VMX_EXIT_QUAL_CRX_GENREG(uExitQualification) == X86_GREG_xSP)
12034 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RSP);
12035 break;
12036 }
12037
12038 case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12039 {
12040 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12041 AssertMsg( rcStrict == VINF_SUCCESS
12042 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12043
12044 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12045 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12046 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12047 break;
12048 }
12049
12050 case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12051 {
12052 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12053 VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQualification));
12054 AssertMsg( rcStrict == VINF_SUCCESS
12055 || rcStrict == VINF_IEM_RAISED_XCPT
12056 || rcStrict == VINF_PGM_CHANGE_MODE,
12057 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12058
12059 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12060 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12061 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12062 break;
12063 }
12064
12065 default:
12066 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12067 VERR_VMX_UNEXPECTED_EXCEPTION);
12068 }
12069
12070 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
12071 : HM_CHANGED_XCPT_RAISED_MASK);
12072 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12073 NOREF(pVM);
12074 return rcStrict;
12075}
12076
12077
12078/**
12079 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12080 * VM-exit.
12081 */
12082HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12083{
12084 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12085 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12086 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
12087
12088 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12089 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12090 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
12091 | CPUMCTX_EXTRN_SREG_MASK
12092 | CPUMCTX_EXTRN_EFER);
12093 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12094 AssertRCReturn(rc, rc);
12095
12096 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12097 uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQualification);
12098 uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQualification);
12099 bool fIOWrite = ( VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQualification)
12100 == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12101 bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQualification);
12102 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12103 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12104 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12105
12106 /*
12107 * Update exit history to see if this exit can be optimized.
12108 */
12109 VBOXSTRICTRC rcStrict;
12110 PCEMEXITREC pExitRec = NULL;
12111 if ( !fGstStepping
12112 && !fDbgStepping)
12113 pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12114 !fIOString
12115 ? !fIOWrite
12116 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12117 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12118 : !fIOWrite
12119 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12120 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12121 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12122 if (!pExitRec)
12123 {
12124 /* I/O operation lookup arrays. */
12125 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12126 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12127 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12128 uint32_t const cbInstr = pVmxTransient->cbInstr;
12129 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12130 PVM pVM = pVCpu->CTX_SUFF(pVM);
12131 if (fIOString)
12132 {
12133 /*
12134 * INS/OUTS - I/O String instruction.
12135 *
12136 * Use instruction-information if available, otherwise fall back on
12137 * interpreting the instruction.
12138 */
12139 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12140 fIOWrite ? 'w' : 'r'));
12141 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12142 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12143 {
12144 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12145 AssertRCReturn(rc2, rc2);
12146 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12147 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12148 IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12149 bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification);
12150 if (fIOWrite)
12151 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12152 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12153 else
12154 {
12155 /*
12156 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12157 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12158 * See Intel Instruction spec. for "INS".
12159 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12160 */
12161 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12162 }
12163 }
12164 else
12165 rcStrict = IEMExecOne(pVCpu);
12166
12167 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12168 fUpdateRipAlready = true;
12169 }
12170 else
12171 {
12172 /*
12173 * IN/OUT - I/O instruction.
12174 */
12175 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12176 fIOWrite ? 'w' : 'r'));
12177 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12178 Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification));
12179 if (fIOWrite)
12180 {
12181 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12182 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12183 }
12184 else
12185 {
12186 uint32_t u32Result = 0;
12187 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12188 if (IOM_SUCCESS(rcStrict))
12189 {
12190 /* Save result of I/O IN instr. in AL/AX/EAX. */
12191 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12192 }
12193 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12194 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12195 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12196 }
12197 }
12198
12199 if (IOM_SUCCESS(rcStrict))
12200 {
12201 if (!fUpdateRipAlready)
12202 {
12203 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12204 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12205 }
12206
12207 /*
12208 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12209 * while booting Fedora 17 64-bit guest.
12210 *
12211 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12212 */
12213 if (fIOString)
12214 {
12215 /** @todo Single-step for INS/OUTS with REP prefix? */
12216 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12217 }
12218 else if ( !fDbgStepping
12219 && fGstStepping)
12220 {
12221 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
12222 AssertRCReturn(rc, rc);
12223 }
12224
12225 /*
12226 * If any I/O breakpoints are armed, we need to check if one triggered
12227 * and take appropriate action.
12228 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12229 */
12230 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12231 AssertRCReturn(rc, rc);
12232
12233 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12234 * execution engines about whether hyper BPs and such are pending. */
12235 uint32_t const uDr7 = pMixedCtx->dr[7];
12236 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12237 && X86_DR7_ANY_RW_IO(uDr7)
12238 && (pMixedCtx->cr4 & X86_CR4_DE))
12239 || DBGFBpIsHwIoArmed(pVM)))
12240 {
12241 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12242
12243 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12244 VMMRZCallRing3Disable(pVCpu);
12245 HM_DISABLE_PREEMPT();
12246
12247 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12248
12249 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12250 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12251 {
12252 /* Raise #DB. */
12253 if (fIsGuestDbgActive)
12254 ASMSetDR6(pMixedCtx->dr[6]);
12255 if (pMixedCtx->dr[7] != uDr7)
12256 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_DR7;
12257
12258 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12259 }
12260 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12261 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12262 else if ( rcStrict2 != VINF_SUCCESS
12263 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12264 rcStrict = rcStrict2;
12265 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12266
12267 HM_RESTORE_PREEMPT();
12268 VMMRZCallRing3Enable(pVCpu);
12269 }
12270 }
12271
12272#ifdef VBOX_STRICT
12273 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12274 Assert(!fIOWrite);
12275 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12276 Assert(fIOWrite);
12277 else
12278 {
12279# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12280 * statuses, that the VMM device and some others may return. See
12281 * IOM_SUCCESS() for guidance. */
12282 AssertMsg( RT_FAILURE(rcStrict)
12283 || rcStrict == VINF_SUCCESS
12284 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12285 || rcStrict == VINF_EM_DBG_BREAKPOINT
12286 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12287 || rcStrict == VINF_EM_RAW_TO_R3
12288 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12289# endif
12290 }
12291#endif
12292 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12293 }
12294 else
12295 {
12296 /*
12297 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12298 */
12299 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12300 AssertRCReturn(rc2, rc2);
12301 STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
12302 : fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
12303 Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12304 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12305 VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification) ? "REP " : "",
12306 fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
12307
12308 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12309 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12310
12311 Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12312 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12313 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12314 }
12315 return rcStrict;
12316}
12317
12318
12319/**
12320 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12321 * VM-exit.
12322 */
12323HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12324{
12325 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12326
12327 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12328 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12329 AssertRCReturn(rc, rc);
12330 if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12331 {
12332 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12333 AssertRCReturn(rc, rc);
12334 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12335 {
12336 uint32_t uErrCode;
12337 RTGCUINTPTR GCPtrFaultAddress;
12338 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12339 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12340 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12341 if (fErrorCodeValid)
12342 {
12343 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12344 AssertRCReturn(rc, rc);
12345 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12346 }
12347 else
12348 uErrCode = 0;
12349
12350 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12351 && uVector == X86_XCPT_PF)
12352 GCPtrFaultAddress = pMixedCtx->cr2;
12353 else
12354 GCPtrFaultAddress = 0;
12355
12356 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12357 0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
12358
12359 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12360 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12361 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12362 }
12363 }
12364
12365 /* Fall back to the interpreter to emulate the task-switch. */
12366 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12367 return VERR_EM_INTERPRETER;
12368}
12369
12370
12371/**
12372 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12373 */
12374HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12375{
12376 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12377 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12378 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12379 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12380 AssertRCReturn(rc, rc);
12381 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12382 return VINF_EM_DBG_STEPPED;
12383}
12384
12385
12386/**
12387 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12388 */
12389HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12390{
12391 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12392
12393 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12394
12395 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12396 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12397 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12398 {
12399 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12400 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12401 {
12402 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12403 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12404 }
12405 }
12406 else
12407 {
12408 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12409 rcStrict1 = VINF_SUCCESS;
12410 return rcStrict1;
12411 }
12412
12413 /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12414 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12415 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12416 AssertRCReturn(rc, rc);
12417
12418 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12419 uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12420 VBOXSTRICTRC rcStrict2;
12421 switch (uAccessType)
12422 {
12423 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12424 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12425 {
12426 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12427 || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
12428 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12429
12430 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12431 GCPhys &= PAGE_BASE_GC_MASK;
12432 GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12433 PVM pVM = pVCpu->CTX_SUFF(pVM);
12434 Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12435 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12436
12437 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12438 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12439 CPUMCTX2CORE(pMixedCtx), GCPhys);
12440 Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12441 if ( rcStrict2 == VINF_SUCCESS
12442 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12443 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12444 {
12445 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12446 | HM_CHANGED_GUEST_RSP
12447 | HM_CHANGED_GUEST_RFLAGS
12448 | HM_CHANGED_GUEST_APIC_TPR);
12449 rcStrict2 = VINF_SUCCESS;
12450 }
12451 break;
12452 }
12453
12454 default:
12455 Log4Func(("uAccessType=%#x\n", uAccessType));
12456 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12457 break;
12458 }
12459
12460 if (rcStrict2 != VINF_SUCCESS)
12461 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12462 return rcStrict2;
12463}
12464
12465
12466/**
12467 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12468 * VM-exit.
12469 */
12470HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12471{
12472 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12473
12474 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12475 if (pVmxTransient->fWasGuestDebugStateActive)
12476 {
12477 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12478 HMVMX_RETURN_UNEXPECTED_EXIT();
12479 }
12480
12481 if ( !pVCpu->hm.s.fSingleInstruction
12482 && !pVmxTransient->fWasHyperDebugStateActive)
12483 {
12484 Assert(!DBGFIsStepping(pVCpu));
12485 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12486
12487 /* Don't intercept MOV DRx any more. */
12488 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12489 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12490 AssertRCReturn(rc, rc);
12491
12492 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12493 VMMRZCallRing3Disable(pVCpu);
12494 HM_DISABLE_PREEMPT();
12495
12496 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12497 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12498 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12499
12500 HM_RESTORE_PREEMPT();
12501 VMMRZCallRing3Enable(pVCpu);
12502
12503#ifdef VBOX_WITH_STATISTICS
12504 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12505 AssertRCReturn(rc, rc);
12506 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12507 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12508 else
12509 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12510#endif
12511 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12512 return VINF_SUCCESS;
12513 }
12514
12515 /*
12516 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12517 * Update the segment registers and DR7 from the CPU.
12518 */
12519 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12520 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK
12521 | CPUMCTX_EXTRN_DR7);
12522 AssertRCReturn(rc, rc);
12523 Log4Func(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12524
12525 PVM pVM = pVCpu->CTX_SUFF(pVM);
12526 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12527 {
12528 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12529 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification),
12530 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification));
12531 if (RT_SUCCESS(rc))
12532 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12533 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12534 }
12535 else
12536 {
12537 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12538 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification),
12539 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification));
12540 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12541 }
12542
12543 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12544 if (RT_SUCCESS(rc))
12545 {
12546 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12547 AssertRCReturn(rc2, rc2);
12548 return VINF_SUCCESS;
12549 }
12550 return rc;
12551}
12552
12553
12554/**
12555 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12556 * Conditional VM-exit.
12557 */
12558HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12559{
12560 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12561 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12562
12563 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12564 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12565 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12566 {
12567 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12568 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12569 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12570 {
12571 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12572 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12573 }
12574 }
12575 else
12576 {
12577 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12578 rcStrict1 = VINF_SUCCESS;
12579 return rcStrict1;
12580 }
12581
12582 /*
12583 * Get sufficent state and update the exit history entry.
12584 */
12585 RTGCPHYS GCPhys;
12586 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12587 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12588 AssertRCReturn(rc, rc);
12589
12590 VBOXSTRICTRC rcStrict;
12591 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12592 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
12593 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12594 if (!pExitRec)
12595 {
12596 /*
12597 * If we succeed, resume guest execution.
12598 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12599 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12600 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12601 * weird case. See @bugref{6043}.
12602 */
12603 PVM pVM = pVCpu->CTX_SUFF(pVM);
12604 rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12605 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12606 if ( rcStrict == VINF_SUCCESS
12607 || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12608 || rcStrict == VERR_PAGE_NOT_PRESENT)
12609 {
12610 /* Successfully handled MMIO operation. */
12611 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12612 | HM_CHANGED_GUEST_RSP
12613 | HM_CHANGED_GUEST_RFLAGS
12614 | HM_CHANGED_GUEST_APIC_TPR);
12615 rcStrict = VINF_SUCCESS;
12616 }
12617 }
12618 else
12619 {
12620 /*
12621 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12622 */
12623 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
12624 int rc2 = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12625 AssertRCReturn(rc2, rc2);
12626
12627 Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12628 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12629
12630 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12631 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12632
12633 Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12634 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12635 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12636 }
12637 return VBOXSTRICTRC_TODO(rcStrict);
12638}
12639
12640
12641/**
12642 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12643 * VM-exit.
12644 */
12645HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12646{
12647 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12648 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12649
12650 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12651 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12652 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12653 {
12654 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12655 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12656 Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12657 }
12658 else
12659 {
12660 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12661 rcStrict1 = VINF_SUCCESS;
12662 return rcStrict1;
12663 }
12664
12665 RTGCPHYS GCPhys;
12666 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12667 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12668 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12669 AssertRCReturn(rc, rc);
12670
12671 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12672 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12673
12674 RTGCUINT uErrorCode = 0;
12675 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
12676 uErrorCode |= X86_TRAP_PF_ID;
12677 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_DATA_WRITE)
12678 uErrorCode |= X86_TRAP_PF_RW;
12679 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
12680 uErrorCode |= X86_TRAP_PF_P;
12681
12682 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12683
12684 Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12685 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12686
12687 /* Handle the pagefault trap for the nested shadow table. */
12688 PVM pVM = pVCpu->CTX_SUFF(pVM);
12689 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12690 TRPMResetTrap(pVCpu);
12691
12692 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12693 if ( rcStrict2 == VINF_SUCCESS
12694 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12695 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12696 {
12697 /* Successfully synced our nested page tables. */
12698 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12699 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12700 | HM_CHANGED_GUEST_RSP
12701 | HM_CHANGED_GUEST_RFLAGS);
12702 return VINF_SUCCESS;
12703 }
12704
12705 Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12706 return rcStrict2;
12707}
12708
12709/** @} */
12710
12711/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12712/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
12713/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12714
12715/** @name VM-exit exception handlers.
12716 * @{
12717 */
12718
12719/**
12720 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12721 */
12722static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12723{
12724 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12725 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12726
12727 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
12728 AssertRCReturn(rc, rc);
12729
12730 if (!(pMixedCtx->cr0 & X86_CR0_NE))
12731 {
12732 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12733 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12734
12735 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12736 * provides VM-exit instruction length. If this causes problem later,
12737 * disassemble the instruction like it's done on AMD-V. */
12738 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12739 AssertRCReturn(rc2, rc2);
12740 return rc;
12741 }
12742
12743 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12744 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12745 return rc;
12746}
12747
12748
12749/**
12750 * VM-exit exception handler for \#BP (Breakpoint exception).
12751 */
12752static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12753{
12754 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12755 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12756
12757 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12758 AssertRCReturn(rc, rc);
12759
12760 rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx));
12761 if (rc == VINF_EM_RAW_GUEST_TRAP)
12762 {
12763 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12764 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12765 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12766 AssertRCReturn(rc, rc);
12767
12768 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12769 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12770 }
12771
12772 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
12773 return rc;
12774}
12775
12776
12777/**
12778 * VM-exit exception handler for \#AC (alignment check exception).
12779 */
12780static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12781{
12782 RT_NOREF_PV(pMixedCtx);
12783 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12784
12785 /*
12786 * Re-inject it. We'll detect any nesting before getting here.
12787 */
12788 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12789 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12790 AssertRCReturn(rc, rc);
12791 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
12792
12793 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12794 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12795 return VINF_SUCCESS;
12796}
12797
12798
12799/**
12800 * VM-exit exception handler for \#DB (Debug exception).
12801 */
12802static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12803{
12804 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12805 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
12806
12807 /*
12808 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
12809 * for processing.
12810 */
12811 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12812
12813 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
12814 uint64_t uDR6 = X86_DR6_INIT_VAL;
12815 uDR6 |= ( pVmxTransient->uExitQualification
12816 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
12817
12818 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
12819 Log6Func(("rc=%Rrc\n", rc));
12820 if (rc == VINF_EM_RAW_GUEST_TRAP)
12821 {
12822 /*
12823 * The exception was for the guest. Update DR6, DR7.GD and
12824 * IA32_DEBUGCTL.LBR before forwarding it.
12825 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
12826 */
12827 VMMRZCallRing3Disable(pVCpu);
12828 HM_DISABLE_PREEMPT();
12829
12830 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
12831 pMixedCtx->dr[6] |= uDR6;
12832 if (CPUMIsGuestDebugStateActive(pVCpu))
12833 ASMSetDR6(pMixedCtx->dr[6]);
12834
12835 HM_RESTORE_PREEMPT();
12836 VMMRZCallRing3Enable(pVCpu);
12837
12838 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12839 AssertRCReturn(rc, rc);
12840
12841 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
12842 pMixedCtx->dr[7] &= ~X86_DR7_GD;
12843
12844 /* Paranoia. */
12845 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
12846 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
12847
12848 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
12849 AssertRCReturn(rc, rc);
12850
12851 /*
12852 * Raise #DB in the guest.
12853 *
12854 * It is important to reflect exactly what the VM-exit gave us (preserving the
12855 * interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
12856 * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
12857 * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
12858 *
12859 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
12860 * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
12861 */
12862 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12863 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12864 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12865 AssertRCReturn(rc, rc);
12866 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12867 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12868 return VINF_SUCCESS;
12869 }
12870
12871 /*
12872 * Not a guest trap, must be a hypervisor related debug event then.
12873 * Update DR6 in case someone is interested in it.
12874 */
12875 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
12876 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
12877 CPUMSetHyperDR6(pVCpu, uDR6);
12878
12879 return rc;
12880}
12881
12882/**
12883 * VM-exit exception handler for \#GP (General-protection exception).
12884 *
12885 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
12886 */
12887static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12888{
12889 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12890 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
12891
12892 int rc;
12893 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
12894 { /* likely */ }
12895 else
12896 {
12897#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12898 Assert(pVCpu->hm.s.fUsingDebugLoop);
12899#endif
12900 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
12901 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12902 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12903 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12904 rc |= hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12905 AssertRCReturn(rc, rc);
12906 Log4Func(("Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
12907 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
12908 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12909 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12910 return rc;
12911 }
12912
12913 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
12914 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
12915
12916 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
12917 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12918 AssertRCReturn(rc, rc);
12919
12920 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12921 uint32_t cbOp = 0;
12922 PVM pVM = pVCpu->CTX_SUFF(pVM);
12923 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12924 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
12925 if (RT_SUCCESS(rc))
12926 {
12927 rc = VINF_SUCCESS;
12928 Assert(cbOp == pDis->cbInstr);
12929 Log4Func(("Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12930 switch (pDis->pCurInstr->uOpcode)
12931 {
12932 case OP_CLI:
12933 {
12934 pMixedCtx->eflags.Bits.u1IF = 0;
12935 pMixedCtx->eflags.Bits.u1RF = 0;
12936 pMixedCtx->rip += pDis->cbInstr;
12937 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12938 if ( !fDbgStepping
12939 && pMixedCtx->eflags.Bits.u1TF)
12940 {
12941 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
12942 AssertRCReturn(rc, rc);
12943 }
12944 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
12945 break;
12946 }
12947
12948 case OP_STI:
12949 {
12950 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
12951 pMixedCtx->eflags.Bits.u1IF = 1;
12952 pMixedCtx->eflags.Bits.u1RF = 0;
12953 pMixedCtx->rip += pDis->cbInstr;
12954 if (!fOldIF)
12955 {
12956 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
12957 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
12958 }
12959 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12960 if ( !fDbgStepping
12961 && pMixedCtx->eflags.Bits.u1TF)
12962 {
12963 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
12964 AssertRCReturn(rc, rc);
12965 }
12966 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
12967 break;
12968 }
12969
12970 case OP_HLT:
12971 {
12972 rc = VINF_EM_HALT;
12973 pMixedCtx->rip += pDis->cbInstr;
12974 pMixedCtx->eflags.Bits.u1RF = 0;
12975 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12976 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
12977 break;
12978 }
12979
12980 case OP_POPF:
12981 {
12982 Log4Func(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12983 uint32_t cbParm;
12984 uint32_t uMask;
12985 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12986 if (pDis->fPrefix & DISPREFIX_OPSIZE)
12987 {
12988 cbParm = 4;
12989 uMask = 0xffffffff;
12990 }
12991 else
12992 {
12993 cbParm = 2;
12994 uMask = 0xffff;
12995 }
12996
12997 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
12998 RTGCPTR GCPtrStack = 0;
12999 X86EFLAGS Eflags;
13000 Eflags.u32 = 0;
13001 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13002 &GCPtrStack);
13003 if (RT_SUCCESS(rc))
13004 {
13005 Assert(sizeof(Eflags.u32) >= cbParm);
13006 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13007 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13008 }
13009 if (RT_FAILURE(rc))
13010 {
13011 rc = VERR_EM_INTERPRETER;
13012 break;
13013 }
13014 Log4Func(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13015 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13016 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13017 pMixedCtx->esp += cbParm;
13018 pMixedCtx->esp &= uMask;
13019 pMixedCtx->rip += pDis->cbInstr;
13020 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13021 | HM_CHANGED_GUEST_RSP
13022 | HM_CHANGED_GUEST_RFLAGS);
13023 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13024 POPF restores EFLAGS.TF. */
13025 if ( !fDbgStepping
13026 && fGstStepping)
13027 {
13028 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13029 AssertRCReturn(rc, rc);
13030 }
13031 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13032 break;
13033 }
13034
13035 case OP_PUSHF:
13036 {
13037 uint32_t cbParm;
13038 uint32_t uMask;
13039 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13040 {
13041 cbParm = 4;
13042 uMask = 0xffffffff;
13043 }
13044 else
13045 {
13046 cbParm = 2;
13047 uMask = 0xffff;
13048 }
13049
13050 /* Get the stack pointer & push the contents of eflags onto the stack. */
13051 RTGCPTR GCPtrStack = 0;
13052 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13053 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13054 if (RT_FAILURE(rc))
13055 {
13056 rc = VERR_EM_INTERPRETER;
13057 break;
13058 }
13059 X86EFLAGS Eflags = pMixedCtx->eflags;
13060 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13061 Eflags.Bits.u1RF = 0;
13062 Eflags.Bits.u1VM = 0;
13063
13064 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13065 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13066 {
13067 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13068 rc = VERR_EM_INTERPRETER;
13069 break;
13070 }
13071 Log4Func(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13072 pMixedCtx->esp -= cbParm;
13073 pMixedCtx->esp &= uMask;
13074 pMixedCtx->rip += pDis->cbInstr;
13075 pMixedCtx->eflags.Bits.u1RF = 0;
13076 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13077 | HM_CHANGED_GUEST_RSP
13078 | HM_CHANGED_GUEST_RFLAGS);
13079 if ( !fDbgStepping
13080 && pMixedCtx->eflags.Bits.u1TF)
13081 {
13082 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13083 AssertRCReturn(rc, rc);
13084 }
13085 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13086 break;
13087 }
13088
13089 case OP_IRET:
13090 {
13091 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13092 * instruction reference. */
13093 RTGCPTR GCPtrStack = 0;
13094 uint32_t uMask = 0xffff;
13095 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13096 uint16_t aIretFrame[3];
13097 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13098 {
13099 rc = VERR_EM_INTERPRETER;
13100 break;
13101 }
13102 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13103 &GCPtrStack);
13104 if (RT_SUCCESS(rc))
13105 {
13106 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13107 PGMACCESSORIGIN_HM));
13108 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13109 }
13110 if (RT_FAILURE(rc))
13111 {
13112 rc = VERR_EM_INTERPRETER;
13113 break;
13114 }
13115 pMixedCtx->eip = 0;
13116 pMixedCtx->ip = aIretFrame[0];
13117 pMixedCtx->cs.Sel = aIretFrame[1];
13118 pMixedCtx->cs.ValidSel = aIretFrame[1];
13119 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13120 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13121 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13122 pMixedCtx->sp += sizeof(aIretFrame);
13123 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13124 | HM_CHANGED_GUEST_CS
13125 | HM_CHANGED_GUEST_RSP
13126 | HM_CHANGED_GUEST_RFLAGS);
13127 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13128 if ( !fDbgStepping
13129 && fGstStepping)
13130 {
13131 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13132 AssertRCReturn(rc, rc);
13133 }
13134 Log4Func(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13135 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13136 break;
13137 }
13138
13139 case OP_INT:
13140 {
13141 uint16_t uVector = pDis->Param1.uValue & 0xff;
13142 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13143 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13144 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13145 break;
13146 }
13147
13148 case OP_INTO:
13149 {
13150 if (pMixedCtx->eflags.Bits.u1OF)
13151 {
13152 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13153 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13154 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13155 }
13156 else
13157 {
13158 pMixedCtx->eflags.Bits.u1RF = 0;
13159 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
13160 }
13161 break;
13162 }
13163
13164 default:
13165 {
13166 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13167 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13168 EMCODETYPE_SUPERVISOR);
13169 rc = VBOXSTRICTRC_VAL(rc2);
13170 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13171 /** @todo We have to set pending-debug exceptions here when the guest is
13172 * single-stepping depending on the instruction that was interpreted. */
13173 Log4Func(("#GP rc=%Rrc\n", rc));
13174 break;
13175 }
13176 }
13177 }
13178 else
13179 rc = VERR_EM_INTERPRETER;
13180
13181 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13182 ("#GP Unexpected rc=%Rrc\n", rc));
13183 return rc;
13184}
13185
13186
13187/**
13188 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13189 * the exception reported in the VMX transient structure back into the VM.
13190 *
13191 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13192 * up-to-date.
13193 */
13194static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13195{
13196 RT_NOREF_PV(pMixedCtx);
13197 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13198#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13199 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13200 ("uVector=%#x u32XcptBitmap=%#X32\n",
13201 VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13202#endif
13203
13204 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13205 hmR0VmxCheckExitDueToEventDelivery(). */
13206 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13207 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13208 AssertRCReturn(rc, rc);
13209 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13210
13211#ifdef DEBUG_ramshankar
13212 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS
13213 | CPUMCTX_EXTRN_RIP);
13214 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13215 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13216#endif
13217
13218 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13219 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13220 return VINF_SUCCESS;
13221}
13222
13223
13224/**
13225 * VM-exit exception handler for \#PF (Page-fault exception).
13226 */
13227static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13228{
13229 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13230 PVM pVM = pVCpu->CTX_SUFF(pVM);
13231 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13232 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13233 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13234 AssertRCReturn(rc, rc);
13235
13236 if (!pVM->hm.s.fNestedPaging)
13237 { /* likely */ }
13238 else
13239 {
13240#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13241 Assert(pVCpu->hm.s.fUsingDebugLoop);
13242#endif
13243 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13244 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13245 {
13246 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13247 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13248 }
13249 else
13250 {
13251 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13252 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13253 Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13254 }
13255 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13256 return rc;
13257 }
13258
13259 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13260 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13261 if (pVmxTransient->fVectoringPF)
13262 {
13263 Assert(pVCpu->hm.s.Event.fPending);
13264 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13265 }
13266
13267 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13268 AssertRCReturn(rc, rc);
13269
13270 Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13271 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13272
13273 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13274 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13275 (RTGCPTR)pVmxTransient->uExitQualification);
13276
13277 Log4Func(("#PF: rc=%Rrc\n", rc));
13278 if (rc == VINF_SUCCESS)
13279 {
13280 /*
13281 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13282 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13283 */
13284 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13285 TRPMResetTrap(pVCpu);
13286 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13287 return rc;
13288 }
13289
13290 if (rc == VINF_EM_RAW_GUEST_TRAP)
13291 {
13292 if (!pVmxTransient->fVectoringDoublePF)
13293 {
13294 /* It's a guest page fault and needs to be reflected to the guest. */
13295 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13296 TRPMResetTrap(pVCpu);
13297 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13298 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13299 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13300 }
13301 else
13302 {
13303 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13304 TRPMResetTrap(pVCpu);
13305 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13306 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13307 Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13308 }
13309
13310 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13311 return VINF_SUCCESS;
13312 }
13313
13314 TRPMResetTrap(pVCpu);
13315 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13316 return rc;
13317}
13318
13319/** @} */
13320
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette