VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 71963

Last change on this file since 71963 was 71912, checked in by vboxsync, 7 years ago

space.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 600.5 KB
Line 
1/* $Id: HMVMXR0.cpp 71912 2018-04-19 08:57:19Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#include <iprt/x86.h>
24#include <iprt/asm-amd64-x86.h>
25#include <iprt/thread.h>
26
27#include <VBox/vmm/pdmapi.h>
28#include <VBox/vmm/dbgf.h>
29#include <VBox/vmm/iem.h>
30#include <VBox/vmm/iom.h>
31#include <VBox/vmm/selm.h>
32#include <VBox/vmm/tm.h>
33#include <VBox/vmm/gim.h>
34#include <VBox/vmm/apic.h>
35#ifdef VBOX_WITH_REM
36# include <VBox/vmm/rem.h>
37#endif
38#include "HMInternal.h"
39#include <VBox/vmm/vm.h>
40#include "HMVMXR0.h"
41#include "dtrace/VBoxVMM.h"
42
43#define HMVMX_USE_IEM_EVENT_REFLECTION
44#ifdef DEBUG_ramshankar
45# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
47# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_CHECK_GUEST_STATE
49# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
50# define HMVMX_ALWAYS_TRAP_PF
51# define HMVMX_ALWAYS_SWAP_FPU_STATE
52# define HMVMX_ALWAYS_FLUSH_TLB
53# define HMVMX_ALWAYS_SWAP_EFER
54#endif
55
56
57/*********************************************************************************************************************************
58* Defined Constants And Macros *
59*********************************************************************************************************************************/
60/** Use the function table. */
61#define HMVMX_USE_FUNCTION_TABLE
62
63/** Determine which tagged-TLB flush handler to use. */
64#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
65#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
66#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
67#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
68
69/** @name Updated-guest-state flags.
70 * @{ */
71#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
72#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
73#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
74#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
75#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
76#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
77#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
78#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
79#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
80#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
81#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
82#define HMVMX_UPDATED_GUEST_DR7 RT_BIT(11)
83#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
84#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
85#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
86#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
87#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
88#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
89#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
90#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
91#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
92 | HMVMX_UPDATED_GUEST_RSP \
93 | HMVMX_UPDATED_GUEST_RFLAGS \
94 | HMVMX_UPDATED_GUEST_CR0 \
95 | HMVMX_UPDATED_GUEST_CR3 \
96 | HMVMX_UPDATED_GUEST_CR4 \
97 | HMVMX_UPDATED_GUEST_GDTR \
98 | HMVMX_UPDATED_GUEST_IDTR \
99 | HMVMX_UPDATED_GUEST_LDTR \
100 | HMVMX_UPDATED_GUEST_TR \
101 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
102 | HMVMX_UPDATED_GUEST_DR7 \
103 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
104 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
105 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
106 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
107 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
108 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
109 | HMVMX_UPDATED_GUEST_INTR_STATE \
110 | HMVMX_UPDATED_GUEST_APIC_STATE)
111/** @} */
112
113/** @name
114 * Flags to skip redundant reads of some common VMCS fields that are not part of
115 * the guest-CPU state but are in the transient structure.
116 */
117#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
118#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
119#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
120#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
123#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
124/** @} */
125
126/** @name
127 * States of the VMCS.
128 *
129 * This does not reflect all possible VMCS states but currently only those
130 * needed for maintaining the VMCS consistently even when thread-context hooks
131 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
132 */
133#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
134#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
135#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
136/** @} */
137
138/**
139 * Exception bitmap mask for real-mode guests (real-on-v86).
140 *
141 * We need to intercept all exceptions manually except:
142 * - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
143 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
144 * due to bugs in Intel CPUs.
145 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
146 * support.
147 */
148#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
149 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
150 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
151 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
152 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
153 /* RT_BIT(X86_XCPT_MF) always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
154 | RT_BIT(X86_XCPT_XF))
155
156/**
157 * Exception bitmap mask for all contributory exceptions.
158 *
159 * Page fault is deliberately excluded here as it's conditional as to whether
160 * it's contributory or benign. Page faults are handled separately.
161 */
162#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
163 | RT_BIT(X86_XCPT_DE))
164
165/** Maximum VM-instruction error number. */
166#define HMVMX_INSTR_ERROR_MAX 28
167
168/** Profiling macro. */
169#ifdef HM_PROFILE_EXIT_DISPATCH
170# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
171# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
172#else
173# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
174# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
175#endif
176
177/** Assert that preemption is disabled or covered by thread-context hooks. */
178#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
179 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
180
181/** Assert that we haven't migrated CPUs when thread-context hooks are not
182 * used. */
183#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
184 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
185 ("Illegal migration! Entered on CPU %u Current %u\n", \
186 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
187
188/** Helper macro for VM-exit handlers called unexpectedly. */
189#define HMVMX_RETURN_UNEXPECTED_EXIT() \
190 do { \
191 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
192 return VERR_VMX_UNEXPECTED_EXIT; \
193 } while (0)
194
195
196/*********************************************************************************************************************************
197* Structures and Typedefs *
198*********************************************************************************************************************************/
199/**
200 * VMX transient state.
201 *
202 * A state structure for holding miscellaneous information across
203 * VMX non-root operation and restored after the transition.
204 */
205typedef struct VMXTRANSIENT
206{
207 /** The host's rflags/eflags. */
208 RTCCUINTREG fEFlags;
209#if HC_ARCH_BITS == 32
210 uint32_t u32Alignment0;
211#endif
212 /** The guest's TPR value used for TPR shadowing. */
213 uint8_t u8GuestTpr;
214 /** Alignment. */
215 uint8_t abAlignment0[7];
216
217 /** The basic VM-exit reason. */
218 uint16_t uExitReason;
219 /** Alignment. */
220 uint16_t u16Alignment0;
221 /** The VM-exit interruption error code. */
222 uint32_t uExitIntErrorCode;
223 /** The VM-exit exit code qualification. */
224 uint64_t uExitQualification;
225
226 /** The VM-exit interruption-information field. */
227 uint32_t uExitIntInfo;
228 /** The VM-exit instruction-length field. */
229 uint32_t cbInstr;
230 /** The VM-exit instruction-information field. */
231 union
232 {
233 /** Plain unsigned int representation. */
234 uint32_t u;
235 /** INS and OUTS information. */
236 struct
237 {
238 uint32_t u7Reserved0 : 7;
239 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
240 uint32_t u3AddrSize : 3;
241 uint32_t u5Reserved1 : 5;
242 /** The segment register (X86_SREG_XXX). */
243 uint32_t iSegReg : 3;
244 uint32_t uReserved2 : 14;
245 } StrIo;
246 /** INVEPT, INVVPID, INVPCID information. */
247 struct
248 {
249 /** Scaling; 0=no scaling, 1=scale-by-2, 2=scale-by-4, 3=scale-by-8. */
250 uint32_t u2Scaling : 2;
251 uint32_t u5Reserved0 : 5;
252 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
253 uint32_t u3AddrSize : 3;
254 uint32_t u1Reserved0 : 1;
255 uint32_t u4Reserved0 : 4;
256 /** The segment register (X86_SREG_XXX). */
257 uint32_t iSegReg : 3;
258 /** The index register (X86_GREG_XXX). */
259 uint32_t iIdxReg : 4;
260 /** Set if index register is invalid. */
261 uint32_t fIdxRegValid : 1;
262 /** The base register (X86_GREG_XXX). */
263 uint32_t iBaseReg : 4;
264 /** Set if base register is invalid. */
265 uint32_t fBaseRegValid : 1;
266 /** Register 2 (X86_GREG_XXX). */
267 uint32_t iReg2 : 4;
268 } Inv;
269 } ExitInstrInfo;
270 /** Whether the VM-entry failed or not. */
271 bool fVMEntryFailed;
272 /** Alignment. */
273 uint8_t abAlignment1[3];
274
275 /** The VM-entry interruption-information field. */
276 uint32_t uEntryIntInfo;
277 /** The VM-entry exception error code field. */
278 uint32_t uEntryXcptErrorCode;
279 /** The VM-entry instruction length field. */
280 uint32_t cbEntryInstr;
281
282 /** IDT-vectoring information field. */
283 uint32_t uIdtVectoringInfo;
284 /** IDT-vectoring error code. */
285 uint32_t uIdtVectoringErrorCode;
286
287 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
288 uint32_t fVmcsFieldsRead;
289
290 /** Whether the guest FPU was active at the time of VM-exit. */
291 bool fWasGuestFPUStateActive;
292 /** Whether the guest debug state was active at the time of VM-exit. */
293 bool fWasGuestDebugStateActive;
294 /** Whether the hyper debug state was active at the time of VM-exit. */
295 bool fWasHyperDebugStateActive;
296 /** Whether TSC-offsetting should be setup before VM-entry. */
297 bool fUpdateTscOffsettingAndPreemptTimer;
298 /** Whether the VM-exit was caused by a page-fault during delivery of a
299 * contributory exception or a page-fault. */
300 bool fVectoringDoublePF;
301 /** Whether the VM-exit was caused by a page-fault during delivery of an
302 * external interrupt or NMI. */
303 bool fVectoringPF;
304} VMXTRANSIENT;
305AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
306AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
307AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
308AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
309AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
310/** Pointer to VMX transient state. */
311typedef VMXTRANSIENT *PVMXTRANSIENT;
312
313
314/**
315 * MSR-bitmap read permissions.
316 */
317typedef enum VMXMSREXITREAD
318{
319 /** Reading this MSR causes a VM-exit. */
320 VMXMSREXIT_INTERCEPT_READ = 0xb,
321 /** Reading this MSR does not cause a VM-exit. */
322 VMXMSREXIT_PASSTHRU_READ
323} VMXMSREXITREAD;
324/** Pointer to MSR-bitmap read permissions. */
325typedef VMXMSREXITREAD* PVMXMSREXITREAD;
326
327/**
328 * MSR-bitmap write permissions.
329 */
330typedef enum VMXMSREXITWRITE
331{
332 /** Writing to this MSR causes a VM-exit. */
333 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
334 /** Writing to this MSR does not cause a VM-exit. */
335 VMXMSREXIT_PASSTHRU_WRITE
336} VMXMSREXITWRITE;
337/** Pointer to MSR-bitmap write permissions. */
338typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
339
340
341/**
342 * VMX VM-exit handler.
343 *
344 * @returns Strict VBox status code (i.e. informational status codes too).
345 * @param pVCpu The cross context virtual CPU structure.
346 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
347 * out-of-sync. Make sure to update the required
348 * fields before using them.
349 * @param pVmxTransient Pointer to the VMX-transient structure.
350 */
351#ifndef HMVMX_USE_FUNCTION_TABLE
352typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
353#else
354typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
355/** Pointer to VM-exit handler. */
356typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
357#endif
358
359/**
360 * VMX VM-exit handler, non-strict status code.
361 *
362 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
363 *
364 * @returns VBox status code, no informational status code returned.
365 * @param pVCpu The cross context virtual CPU structure.
366 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
367 * out-of-sync. Make sure to update the required
368 * fields before using them.
369 * @param pVmxTransient Pointer to the VMX-transient structure.
370 *
371 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
372 * use of that status code will be replaced with VINF_EM_SOMETHING
373 * later when switching over to IEM.
374 */
375#ifndef HMVMX_USE_FUNCTION_TABLE
376typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
377#else
378typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
379#endif
380
381
382/*********************************************************************************************************************************
383* Internal Functions *
384*********************************************************************************************************************************/
385static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
386static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
387static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
388static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
389 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
390 bool fStepping, uint32_t *puIntState);
391#if HC_ARCH_BITS == 32
392static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
393#endif
394#ifndef HMVMX_USE_FUNCTION_TABLE
395DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
396# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
397# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
398#else
399# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
400# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
401#endif
402
403
404/** @name VM-exit handlers.
405 * @{
406 */
407static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
408static FNVMXEXITHANDLER hmR0VmxExitExtInt;
409static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
410static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
411static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
412static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
413static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
414static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
415static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
416static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
417static FNVMXEXITHANDLER hmR0VmxExitCpuid;
418static FNVMXEXITHANDLER hmR0VmxExitGetsec;
419static FNVMXEXITHANDLER hmR0VmxExitHlt;
420static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
421static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
422static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
423static FNVMXEXITHANDLER hmR0VmxExitVmcall;
424static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
425static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
426static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
427static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
428static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
429static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
430static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
431static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
432static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
433static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
434static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
435static FNVMXEXITHANDLER hmR0VmxExitMwait;
436static FNVMXEXITHANDLER hmR0VmxExitMtf;
437static FNVMXEXITHANDLER hmR0VmxExitMonitor;
438static FNVMXEXITHANDLER hmR0VmxExitPause;
439static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
440static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
441static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
442static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
443static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
444static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
445static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
446static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
447static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
448static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
449static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
450static FNVMXEXITHANDLER hmR0VmxExitRdrand;
451static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
452/** @} */
453
454static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
455static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
456static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
457static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
458static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
459static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
460static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
461static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
462static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
463
464
465/*********************************************************************************************************************************
466* Global Variables *
467*********************************************************************************************************************************/
468#ifdef HMVMX_USE_FUNCTION_TABLE
469
470/**
471 * VMX_EXIT dispatch table.
472 */
473static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
474{
475 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
476 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
477 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
478 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
479 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
480 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
481 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
482 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
483 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
484 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
485 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
486 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
487 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
488 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
489 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
490 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
491 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
492 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
493 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
494 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
495 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
496 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
497 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
498 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
499 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
500 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
501 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
502 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
503 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
504 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
505 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
506 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
507 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
508 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
509 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
510 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
511 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
512 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
513 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
514 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
515 /* 40 UNDEFINED */ hmR0VmxExitPause,
516 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
517 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
518 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
519 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
520 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
521 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
522 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
523 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
524 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
525 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
526 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
527 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
528 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
529 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
530 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
531 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
532 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
533 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
534 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
535 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
536 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
537 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
538 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
539 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
540};
541#endif /* HMVMX_USE_FUNCTION_TABLE */
542
543#ifdef VBOX_STRICT
544static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
545{
546 /* 0 */ "(Not Used)",
547 /* 1 */ "VMCALL executed in VMX root operation.",
548 /* 2 */ "VMCLEAR with invalid physical address.",
549 /* 3 */ "VMCLEAR with VMXON pointer.",
550 /* 4 */ "VMLAUNCH with non-clear VMCS.",
551 /* 5 */ "VMRESUME with non-launched VMCS.",
552 /* 6 */ "VMRESUME after VMXOFF",
553 /* 7 */ "VM-entry with invalid control fields.",
554 /* 8 */ "VM-entry with invalid host state fields.",
555 /* 9 */ "VMPTRLD with invalid physical address.",
556 /* 10 */ "VMPTRLD with VMXON pointer.",
557 /* 11 */ "VMPTRLD with incorrect revision identifier.",
558 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
559 /* 13 */ "VMWRITE to read-only VMCS component.",
560 /* 14 */ "(Not Used)",
561 /* 15 */ "VMXON executed in VMX root operation.",
562 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
563 /* 17 */ "VM-entry with non-launched executing VMCS.",
564 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
565 /* 19 */ "VMCALL with non-clear VMCS.",
566 /* 20 */ "VMCALL with invalid VM-exit control fields.",
567 /* 21 */ "(Not Used)",
568 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
569 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
570 /* 24 */ "VMCALL with invalid SMM-monitor features.",
571 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
572 /* 26 */ "VM-entry with events blocked by MOV SS.",
573 /* 27 */ "(Not Used)",
574 /* 28 */ "Invalid operand to INVEPT/INVVPID."
575};
576#endif /* VBOX_STRICT */
577
578
579
580/**
581 * Updates the VM's last error record.
582 *
583 * If there was a VMX instruction error, reads the error data from the VMCS and
584 * updates VCPU's last error record as well.
585 *
586 * @param pVM The cross context VM structure.
587 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
588 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
589 * VERR_VMX_INVALID_VMCS_FIELD.
590 * @param rc The error code.
591 */
592static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
593{
594 AssertPtr(pVM);
595 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
596 || rc == VERR_VMX_UNABLE_TO_START_VM)
597 {
598 AssertPtrReturnVoid(pVCpu);
599 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
600 }
601 pVM->hm.s.lLastError = rc;
602}
603
604
605/**
606 * Reads the VM-entry interruption-information field from the VMCS into the VMX
607 * transient structure.
608 *
609 * @returns VBox status code.
610 * @param pVmxTransient Pointer to the VMX transient structure.
611 *
612 * @remarks No-long-jump zone!!!
613 */
614DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
615{
616 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
617 AssertRCReturn(rc, rc);
618 return VINF_SUCCESS;
619}
620
621
622#ifdef VBOX_STRICT
623/**
624 * Reads the VM-entry exception error code field from the VMCS into
625 * the VMX transient structure.
626 *
627 * @returns VBox status code.
628 * @param pVmxTransient Pointer to the VMX transient structure.
629 *
630 * @remarks No-long-jump zone!!!
631 */
632DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
633{
634 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
635 AssertRCReturn(rc, rc);
636 return VINF_SUCCESS;
637}
638#endif /* VBOX_STRICT */
639
640
641#ifdef VBOX_STRICT
642/**
643 * Reads the VM-entry exception error code field from the VMCS into
644 * the VMX transient structure.
645 *
646 * @returns VBox status code.
647 * @param pVmxTransient Pointer to the VMX transient structure.
648 *
649 * @remarks No-long-jump zone!!!
650 */
651DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
652{
653 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
654 AssertRCReturn(rc, rc);
655 return VINF_SUCCESS;
656}
657#endif /* VBOX_STRICT */
658
659
660/**
661 * Reads the VM-exit interruption-information field from the VMCS into the VMX
662 * transient structure.
663 *
664 * @returns VBox status code.
665 * @param pVmxTransient Pointer to the VMX transient structure.
666 */
667DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
668{
669 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
670 {
671 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
672 AssertRCReturn(rc, rc);
673 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
674 }
675 return VINF_SUCCESS;
676}
677
678
679/**
680 * Reads the VM-exit interruption error code from the VMCS into the VMX
681 * transient structure.
682 *
683 * @returns VBox status code.
684 * @param pVmxTransient Pointer to the VMX transient structure.
685 */
686DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
687{
688 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
689 {
690 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
691 AssertRCReturn(rc, rc);
692 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
693 }
694 return VINF_SUCCESS;
695}
696
697
698/**
699 * Reads the VM-exit instruction length field from the VMCS into the VMX
700 * transient structure.
701 *
702 * @returns VBox status code.
703 * @param pVmxTransient Pointer to the VMX transient structure.
704 */
705DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
706{
707 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
708 {
709 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
710 AssertRCReturn(rc, rc);
711 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
712 }
713 return VINF_SUCCESS;
714}
715
716
717/**
718 * Reads the VM-exit instruction-information field from the VMCS into
719 * the VMX transient structure.
720 *
721 * @returns VBox status code.
722 * @param pVmxTransient Pointer to the VMX transient structure.
723 */
724DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
725{
726 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
727 {
728 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
729 AssertRCReturn(rc, rc);
730 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
731 }
732 return VINF_SUCCESS;
733}
734
735
736/**
737 * Reads the exit code qualification from the VMCS into the VMX transient
738 * structure.
739 *
740 * @returns VBox status code.
741 * @param pVCpu The cross context virtual CPU structure of the
742 * calling EMT. (Required for the VMCS cache case.)
743 * @param pVmxTransient Pointer to the VMX transient structure.
744 */
745DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
746{
747 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
748 {
749 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
750 AssertRCReturn(rc, rc);
751 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
752 }
753 return VINF_SUCCESS;
754}
755
756
757/**
758 * Reads the IDT-vectoring information field from the VMCS into the VMX
759 * transient structure.
760 *
761 * @returns VBox status code.
762 * @param pVmxTransient Pointer to the VMX transient structure.
763 *
764 * @remarks No-long-jump zone!!!
765 */
766DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
767{
768 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
769 {
770 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
771 AssertRCReturn(rc, rc);
772 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
773 }
774 return VINF_SUCCESS;
775}
776
777
778/**
779 * Reads the IDT-vectoring error code from the VMCS into the VMX
780 * transient structure.
781 *
782 * @returns VBox status code.
783 * @param pVmxTransient Pointer to the VMX transient structure.
784 */
785DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
786{
787 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
788 {
789 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
790 AssertRCReturn(rc, rc);
791 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
792 }
793 return VINF_SUCCESS;
794}
795
796
797/**
798 * Enters VMX root mode operation on the current CPU.
799 *
800 * @returns VBox status code.
801 * @param pVM The cross context VM structure. Can be
802 * NULL, after a resume.
803 * @param HCPhysCpuPage Physical address of the VMXON region.
804 * @param pvCpuPage Pointer to the VMXON region.
805 */
806static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
807{
808 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
809 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
810 Assert(pvCpuPage);
811 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
812
813 if (pVM)
814 {
815 /* Write the VMCS revision dword to the VMXON region. */
816 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
817 }
818
819 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
820 RTCCUINTREG fEFlags = ASMIntDisableFlags();
821
822 /* Enable the VMX bit in CR4 if necessary. */
823 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
824
825 /* Enter VMX root mode. */
826 int rc = VMXEnable(HCPhysCpuPage);
827 if (RT_FAILURE(rc))
828 {
829 if (!(uOldCr4 & X86_CR4_VMXE))
830 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
831
832 if (pVM)
833 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
834 }
835
836 /* Restore interrupts. */
837 ASMSetFlags(fEFlags);
838 return rc;
839}
840
841
842/**
843 * Exits VMX root mode operation on the current CPU.
844 *
845 * @returns VBox status code.
846 */
847static int hmR0VmxLeaveRootMode(void)
848{
849 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
850
851 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
852 RTCCUINTREG fEFlags = ASMIntDisableFlags();
853
854 /* If we're for some reason not in VMX root mode, then don't leave it. */
855 RTCCUINTREG uHostCR4 = ASMGetCR4();
856
857 int rc;
858 if (uHostCR4 & X86_CR4_VMXE)
859 {
860 /* Exit VMX root mode and clear the VMX bit in CR4. */
861 VMXDisable();
862 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
863 rc = VINF_SUCCESS;
864 }
865 else
866 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
867
868 /* Restore interrupts. */
869 ASMSetFlags(fEFlags);
870 return rc;
871}
872
873
874/**
875 * Allocates and maps one physically contiguous page. The allocated page is
876 * zero'd out. (Used by various VT-x structures).
877 *
878 * @returns IPRT status code.
879 * @param pMemObj Pointer to the ring-0 memory object.
880 * @param ppVirt Where to store the virtual address of the
881 * allocation.
882 * @param pHCPhys Where to store the physical address of the
883 * allocation.
884 */
885DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
886{
887 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
888 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
889 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
890
891 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
892 if (RT_FAILURE(rc))
893 return rc;
894 *ppVirt = RTR0MemObjAddress(*pMemObj);
895 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
896 ASMMemZero32(*ppVirt, PAGE_SIZE);
897 return VINF_SUCCESS;
898}
899
900
901/**
902 * Frees and unmaps an allocated physical page.
903 *
904 * @param pMemObj Pointer to the ring-0 memory object.
905 * @param ppVirt Where to re-initialize the virtual address of
906 * allocation as 0.
907 * @param pHCPhys Where to re-initialize the physical address of the
908 * allocation as 0.
909 */
910DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
911{
912 AssertPtr(pMemObj);
913 AssertPtr(ppVirt);
914 AssertPtr(pHCPhys);
915 if (*pMemObj != NIL_RTR0MEMOBJ)
916 {
917 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
918 AssertRC(rc);
919 *pMemObj = NIL_RTR0MEMOBJ;
920 *ppVirt = 0;
921 *pHCPhys = 0;
922 }
923}
924
925
926/**
927 * Worker function to free VT-x related structures.
928 *
929 * @returns IPRT status code.
930 * @param pVM The cross context VM structure.
931 */
932static void hmR0VmxStructsFree(PVM pVM)
933{
934 for (VMCPUID i = 0; i < pVM->cCpus; i++)
935 {
936 PVMCPU pVCpu = &pVM->aCpus[i];
937 AssertPtr(pVCpu);
938
939 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
940 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
941
942 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
943 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
944
945 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
946 }
947
948 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
949#ifdef VBOX_WITH_CRASHDUMP_MAGIC
950 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
951#endif
952}
953
954
955/**
956 * Worker function to allocate VT-x related VM structures.
957 *
958 * @returns IPRT status code.
959 * @param pVM The cross context VM structure.
960 */
961static int hmR0VmxStructsAlloc(PVM pVM)
962{
963 /*
964 * Initialize members up-front so we can cleanup properly on allocation failure.
965 */
966#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
967 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
968 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
969 pVM->hm.s.vmx.HCPhys##a_Name = 0;
970
971#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
972 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
973 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
974 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
975
976#ifdef VBOX_WITH_CRASHDUMP_MAGIC
977 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
978#endif
979 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
980
981 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
982 for (VMCPUID i = 0; i < pVM->cCpus; i++)
983 {
984 PVMCPU pVCpu = &pVM->aCpus[i];
985 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
986 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
987 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
988 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
989 }
990#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
991#undef VMXLOCAL_INIT_VM_MEMOBJ
992
993 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
994 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
995 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
996 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
997
998 /*
999 * Allocate all the VT-x structures.
1000 */
1001 int rc = VINF_SUCCESS;
1002#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1003 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1004 if (RT_FAILURE(rc))
1005 goto cleanup;
1006 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1007 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1008#endif
1009
1010 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1011 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1012 {
1013 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1014 &pVM->hm.s.vmx.HCPhysApicAccess);
1015 if (RT_FAILURE(rc))
1016 goto cleanup;
1017 }
1018
1019 /*
1020 * Initialize per-VCPU VT-x structures.
1021 */
1022 for (VMCPUID i = 0; i < pVM->cCpus; i++)
1023 {
1024 PVMCPU pVCpu = &pVM->aCpus[i];
1025 AssertPtr(pVCpu);
1026
1027 /* Allocate the VM control structure (VMCS). */
1028 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1029 if (RT_FAILURE(rc))
1030 goto cleanup;
1031
1032 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1033 if ( PDMHasApic(pVM)
1034 && (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW))
1035 {
1036 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1037 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1038 if (RT_FAILURE(rc))
1039 goto cleanup;
1040 }
1041
1042 /*
1043 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1044 * transparent accesses of specific MSRs.
1045 *
1046 * If the condition for enabling MSR bitmaps changes here, don't forget to
1047 * update HMAreMsrBitmapsAvailable().
1048 */
1049 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1050 {
1051 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1052 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1053 if (RT_FAILURE(rc))
1054 goto cleanup;
1055 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1056 }
1057
1058 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1059 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1060 if (RT_FAILURE(rc))
1061 goto cleanup;
1062
1063 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1064 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1065 if (RT_FAILURE(rc))
1066 goto cleanup;
1067 }
1068
1069 return VINF_SUCCESS;
1070
1071cleanup:
1072 hmR0VmxStructsFree(pVM);
1073 return rc;
1074}
1075
1076
1077/**
1078 * Does global VT-x initialization (called during module initialization).
1079 *
1080 * @returns VBox status code.
1081 */
1082VMMR0DECL(int) VMXR0GlobalInit(void)
1083{
1084#ifdef HMVMX_USE_FUNCTION_TABLE
1085 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1086# ifdef VBOX_STRICT
1087 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1088 Assert(g_apfnVMExitHandlers[i]);
1089# endif
1090#endif
1091 return VINF_SUCCESS;
1092}
1093
1094
1095/**
1096 * Does global VT-x termination (called during module termination).
1097 */
1098VMMR0DECL(void) VMXR0GlobalTerm()
1099{
1100 /* Nothing to do currently. */
1101}
1102
1103
1104/**
1105 * Sets up and activates VT-x on the current CPU.
1106 *
1107 * @returns VBox status code.
1108 * @param pCpu Pointer to the global CPU info struct.
1109 * @param pVM The cross context VM structure. Can be
1110 * NULL after a host resume operation.
1111 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1112 * fEnabledByHost is @c true).
1113 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1114 * @a fEnabledByHost is @c true).
1115 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1116 * enable VT-x on the host.
1117 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1118 */
1119VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1120 void *pvMsrs)
1121{
1122 Assert(pCpu);
1123 Assert(pvMsrs);
1124 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1125
1126 /* Enable VT-x if it's not already enabled by the host. */
1127 if (!fEnabledByHost)
1128 {
1129 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1130 if (RT_FAILURE(rc))
1131 return rc;
1132 }
1133
1134 /*
1135 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1136 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1137 */
1138 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1139 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1140 {
1141 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1142 pCpu->fFlushAsidBeforeUse = false;
1143 }
1144 else
1145 pCpu->fFlushAsidBeforeUse = true;
1146
1147 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1148 ++pCpu->cTlbFlushes;
1149
1150 return VINF_SUCCESS;
1151}
1152
1153
1154/**
1155 * Deactivates VT-x on the current CPU.
1156 *
1157 * @returns VBox status code.
1158 * @param pCpu Pointer to the global CPU info struct.
1159 * @param pvCpuPage Pointer to the VMXON region.
1160 * @param HCPhysCpuPage Physical address of the VMXON region.
1161 *
1162 * @remarks This function should never be called when SUPR0EnableVTx() or
1163 * similar was used to enable VT-x on the host.
1164 */
1165VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1166{
1167 NOREF(pCpu);
1168 NOREF(pvCpuPage);
1169 NOREF(HCPhysCpuPage);
1170
1171 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1172 return hmR0VmxLeaveRootMode();
1173}
1174
1175
1176/**
1177 * Sets the permission bits for the specified MSR in the MSR bitmap.
1178 *
1179 * @param pVCpu The cross context virtual CPU structure.
1180 * @param uMsr The MSR value.
1181 * @param enmRead Whether reading this MSR causes a VM-exit.
1182 * @param enmWrite Whether writing this MSR causes a VM-exit.
1183 */
1184static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1185{
1186 int32_t iBit;
1187 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1188
1189 /*
1190 * Layout:
1191 * 0x000 - 0x3ff - Low MSR read bits
1192 * 0x400 - 0x7ff - High MSR read bits
1193 * 0x800 - 0xbff - Low MSR write bits
1194 * 0xc00 - 0xfff - High MSR write bits
1195 */
1196 if (uMsr <= 0x00001FFF)
1197 iBit = uMsr;
1198 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1199 {
1200 iBit = uMsr - UINT32_C(0xC0000000);
1201 pbMsrBitmap += 0x400;
1202 }
1203 else
1204 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1205
1206 Assert(iBit <= 0x1fff);
1207 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1208 ASMBitSet(pbMsrBitmap, iBit);
1209 else
1210 ASMBitClear(pbMsrBitmap, iBit);
1211
1212 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1213 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1214 else
1215 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1216}
1217
1218
1219#ifdef VBOX_STRICT
1220/**
1221 * Gets the permission bits for the specified MSR in the MSR bitmap.
1222 *
1223 * @returns VBox status code.
1224 * @retval VINF_SUCCESS if the specified MSR is found.
1225 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1226 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1227 *
1228 * @param pVCpu The cross context virtual CPU structure.
1229 * @param uMsr The MSR.
1230 * @param penmRead Where to store the read permissions.
1231 * @param penmWrite Where to store the write permissions.
1232 */
1233static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1234{
1235 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1236 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1237 int32_t iBit;
1238 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1239
1240 /* See hmR0VmxSetMsrPermission() for the layout. */
1241 if (uMsr <= 0x00001FFF)
1242 iBit = uMsr;
1243 else if ( uMsr >= 0xC0000000
1244 && uMsr <= 0xC0001FFF)
1245 {
1246 iBit = (uMsr - 0xC0000000);
1247 pbMsrBitmap += 0x400;
1248 }
1249 else
1250 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1251
1252 Assert(iBit <= 0x1fff);
1253 if (ASMBitTest(pbMsrBitmap, iBit))
1254 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1255 else
1256 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1257
1258 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1259 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1260 else
1261 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1262 return VINF_SUCCESS;
1263}
1264#endif /* VBOX_STRICT */
1265
1266
1267/**
1268 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1269 * area.
1270 *
1271 * @returns VBox status code.
1272 * @param pVCpu The cross context virtual CPU structure.
1273 * @param cMsrs The number of MSRs.
1274 */
1275DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1276{
1277 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1278 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1279 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1280 {
1281 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1282 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1283 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1284 }
1285
1286 /* Update number of guest MSRs to load/store across the world-switch. */
1287 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1288 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1289
1290 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1291 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1292 AssertRCReturn(rc, rc);
1293
1294 /* Update the VCPU's copy of the MSR count. */
1295 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1296
1297 return VINF_SUCCESS;
1298}
1299
1300
1301/**
1302 * Adds a new (or updates the value of an existing) guest/host MSR
1303 * pair to be swapped during the world-switch as part of the
1304 * auto-load/store MSR area in the VMCS.
1305 *
1306 * @returns VBox status code.
1307 * @param pVCpu The cross context virtual CPU structure.
1308 * @param uMsr The MSR.
1309 * @param uGuestMsrValue Value of the guest MSR.
1310 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1311 * necessary.
1312 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1313 * its value was updated. Optional, can be NULL.
1314 */
1315static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1316 bool *pfAddedAndUpdated)
1317{
1318 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1319 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1320 uint32_t i;
1321 for (i = 0; i < cMsrs; i++)
1322 {
1323 if (pGuestMsr->u32Msr == uMsr)
1324 break;
1325 pGuestMsr++;
1326 }
1327
1328 bool fAdded = false;
1329 if (i == cMsrs)
1330 {
1331 ++cMsrs;
1332 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1333 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1334
1335 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1336 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1337 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1338
1339 fAdded = true;
1340 }
1341
1342 /* Update the MSR values in the auto-load/store MSR area. */
1343 pGuestMsr->u32Msr = uMsr;
1344 pGuestMsr->u64Value = uGuestMsrValue;
1345
1346 /* Create/update the MSR slot in the host MSR area. */
1347 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1348 pHostMsr += i;
1349 pHostMsr->u32Msr = uMsr;
1350
1351 /*
1352 * Update the host MSR only when requested by the caller AND when we're
1353 * adding it to the auto-load/store area. Otherwise, it would have been
1354 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1355 */
1356 bool fUpdatedMsrValue = false;
1357 if ( fAdded
1358 && fUpdateHostMsr)
1359 {
1360 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1361 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1362 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1363 fUpdatedMsrValue = true;
1364 }
1365
1366 if (pfAddedAndUpdated)
1367 *pfAddedAndUpdated = fUpdatedMsrValue;
1368 return VINF_SUCCESS;
1369}
1370
1371
1372/**
1373 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1374 * auto-load/store MSR area in the VMCS.
1375 *
1376 * @returns VBox status code.
1377 * @param pVCpu The cross context virtual CPU structure.
1378 * @param uMsr The MSR.
1379 */
1380static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1381{
1382 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1383 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1384 for (uint32_t i = 0; i < cMsrs; i++)
1385 {
1386 /* Find the MSR. */
1387 if (pGuestMsr->u32Msr == uMsr)
1388 {
1389 /* If it's the last MSR, simply reduce the count. */
1390 if (i == cMsrs - 1)
1391 {
1392 --cMsrs;
1393 break;
1394 }
1395
1396 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1397 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1398 pLastGuestMsr += cMsrs - 1;
1399 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1400 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1401
1402 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1403 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1404 pLastHostMsr += cMsrs - 1;
1405 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1406 pHostMsr->u64Value = pLastHostMsr->u64Value;
1407 --cMsrs;
1408 break;
1409 }
1410 pGuestMsr++;
1411 }
1412
1413 /* Update the VMCS if the count changed (meaning the MSR was found). */
1414 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1415 {
1416 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1417 AssertRCReturn(rc, rc);
1418
1419 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1420 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1421 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1422
1423 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1424 return VINF_SUCCESS;
1425 }
1426
1427 return VERR_NOT_FOUND;
1428}
1429
1430
1431/**
1432 * Checks if the specified guest MSR is part of the auto-load/store area in
1433 * the VMCS.
1434 *
1435 * @returns true if found, false otherwise.
1436 * @param pVCpu The cross context virtual CPU structure.
1437 * @param uMsr The MSR to find.
1438 */
1439static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1440{
1441 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1442 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1443
1444 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1445 {
1446 if (pGuestMsr->u32Msr == uMsr)
1447 return true;
1448 }
1449 return false;
1450}
1451
1452
1453/**
1454 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1455 *
1456 * @param pVCpu The cross context virtual CPU structure.
1457 *
1458 * @remarks No-long-jump zone!!!
1459 */
1460static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1461{
1462 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1463 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1464 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1465 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1466
1467 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1468 {
1469 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1470
1471 /*
1472 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1473 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1474 */
1475 if (pHostMsr->u32Msr == MSR_K6_EFER)
1476 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1477 else
1478 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1479 }
1480
1481 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1482}
1483
1484
1485/**
1486 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1487 * perform lazy restoration of the host MSRs while leaving VT-x.
1488 *
1489 * @param pVCpu The cross context virtual CPU structure.
1490 *
1491 * @remarks No-long-jump zone!!!
1492 */
1493static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1494{
1495 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1496
1497 /*
1498 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1499 */
1500 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1501 {
1502 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1503#if HC_ARCH_BITS == 64
1504 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1505 {
1506 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1507 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1508 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1509 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1510 }
1511#endif
1512 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1513 }
1514}
1515
1516
1517/**
1518 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1519 * lazily while leaving VT-x.
1520 *
1521 * @returns true if it does, false otherwise.
1522 * @param pVCpu The cross context virtual CPU structure.
1523 * @param uMsr The MSR to check.
1524 */
1525static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1526{
1527 NOREF(pVCpu);
1528#if HC_ARCH_BITS == 64
1529 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1530 {
1531 switch (uMsr)
1532 {
1533 case MSR_K8_LSTAR:
1534 case MSR_K6_STAR:
1535 case MSR_K8_SF_MASK:
1536 case MSR_K8_KERNEL_GS_BASE:
1537 return true;
1538 }
1539 }
1540#else
1541 RT_NOREF(pVCpu, uMsr);
1542#endif
1543 return false;
1544}
1545
1546
1547/**
1548 * Saves a set of guest MSRs back into the guest-CPU context.
1549 *
1550 * @param pVCpu The cross context virtual CPU structure.
1551 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1552 * out-of-sync. Make sure to update the required fields
1553 * before using them.
1554 *
1555 * @remarks No-long-jump zone!!!
1556 */
1557static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1558{
1559 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1560 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1561
1562 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1563 {
1564 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1565#if HC_ARCH_BITS == 64
1566 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1567 {
1568 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1569 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1570 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1571 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1572 }
1573#else
1574 NOREF(pMixedCtx);
1575#endif
1576 }
1577}
1578
1579
1580/**
1581 * Loads a set of guests MSRs to allow read/passthru to the guest.
1582 *
1583 * The name of this function is slightly confusing. This function does NOT
1584 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1585 * common prefix for functions dealing with "lazy restoration" of the shared
1586 * MSRs.
1587 *
1588 * @param pVCpu The cross context virtual CPU structure.
1589 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1590 * out-of-sync. Make sure to update the required fields
1591 * before using them.
1592 *
1593 * @remarks No-long-jump zone!!!
1594 */
1595static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1596{
1597 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1598 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1599
1600 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1601#if HC_ARCH_BITS == 64
1602 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1603 {
1604 /*
1605 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1606 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1607 * we can skip a few MSR writes.
1608 *
1609 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1610 * guest MSR values in the guest-CPU context might be different to what's currently
1611 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1612 * CPU, see @bugref{8728}.
1613 */
1614 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1615 && pMixedCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1616 && pMixedCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1617 && pMixedCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1618 && pMixedCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1619 {
1620#ifdef VBOX_STRICT
1621 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pMixedCtx->msrKERNELGSBASE);
1622 Assert(ASMRdMsr(MSR_K8_LSTAR) == pMixedCtx->msrLSTAR);
1623 Assert(ASMRdMsr(MSR_K6_STAR) == pMixedCtx->msrSTAR);
1624 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pMixedCtx->msrSFMASK);
1625#endif
1626 }
1627 else
1628 {
1629 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1630 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1631 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1632 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1633 }
1634 }
1635#else
1636 RT_NOREF(pMixedCtx);
1637#endif
1638 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1639}
1640
1641
1642/**
1643 * Performs lazy restoration of the set of host MSRs if they were previously
1644 * loaded with guest MSR values.
1645 *
1646 * @param pVCpu The cross context virtual CPU structure.
1647 *
1648 * @remarks No-long-jump zone!!!
1649 * @remarks The guest MSRs should have been saved back into the guest-CPU
1650 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1651 */
1652static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1653{
1654 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1655 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1656
1657 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1658 {
1659 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1660#if HC_ARCH_BITS == 64
1661 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1662 {
1663 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1664 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1665 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1666 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1667 }
1668#endif
1669 }
1670 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1671}
1672
1673
1674/**
1675 * Verifies that our cached values of the VMCS controls are all
1676 * consistent with what's actually present in the VMCS.
1677 *
1678 * @returns VBox status code.
1679 * @param pVCpu The cross context virtual CPU structure.
1680 */
1681static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1682{
1683 uint32_t u32Val;
1684 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1685 AssertRCReturn(rc, rc);
1686 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1687 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1688
1689 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1690 AssertRCReturn(rc, rc);
1691 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1692 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1693
1694 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1695 AssertRCReturn(rc, rc);
1696 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1697 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1698
1699 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1700 AssertRCReturn(rc, rc);
1701 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1702 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1703
1704 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1705 {
1706 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1707 AssertRCReturn(rc, rc);
1708 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1709 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1710 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1711 }
1712
1713 return VINF_SUCCESS;
1714}
1715
1716
1717#ifdef VBOX_STRICT
1718/**
1719 * Verifies that our cached host EFER value has not changed
1720 * since we cached it.
1721 *
1722 * @param pVCpu The cross context virtual CPU structure.
1723 */
1724static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1725{
1726 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1727
1728 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1729 {
1730 uint64_t u64Val;
1731 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1732 AssertRC(rc);
1733
1734 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1735 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1736 }
1737}
1738
1739
1740/**
1741 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1742 * VMCS are correct.
1743 *
1744 * @param pVCpu The cross context virtual CPU structure.
1745 */
1746static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1747{
1748 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1749
1750 /* Verify MSR counts in the VMCS are what we think it should be. */
1751 uint32_t cMsrs;
1752 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1753 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1754
1755 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1756 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1757
1758 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1759 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1760
1761 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1762 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1763 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1764 {
1765 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1766 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1767 pGuestMsr->u32Msr, cMsrs));
1768
1769 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1770 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1771 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1772
1773 /* Verify that the permissions are as expected in the MSR bitmap. */
1774 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1775 {
1776 VMXMSREXITREAD enmRead;
1777 VMXMSREXITWRITE enmWrite;
1778 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1779 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1780 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1781 {
1782 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1783 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1784 }
1785 else
1786 {
1787 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1788 pGuestMsr->u32Msr, cMsrs));
1789 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1790 pGuestMsr->u32Msr, cMsrs));
1791 }
1792 }
1793 }
1794}
1795#endif /* VBOX_STRICT */
1796
1797
1798/**
1799 * Flushes the TLB using EPT.
1800 *
1801 * @returns VBox status code.
1802 * @param pVCpu The cross context virtual CPU structure of the calling
1803 * EMT. Can be NULL depending on @a enmFlush.
1804 * @param enmFlush Type of flush.
1805 *
1806 * @remarks Caller is responsible for making sure this function is called only
1807 * when NestedPaging is supported and providing @a enmFlush that is
1808 * supported by the CPU.
1809 * @remarks Can be called with interrupts disabled.
1810 */
1811static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1812{
1813 uint64_t au64Descriptor[2];
1814 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1815 au64Descriptor[0] = 0;
1816 else
1817 {
1818 Assert(pVCpu);
1819 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1820 }
1821 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1822
1823 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1824 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1825 rc));
1826 if ( RT_SUCCESS(rc)
1827 && pVCpu)
1828 {
1829 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1830 }
1831}
1832
1833
1834/**
1835 * Flushes the TLB using VPID.
1836 *
1837 * @returns VBox status code.
1838 * @param pVM The cross context VM structure.
1839 * @param pVCpu The cross context virtual CPU structure of the calling
1840 * EMT. Can be NULL depending on @a enmFlush.
1841 * @param enmFlush Type of flush.
1842 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1843 * on @a enmFlush).
1844 *
1845 * @remarks Can be called with interrupts disabled.
1846 */
1847static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1848{
1849 NOREF(pVM);
1850 AssertPtr(pVM);
1851 Assert(pVM->hm.s.vmx.fVpid);
1852
1853 uint64_t au64Descriptor[2];
1854 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1855 {
1856 au64Descriptor[0] = 0;
1857 au64Descriptor[1] = 0;
1858 }
1859 else
1860 {
1861 AssertPtr(pVCpu);
1862 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1863 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1864 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1865 au64Descriptor[1] = GCPtr;
1866 }
1867
1868 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1869 AssertMsg(rc == VINF_SUCCESS,
1870 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1871 if ( RT_SUCCESS(rc)
1872 && pVCpu)
1873 {
1874 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1875 }
1876}
1877
1878
1879/**
1880 * Invalidates a guest page by guest virtual address. Only relevant for
1881 * EPT/VPID, otherwise there is nothing really to invalidate.
1882 *
1883 * @returns VBox status code.
1884 * @param pVM The cross context VM structure.
1885 * @param pVCpu The cross context virtual CPU structure.
1886 * @param GCVirt Guest virtual address of the page to invalidate.
1887 */
1888VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1889{
1890 AssertPtr(pVM);
1891 AssertPtr(pVCpu);
1892 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1893
1894 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1895 if (!fFlushPending)
1896 {
1897 /*
1898 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1899 * See @bugref{6043} and @bugref{6177}.
1900 *
1901 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1902 * function maybe called in a loop with individual addresses.
1903 */
1904 if (pVM->hm.s.vmx.fVpid)
1905 {
1906 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1907 {
1908 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1909 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1910 }
1911 else
1912 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1913 }
1914 else if (pVM->hm.s.fNestedPaging)
1915 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1916 }
1917
1918 return VINF_SUCCESS;
1919}
1920
1921
1922/**
1923 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1924 * otherwise there is nothing really to invalidate.
1925 *
1926 * @returns VBox status code.
1927 * @param pVM The cross context VM structure.
1928 * @param pVCpu The cross context virtual CPU structure.
1929 * @param GCPhys Guest physical address of the page to invalidate.
1930 */
1931VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1932{
1933 NOREF(pVM); NOREF(GCPhys);
1934 LogFlowFunc(("%RGp\n", GCPhys));
1935
1936 /*
1937 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1938 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1939 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1940 */
1941 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1942 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1943 return VINF_SUCCESS;
1944}
1945
1946
1947/**
1948 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1949 * case where neither EPT nor VPID is supported by the CPU.
1950 *
1951 * @param pVM The cross context VM structure.
1952 * @param pVCpu The cross context virtual CPU structure.
1953 * @param pCpu Pointer to the global HM struct.
1954 *
1955 * @remarks Called with interrupts disabled.
1956 */
1957static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1958{
1959 AssertPtr(pVCpu);
1960 AssertPtr(pCpu);
1961 NOREF(pVM);
1962
1963 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1964
1965 Assert(pCpu->idCpu != NIL_RTCPUID);
1966 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1967 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1968 pVCpu->hm.s.fForceTLBFlush = false;
1969 return;
1970}
1971
1972
1973/**
1974 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1975 *
1976 * @param pVM The cross context VM structure.
1977 * @param pVCpu The cross context virtual CPU structure.
1978 * @param pCpu Pointer to the global HM CPU struct.
1979 * @remarks All references to "ASID" in this function pertains to "VPID" in
1980 * Intel's nomenclature. The reason is, to avoid confusion in compare
1981 * statements since the host-CPU copies are named "ASID".
1982 *
1983 * @remarks Called with interrupts disabled.
1984 */
1985static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1986{
1987#ifdef VBOX_WITH_STATISTICS
1988 bool fTlbFlushed = false;
1989# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1990# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1991 if (!fTlbFlushed) \
1992 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1993 } while (0)
1994#else
1995# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1996# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1997#endif
1998
1999 AssertPtr(pVM);
2000 AssertPtr(pCpu);
2001 AssertPtr(pVCpu);
2002 Assert(pCpu->idCpu != NIL_RTCPUID);
2003
2004 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2005 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2006 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2007
2008 /*
2009 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2010 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2011 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2012 */
2013 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2014 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2015 {
2016 ++pCpu->uCurrentAsid;
2017 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2018 {
2019 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2020 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2021 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2022 }
2023
2024 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2025 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2026 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2027
2028 /*
2029 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2030 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2031 */
2032 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2033 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2034 HMVMX_SET_TAGGED_TLB_FLUSHED();
2035 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
2036 }
2037
2038 /* Check for explicit TLB flushes. */
2039 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2040 {
2041 /*
2042 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
2043 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2044 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2045 * but not guest-physical mappings.
2046 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2047 */
2048 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2049 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2050 HMVMX_SET_TAGGED_TLB_FLUSHED();
2051 }
2052
2053 pVCpu->hm.s.fForceTLBFlush = false;
2054 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2055
2056 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2057 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2058 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2059 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2060 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2061 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2062 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2063 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2064 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2065
2066 /* Update VMCS with the VPID. */
2067 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2068 AssertRC(rc);
2069
2070#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2071}
2072
2073
2074/**
2075 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2076 *
2077 * @returns VBox status code.
2078 * @param pVM The cross context VM structure.
2079 * @param pVCpu The cross context virtual CPU structure.
2080 * @param pCpu Pointer to the global HM CPU struct.
2081 *
2082 * @remarks Called with interrupts disabled.
2083 */
2084static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2085{
2086 AssertPtr(pVM);
2087 AssertPtr(pVCpu);
2088 AssertPtr(pCpu);
2089 Assert(pCpu->idCpu != NIL_RTCPUID);
2090 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2091 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2092
2093 /*
2094 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2095 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2096 */
2097 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2098 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2099 {
2100 pVCpu->hm.s.fForceTLBFlush = true;
2101 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2102 }
2103
2104 /* Check for explicit TLB flushes. */
2105 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2106 {
2107 pVCpu->hm.s.fForceTLBFlush = true;
2108 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2109 }
2110
2111 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2112 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2113
2114 if (pVCpu->hm.s.fForceTLBFlush)
2115 {
2116 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2117 pVCpu->hm.s.fForceTLBFlush = false;
2118 }
2119}
2120
2121
2122/**
2123 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2124 *
2125 * @returns VBox status code.
2126 * @param pVM The cross context VM structure.
2127 * @param pVCpu The cross context virtual CPU structure.
2128 * @param pCpu Pointer to the global HM CPU struct.
2129 *
2130 * @remarks Called with interrupts disabled.
2131 */
2132static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2133{
2134 AssertPtr(pVM);
2135 AssertPtr(pVCpu);
2136 AssertPtr(pCpu);
2137 Assert(pCpu->idCpu != NIL_RTCPUID);
2138 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2139 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2140
2141 /*
2142 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2143 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2144 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2145 */
2146 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2147 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2148 {
2149 pVCpu->hm.s.fForceTLBFlush = true;
2150 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2151 }
2152
2153 /* Check for explicit TLB flushes. */
2154 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2155 {
2156 /*
2157 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2158 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2159 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2160 */
2161 pVCpu->hm.s.fForceTLBFlush = true;
2162 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2163 }
2164
2165 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2166 if (pVCpu->hm.s.fForceTLBFlush)
2167 {
2168 ++pCpu->uCurrentAsid;
2169 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2170 {
2171 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2172 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2173 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2174 }
2175
2176 pVCpu->hm.s.fForceTLBFlush = false;
2177 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2178 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2179 if (pCpu->fFlushAsidBeforeUse)
2180 {
2181 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2182 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2183 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2184 {
2185 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2186 pCpu->fFlushAsidBeforeUse = false;
2187 }
2188 else
2189 {
2190 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2191 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2192 }
2193 }
2194 }
2195
2196 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2197 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2198 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2199 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2200 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2201 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2202 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2203
2204 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2205 AssertRC(rc);
2206}
2207
2208
2209/**
2210 * Flushes the guest TLB entry based on CPU capabilities.
2211 *
2212 * @param pVCpu The cross context virtual CPU structure.
2213 * @param pCpu Pointer to the global HM CPU struct.
2214 */
2215DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2216{
2217#ifdef HMVMX_ALWAYS_FLUSH_TLB
2218 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2219#endif
2220 PVM pVM = pVCpu->CTX_SUFF(pVM);
2221 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2222 {
2223 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2224 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2225 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2226 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2227 default:
2228 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2229 break;
2230 }
2231
2232 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2233}
2234
2235
2236/**
2237 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2238 * TLB entries from the host TLB before VM-entry.
2239 *
2240 * @returns VBox status code.
2241 * @param pVM The cross context VM structure.
2242 */
2243static int hmR0VmxSetupTaggedTlb(PVM pVM)
2244{
2245 /*
2246 * Determine optimal flush type for Nested Paging.
2247 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2248 * guest execution (see hmR3InitFinalizeR0()).
2249 */
2250 if (pVM->hm.s.fNestedPaging)
2251 {
2252 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2253 {
2254 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2255 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2256 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2257 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2258 else
2259 {
2260 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2261 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2262 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2263 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2264 }
2265
2266 /* Make sure the write-back cacheable memory type for EPT is supported. */
2267 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2268 {
2269 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2270 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2271 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2272 }
2273
2274 /* EPT requires a page-walk length of 4. */
2275 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2276 {
2277 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2278 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2279 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2280 }
2281 }
2282 else
2283 {
2284 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2285 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2286 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2287 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2288 }
2289 }
2290
2291 /*
2292 * Determine optimal flush type for VPID.
2293 */
2294 if (pVM->hm.s.vmx.fVpid)
2295 {
2296 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2297 {
2298 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2299 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2300 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2301 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2302 else
2303 {
2304 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2305 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2306 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2307 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2308 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2309 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2310 pVM->hm.s.vmx.fVpid = false;
2311 }
2312 }
2313 else
2314 {
2315 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2316 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2317 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2318 pVM->hm.s.vmx.fVpid = false;
2319 }
2320 }
2321
2322 /*
2323 * Setup the handler for flushing tagged-TLBs.
2324 */
2325 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2326 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2327 else if (pVM->hm.s.fNestedPaging)
2328 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2329 else if (pVM->hm.s.vmx.fVpid)
2330 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2331 else
2332 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2333 return VINF_SUCCESS;
2334}
2335
2336
2337/**
2338 * Sets up pin-based VM-execution controls in the VMCS.
2339 *
2340 * @returns VBox status code.
2341 * @param pVM The cross context VM structure.
2342 * @param pVCpu The cross context virtual CPU structure.
2343 */
2344static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2345{
2346 AssertPtr(pVM);
2347 AssertPtr(pVCpu);
2348
2349 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2350 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2351
2352 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2353 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2354
2355 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2356 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2357
2358 /* Enable the VMX preemption timer. */
2359 if (pVM->hm.s.vmx.fUsePreemptTimer)
2360 {
2361 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2362 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2363 }
2364
2365#if 0
2366 /* Enable posted-interrupt processing. */
2367 if (pVM->hm.s.fPostedIntrs)
2368 {
2369 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2370 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2371 val |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2372 }
2373#endif
2374
2375 if ((val & zap) != val)
2376 {
2377 LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2378 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2379 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2380 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2381 }
2382
2383 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2384 AssertRCReturn(rc, rc);
2385
2386 pVCpu->hm.s.vmx.u32PinCtls = val;
2387 return rc;
2388}
2389
2390
2391/**
2392 * Sets up processor-based VM-execution controls in the VMCS.
2393 *
2394 * @returns VBox status code.
2395 * @param pVM The cross context VM structure.
2396 * @param pVCpu The cross context virtual CPU structure.
2397 */
2398static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2399{
2400 AssertPtr(pVM);
2401 AssertPtr(pVCpu);
2402
2403 int rc = VERR_INTERNAL_ERROR_5;
2404 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2405 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2406
2407 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2408 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2409 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2410 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2411 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2412 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2413 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2414
2415 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2416 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2417 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2418 {
2419 LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2420 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2421 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2422 }
2423
2424 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2425 if (!pVM->hm.s.fNestedPaging)
2426 {
2427 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2428 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2429 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2430 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2431 }
2432
2433 /* Use TPR shadowing if supported by the CPU. */
2434 if ( PDMHasApic(pVM)
2435 && pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2436 {
2437 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2438 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2439 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2440 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2441 AssertRCReturn(rc, rc);
2442
2443 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2444 /* CR8 writes cause a VM-exit based on TPR threshold. */
2445 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2446 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2447 }
2448 else
2449 {
2450 /*
2451 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2452 * Set this control only for 64-bit guests.
2453 */
2454 if (pVM->hm.s.fAllow64BitGuests)
2455 {
2456 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2457 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2458 }
2459 }
2460
2461 /* Use MSR-bitmaps if supported by the CPU. */
2462 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2463 {
2464 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2465
2466 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2467 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2468 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2469 AssertRCReturn(rc, rc);
2470
2471 /*
2472 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2473 * automatically using dedicated fields in the VMCS.
2474 */
2475 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2476 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2477 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2478 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2479 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2480
2481#if HC_ARCH_BITS == 64
2482 /*
2483 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2484 */
2485 if (pVM->hm.s.fAllow64BitGuests)
2486 {
2487 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2488 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2489 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2490 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2491 }
2492#endif
2493 /*
2494 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2495 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2496 */
2497 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2498 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2499
2500 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2501 }
2502
2503 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2504 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2505 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2506
2507 if ((val & zap) != val)
2508 {
2509 LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2510 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2511 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2512 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2513 }
2514
2515 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2516 AssertRCReturn(rc, rc);
2517
2518 pVCpu->hm.s.vmx.u32ProcCtls = val;
2519
2520 /*
2521 * Secondary processor-based VM-execution controls.
2522 */
2523 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2524 {
2525 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2526 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2527
2528 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2529 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2530
2531 if (pVM->hm.s.fNestedPaging)
2532 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2533
2534 /*
2535 * Enable the INVPCID instruction if supported by the hardware and we expose
2536 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2537 */
2538 if ( (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2539 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2540 {
2541 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2542 }
2543
2544 if (pVM->hm.s.vmx.fVpid)
2545 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2546
2547 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2548 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2549
2550#if 0
2551 if (pVM->hm.s.fVirtApicRegs)
2552 {
2553 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2554 val |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2555
2556 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2557 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2558 }
2559#endif
2560
2561 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2562 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2563 * done dynamically. */
2564 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2565 {
2566 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2567 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2568 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2569 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2570 AssertRCReturn(rc, rc);
2571 }
2572
2573 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2574 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2575
2576 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2577 && pVM->hm.s.vmx.cPleGapTicks
2578 && pVM->hm.s.vmx.cPleWindowTicks)
2579 {
2580 val |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2581
2582 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2583 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2584 AssertRCReturn(rc, rc);
2585 }
2586
2587 if ((val & zap) != val)
2588 {
2589 LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2590 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2591 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2592 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2593 }
2594
2595 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2596 AssertRCReturn(rc, rc);
2597
2598 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2599 }
2600 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2601 {
2602 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2603 "available\n"));
2604 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2605 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2606 }
2607
2608 return VINF_SUCCESS;
2609}
2610
2611
2612/**
2613 * Sets up miscellaneous (everything other than Pin & Processor-based
2614 * VM-execution) control fields in the VMCS.
2615 *
2616 * @returns VBox status code.
2617 * @param pVM The cross context VM structure.
2618 * @param pVCpu The cross context virtual CPU structure.
2619 */
2620static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2621{
2622 NOREF(pVM);
2623 AssertPtr(pVM);
2624 AssertPtr(pVCpu);
2625
2626 int rc = VERR_GENERAL_FAILURE;
2627
2628 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2629#if 0
2630 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2631 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2632 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2633
2634 /*
2635 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2636 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2637 * We thus use the exception bitmap to control it rather than use both.
2638 */
2639 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2640 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2641
2642 /** @todo Explore possibility of using IO-bitmaps. */
2643 /* All IO & IOIO instructions cause VM-exits. */
2644 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2645 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2646
2647 /* Initialize the MSR-bitmap area. */
2648 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2649 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2650 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2651 AssertRCReturn(rc, rc);
2652#endif
2653
2654 /* Setup MSR auto-load/store area. */
2655 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2656 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2657 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2658 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2659 AssertRCReturn(rc, rc);
2660
2661 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2662 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2663 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2664 AssertRCReturn(rc, rc);
2665
2666 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2667 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2668 AssertRCReturn(rc, rc);
2669
2670 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2671#if 0
2672 /* Setup debug controls */
2673 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2674 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2675 AssertRCReturn(rc, rc);
2676#endif
2677
2678 return rc;
2679}
2680
2681
2682/**
2683 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2684 *
2685 * We shall setup those exception intercepts that don't change during the
2686 * lifetime of the VM here. The rest are done dynamically while loading the
2687 * guest state.
2688 *
2689 * @returns VBox status code.
2690 * @param pVM The cross context VM structure.
2691 * @param pVCpu The cross context virtual CPU structure.
2692 */
2693static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2694{
2695 AssertPtr(pVM);
2696 AssertPtr(pVCpu);
2697
2698 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2699
2700 uint32_t u32XcptBitmap = 0;
2701
2702 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2703 u32XcptBitmap |= RT_BIT_32(X86_XCPT_AC);
2704
2705 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2706 and writes, and because recursive #DBs can cause the CPU hang, we must always
2707 intercept #DB. */
2708 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2709
2710 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2711 if (!pVM->hm.s.fNestedPaging)
2712 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2713
2714 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2715 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2716 AssertRCReturn(rc, rc);
2717 return rc;
2718}
2719
2720
2721/**
2722 * Sets up the initial guest-state mask. The guest-state mask is consulted
2723 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2724 * for the nested virtualization case (as it would cause a VM-exit).
2725 *
2726 * @param pVCpu The cross context virtual CPU structure.
2727 */
2728static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2729{
2730 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2731 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2732 return VINF_SUCCESS;
2733}
2734
2735
2736/**
2737 * Does per-VM VT-x initialization.
2738 *
2739 * @returns VBox status code.
2740 * @param pVM The cross context VM structure.
2741 */
2742VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2743{
2744 LogFlowFunc(("pVM=%p\n", pVM));
2745
2746 int rc = hmR0VmxStructsAlloc(pVM);
2747 if (RT_FAILURE(rc))
2748 {
2749 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2750 return rc;
2751 }
2752
2753 return VINF_SUCCESS;
2754}
2755
2756
2757/**
2758 * Does per-VM VT-x termination.
2759 *
2760 * @returns VBox status code.
2761 * @param pVM The cross context VM structure.
2762 */
2763VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2764{
2765 LogFlowFunc(("pVM=%p\n", pVM));
2766
2767#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2768 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2769 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2770#endif
2771 hmR0VmxStructsFree(pVM);
2772 return VINF_SUCCESS;
2773}
2774
2775
2776/**
2777 * Sets up the VM for execution under VT-x.
2778 * This function is only called once per-VM during initialization.
2779 *
2780 * @returns VBox status code.
2781 * @param pVM The cross context VM structure.
2782 */
2783VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2784{
2785 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2786 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2787
2788 LogFlowFunc(("pVM=%p\n", pVM));
2789
2790 /*
2791 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2792 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2793 */
2794 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2795 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2796 || !pVM->hm.s.vmx.pRealModeTSS))
2797 {
2798 LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2799 return VERR_INTERNAL_ERROR;
2800 }
2801
2802 /* Initialize these always, see hmR3InitFinalizeR0().*/
2803 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2804 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2805
2806 /* Setup the tagged-TLB flush handlers. */
2807 int rc = hmR0VmxSetupTaggedTlb(pVM);
2808 if (RT_FAILURE(rc))
2809 {
2810 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2811 return rc;
2812 }
2813
2814 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2815 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2816#if HC_ARCH_BITS == 64
2817 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2818 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2819 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2820 {
2821 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2822 }
2823#endif
2824
2825 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2826 RTCCUINTREG uHostCR4 = ASMGetCR4();
2827 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2828 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2829
2830 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2831 {
2832 PVMCPU pVCpu = &pVM->aCpus[i];
2833 AssertPtr(pVCpu);
2834 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2835
2836 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2837 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2838
2839 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2840 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2841 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2842
2843 /* Set revision dword at the beginning of the VMCS structure. */
2844 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2845
2846 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2847 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2848 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2849 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2850
2851 /* Load this VMCS as the current VMCS. */
2852 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2853 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2854 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2855
2856 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2857 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2858 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2859
2860 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2861 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2862 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2863
2864 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2865 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2866 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2867
2868 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2869 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2870 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2871
2872 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2873 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2874 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2875
2876#if HC_ARCH_BITS == 32
2877 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2878 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2879 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2880#endif
2881
2882 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2883 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2884 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2885 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2886
2887 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2888
2889 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2890 }
2891
2892 return VINF_SUCCESS;
2893}
2894
2895
2896/**
2897 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2898 * the VMCS.
2899 *
2900 * @returns VBox status code.
2901 * @param pVM The cross context VM structure.
2902 * @param pVCpu The cross context virtual CPU structure.
2903 */
2904DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2905{
2906 NOREF(pVM); NOREF(pVCpu);
2907
2908 RTCCUINTREG uReg = ASMGetCR0();
2909 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2910 AssertRCReturn(rc, rc);
2911
2912 uReg = ASMGetCR3();
2913 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2914 AssertRCReturn(rc, rc);
2915
2916 uReg = ASMGetCR4();
2917 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2918 AssertRCReturn(rc, rc);
2919 return rc;
2920}
2921
2922
2923#if HC_ARCH_BITS == 64
2924/**
2925 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2926 * requirements. See hmR0VmxSaveHostSegmentRegs().
2927 */
2928# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2929 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2930 { \
2931 bool fValidSelector = true; \
2932 if ((selValue) & X86_SEL_LDT) \
2933 { \
2934 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2935 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2936 } \
2937 if (fValidSelector) \
2938 { \
2939 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2940 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2941 } \
2942 (selValue) = 0; \
2943 }
2944#endif
2945
2946
2947/**
2948 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2949 * the host-state area in the VMCS.
2950 *
2951 * @returns VBox status code.
2952 * @param pVM The cross context VM structure.
2953 * @param pVCpu The cross context virtual CPU structure.
2954 */
2955DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2956{
2957 int rc = VERR_INTERNAL_ERROR_5;
2958
2959#if HC_ARCH_BITS == 64
2960 /*
2961 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2962 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2963 *
2964 * This apparently can happen (most likely the FPU changes), deal with it rather than asserting.
2965 * Was observed booting Solaris10u10 32-bit guest.
2966 */
2967 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2968 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2969 {
2970 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2971 pVCpu->idCpu));
2972 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2973 }
2974 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2975#else
2976 RT_NOREF(pVCpu);
2977#endif
2978
2979 /*
2980 * Host DS, ES, FS and GS segment registers.
2981 */
2982#if HC_ARCH_BITS == 64
2983 RTSEL uSelDS = ASMGetDS();
2984 RTSEL uSelES = ASMGetES();
2985 RTSEL uSelFS = ASMGetFS();
2986 RTSEL uSelGS = ASMGetGS();
2987#else
2988 RTSEL uSelDS = 0;
2989 RTSEL uSelES = 0;
2990 RTSEL uSelFS = 0;
2991 RTSEL uSelGS = 0;
2992#endif
2993
2994 /*
2995 * Host CS and SS segment registers.
2996 */
2997 RTSEL uSelCS = ASMGetCS();
2998 RTSEL uSelSS = ASMGetSS();
2999
3000 /*
3001 * Host TR segment register.
3002 */
3003 RTSEL uSelTR = ASMGetTR();
3004
3005#if HC_ARCH_BITS == 64
3006 /*
3007 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
3008 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
3009 */
3010 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
3011 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
3012 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
3013 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
3014# undef VMXLOCAL_ADJUST_HOST_SEG
3015#endif
3016
3017 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
3018 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
3019 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
3020 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
3021 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
3022 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
3023 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
3024 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
3025 Assert(uSelCS);
3026 Assert(uSelTR);
3027
3028 /* Assertion is right but we would not have updated u32ExitCtls yet. */
3029#if 0
3030 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
3031 Assert(uSelSS != 0);
3032#endif
3033
3034 /* Write these host selector fields into the host-state area in the VMCS. */
3035 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3036 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3037#if HC_ARCH_BITS == 64
3038 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3039 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3040 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3041 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3042#else
3043 NOREF(uSelDS);
3044 NOREF(uSelES);
3045 NOREF(uSelFS);
3046 NOREF(uSelGS);
3047#endif
3048 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3049 AssertRCReturn(rc, rc);
3050
3051 /*
3052 * Host GDTR and IDTR.
3053 */
3054 RTGDTR Gdtr;
3055 RTIDTR Idtr;
3056 RT_ZERO(Gdtr);
3057 RT_ZERO(Idtr);
3058 ASMGetGDTR(&Gdtr);
3059 ASMGetIDTR(&Idtr);
3060 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3061 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3062 AssertRCReturn(rc, rc);
3063
3064#if HC_ARCH_BITS == 64
3065 /*
3066 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3067 * maximum limit (0xffff) on every VM-exit.
3068 */
3069 if (Gdtr.cbGdt != 0xffff)
3070 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3071
3072 /*
3073 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3074 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3075 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3076 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3077 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3078 * hosts where we are pretty sure it won't cause trouble.
3079 */
3080# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3081 if (Idtr.cbIdt < 0x0fff)
3082# else
3083 if (Idtr.cbIdt != 0xffff)
3084# endif
3085 {
3086 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3087 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3088 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3089 }
3090#endif
3091
3092 /*
3093 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3094 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3095 */
3096 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3097 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3098 VERR_VMX_INVALID_HOST_STATE);
3099
3100 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3101#if HC_ARCH_BITS == 64
3102 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3103
3104 /*
3105 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3106 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3107 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3108 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3109 *
3110 * [1] See Intel spec. 3.5 "System Descriptor Types".
3111 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3112 */
3113 Assert(pDesc->System.u4Type == 11);
3114 if ( pDesc->System.u16LimitLow != 0x67
3115 || pDesc->System.u4LimitHigh)
3116 {
3117 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3118 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3119 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3120 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3121 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3122 }
3123
3124 /*
3125 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3126 */
3127 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3128 {
3129 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3130 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3131 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3132 {
3133 /* The GDT is read-only but the writable GDT is available. */
3134 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3135 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3136 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3137 AssertRCReturn(rc, rc);
3138 }
3139 }
3140#else
3141 NOREF(pVM);
3142 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3143#endif
3144 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3145 AssertRCReturn(rc, rc);
3146
3147 /*
3148 * Host FS base and GS base.
3149 */
3150#if HC_ARCH_BITS == 64
3151 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3152 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3153 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3154 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3155 AssertRCReturn(rc, rc);
3156
3157 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3158 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3159 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3160 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3161 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3162#endif
3163 return rc;
3164}
3165
3166
3167/**
3168 * Saves certain host MSRs in the VM-exit MSR-load area and some in the
3169 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3170 * the host after every successful VM-exit.
3171 *
3172 * @returns VBox status code.
3173 * @param pVM The cross context VM structure.
3174 * @param pVCpu The cross context virtual CPU structure.
3175 *
3176 * @remarks No-long-jump zone!!!
3177 */
3178DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3179{
3180 NOREF(pVM);
3181
3182 AssertPtr(pVCpu);
3183 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3184
3185 /*
3186 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3187 * rather than swapping them on every VM-entry.
3188 */
3189 hmR0VmxLazySaveHostMsrs(pVCpu);
3190
3191 /*
3192 * Host Sysenter MSRs.
3193 */
3194 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3195#if HC_ARCH_BITS == 32
3196 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3197 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3198#else
3199 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3200 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3201#endif
3202 AssertRCReturn(rc, rc);
3203
3204 /*
3205 * Host EFER MSR.
3206 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3207 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3208 */
3209 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3210 {
3211 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3212 AssertRCReturn(rc, rc);
3213 }
3214
3215 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3216 * hmR0VmxLoadGuestExitCtls() !! */
3217
3218 return rc;
3219}
3220
3221
3222/**
3223 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3224 *
3225 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3226 * these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3227 * hmR0VMxLoadGuestEntryCtls().
3228 *
3229 * @returns true if we need to load guest EFER, false otherwise.
3230 * @param pVCpu The cross context virtual CPU structure.
3231 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3232 * out-of-sync. Make sure to update the required fields
3233 * before using them.
3234 *
3235 * @remarks Requires EFER, CR4.
3236 * @remarks No-long-jump zone!!!
3237 */
3238static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3239{
3240#ifdef HMVMX_ALWAYS_SWAP_EFER
3241 return true;
3242#endif
3243
3244#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3245 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3246 if (CPUMIsGuestInLongMode(pVCpu))
3247 return false;
3248#endif
3249
3250 PVM pVM = pVCpu->CTX_SUFF(pVM);
3251 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3252 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3253
3254 /*
3255 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3256 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3257 */
3258 if ( CPUMIsGuestInLongMode(pVCpu)
3259 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3260 {
3261 return true;
3262 }
3263
3264 /*
3265 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3266 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3267 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3268 */
3269 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3270 && (pMixedCtx->cr0 & X86_CR0_PG)
3271 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3272 {
3273 /* Assert that host is PAE capable. */
3274 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3275 return true;
3276 }
3277
3278 /** @todo Check the latest Intel spec. for any other bits,
3279 * like SMEP/SMAP? */
3280 return false;
3281}
3282
3283
3284/**
3285 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3286 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3287 * controls".
3288 *
3289 * @returns VBox status code.
3290 * @param pVCpu The cross context virtual CPU structure.
3291 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3292 * out-of-sync. Make sure to update the required fields
3293 * before using them.
3294 *
3295 * @remarks Requires EFER.
3296 * @remarks No-long-jump zone!!!
3297 */
3298DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3299{
3300 int rc = VINF_SUCCESS;
3301 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3302 {
3303 PVM pVM = pVCpu->CTX_SUFF(pVM);
3304 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3305 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3306
3307 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3308 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3309
3310 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3311 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3312 {
3313 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3314 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3315 }
3316 else
3317 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3318
3319 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3320 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3321 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3322 {
3323 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3324 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3325 }
3326
3327 /*
3328 * The following should -not- be set (since we're not in SMM mode):
3329 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3330 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3331 */
3332
3333 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3334 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3335
3336 if ((val & zap) != val)
3337 {
3338 LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3339 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3340 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3341 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3342 }
3343
3344 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3345 AssertRCReturn(rc, rc);
3346
3347 pVCpu->hm.s.vmx.u32EntryCtls = val;
3348 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3349 }
3350 return rc;
3351}
3352
3353
3354/**
3355 * Sets up the VM-exit controls in the VMCS.
3356 *
3357 * @returns VBox status code.
3358 * @param pVCpu The cross context virtual CPU structure.
3359 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3360 * out-of-sync. Make sure to update the required fields
3361 * before using them.
3362 *
3363 * @remarks Requires EFER.
3364 */
3365DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3366{
3367 NOREF(pMixedCtx);
3368
3369 int rc = VINF_SUCCESS;
3370 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3371 {
3372 PVM pVM = pVCpu->CTX_SUFF(pVM);
3373 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3374 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3375
3376 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3377 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3378
3379 /*
3380 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3381 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3382 */
3383#if HC_ARCH_BITS == 64
3384 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3385 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3386#else
3387 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3388 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3389 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3390 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3391 {
3392 /* The switcher returns to long mode, EFER is managed by the switcher. */
3393 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3394 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3395 }
3396 else
3397 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3398#endif
3399
3400 /* If the newer VMCS fields for managing EFER exists, use it. */
3401 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3402 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3403 {
3404 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3405 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3406 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3407 }
3408
3409 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3410 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3411
3412 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3413 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3414 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3415
3416 if ( pVM->hm.s.vmx.fUsePreemptTimer
3417 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3418 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3419
3420 if ((val & zap) != val)
3421 {
3422 LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3423 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3424 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3425 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3426 }
3427
3428 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3429 AssertRCReturn(rc, rc);
3430
3431 pVCpu->hm.s.vmx.u32ExitCtls = val;
3432 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3433 }
3434 return rc;
3435}
3436
3437
3438/**
3439 * Sets the TPR threshold in the VMCS.
3440 *
3441 * @returns VBox status code.
3442 * @param pVCpu The cross context virtual CPU structure.
3443 * @param u32TprThreshold The TPR threshold (task-priority class only).
3444 */
3445DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3446{
3447 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3448 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3449 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3450}
3451
3452
3453/**
3454 * Loads the guest APIC and related state.
3455 *
3456 * @returns VBox status code.
3457 * @param pVCpu The cross context virtual CPU structure.
3458 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3459 * out-of-sync. Make sure to update the required fields
3460 * before using them.
3461 *
3462 * @remarks No-long-jump zone!!!
3463 */
3464DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3465{
3466 NOREF(pMixedCtx);
3467
3468 int rc = VINF_SUCCESS;
3469 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_APIC_STATE))
3470 {
3471 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3472 && APICIsEnabled(pVCpu))
3473 {
3474 /*
3475 * Setup TPR shadowing.
3476 */
3477 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3478 {
3479 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3480
3481 bool fPendingIntr = false;
3482 uint8_t u8Tpr = 0;
3483 uint8_t u8PendingIntr = 0;
3484 rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3485 AssertRCReturn(rc, rc);
3486
3487 /*
3488 * If there are interrupts pending but masked by the TPR, instruct VT-x to cause a TPR-below-threshold VM-exit
3489 * when the guest lowers its TPR below the priority of the pending interrupt so we can deliver the interrupt.
3490 * If there are no interrupts pending, set threshold to 0 to not cause any TPR-below-threshold VM-exits.
3491 */
3492 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3493 uint32_t u32TprThreshold = 0;
3494 if (fPendingIntr)
3495 {
3496 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3497 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3498 const uint8_t u8TprPriority = u8Tpr >> 4;
3499 if (u8PendingPriority <= u8TprPriority)
3500 u32TprThreshold = u8PendingPriority;
3501 }
3502
3503 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3504 AssertRCReturn(rc, rc);
3505 }
3506 }
3507 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_APIC_STATE);
3508 }
3509
3510 return rc;
3511}
3512
3513
3514/**
3515 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3516 *
3517 * @returns Guest's interruptibility-state.
3518 * @param pVCpu The cross context virtual CPU structure.
3519 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3520 * out-of-sync. Make sure to update the required fields
3521 * before using them.
3522 *
3523 * @remarks No-long-jump zone!!!
3524 */
3525DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3526{
3527 /*
3528 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3529 */
3530 uint32_t uIntrState = 0;
3531 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3532 {
3533 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3534 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3535 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3536 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3537 {
3538 if (pMixedCtx->eflags.Bits.u1IF)
3539 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3540 else
3541 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3542 }
3543 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3544 {
3545 /*
3546 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3547 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3548 */
3549 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3550 }
3551 }
3552
3553 /*
3554 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3555 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3556 * setting this would block host-NMIs and IRET will not clear the blocking.
3557 *
3558 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3559 */
3560 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3561 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3562 {
3563 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3564 }
3565
3566 return uIntrState;
3567}
3568
3569
3570/**
3571 * Loads the guest's interruptibility-state into the guest-state area in the
3572 * VMCS.
3573 *
3574 * @returns VBox status code.
3575 * @param pVCpu The cross context virtual CPU structure.
3576 * @param uIntrState The interruptibility-state to set.
3577 */
3578static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3579{
3580 NOREF(pVCpu);
3581 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3582 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3583 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3584 AssertRC(rc);
3585 return rc;
3586}
3587
3588
3589/**
3590 * Loads the exception intercepts required for guest execution in the VMCS.
3591 *
3592 * @returns VBox status code.
3593 * @param pVCpu The cross context virtual CPU structure.
3594 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3595 * out-of-sync. Make sure to update the required fields
3596 * before using them.
3597 */
3598static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3599{
3600 NOREF(pMixedCtx);
3601 int rc = VINF_SUCCESS;
3602 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMM_GUEST_XCPT_INTERCEPTS))
3603 {
3604 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3605 if (pVCpu->hm.s.fGIMTrapXcptUD)
3606 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3607#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3608 else
3609 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3610#endif
3611
3612 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3613 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3614
3615 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3616 AssertRCReturn(rc, rc);
3617
3618 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMM_GUEST_XCPT_INTERCEPTS);
3619 Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3620 pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3621 }
3622 return rc;
3623}
3624
3625
3626/**
3627 * Loads the guest's RIP into the guest-state area in the VMCS.
3628 *
3629 * @returns VBox status code.
3630 * @param pVCpu The cross context virtual CPU structure.
3631 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3632 * out-of-sync. Make sure to update the required fields
3633 * before using them.
3634 *
3635 * @remarks No-long-jump zone!!!
3636 */
3637static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3638{
3639 int rc = VINF_SUCCESS;
3640 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3641 {
3642 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3643 AssertRCReturn(rc, rc);
3644
3645 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3646 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3647 HMCPU_CF_VALUE(pVCpu)));
3648 }
3649 return rc;
3650}
3651
3652
3653/**
3654 * Loads the guest's RSP into the guest-state area in the VMCS.
3655 *
3656 * @returns VBox status code.
3657 * @param pVCpu The cross context virtual CPU structure.
3658 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3659 * out-of-sync. Make sure to update the required fields
3660 * before using them.
3661 *
3662 * @remarks No-long-jump zone!!!
3663 */
3664static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3665{
3666 int rc = VINF_SUCCESS;
3667 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3668 {
3669 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3670 AssertRCReturn(rc, rc);
3671
3672 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3673 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3674 }
3675 return rc;
3676}
3677
3678
3679/**
3680 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3681 *
3682 * @returns VBox status code.
3683 * @param pVCpu The cross context virtual CPU structure.
3684 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3685 * out-of-sync. Make sure to update the required fields
3686 * before using them.
3687 *
3688 * @remarks No-long-jump zone!!!
3689 */
3690static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3691{
3692 int rc = VINF_SUCCESS;
3693 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3694 {
3695 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3696 Let us assert it as such and use 32-bit VMWRITE. */
3697 Assert(!(pMixedCtx->rflags.u64 >> 32));
3698 X86EFLAGS Eflags = pMixedCtx->eflags;
3699 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3700 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3701 * These will never be cleared/set, unless some other part of the VMM
3702 * code is buggy - in which case we're better of finding and fixing
3703 * those bugs than hiding them. */
3704 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3705 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3706 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3707 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3708
3709 /*
3710 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3711 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3712 */
3713 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3714 {
3715 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3716 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3717 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3718 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3719 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3720 }
3721
3722 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3723 AssertRCReturn(rc, rc);
3724
3725 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3726 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3727 }
3728 return rc;
3729}
3730
3731
3732/**
3733 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3734 *
3735 * @returns VBox status code.
3736 * @param pVCpu The cross context virtual CPU structure.
3737 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3738 * out-of-sync. Make sure to update the required fields
3739 * before using them.
3740 *
3741 * @remarks No-long-jump zone!!!
3742 */
3743DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3744{
3745 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3746 rc |= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3747 rc |= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3748 AssertRCReturn(rc, rc);
3749 return rc;
3750}
3751
3752
3753/**
3754 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3755 * CR0 is partially shared with the host and we have to consider the FPU bits.
3756 *
3757 * @returns VBox status code.
3758 * @param pVCpu The cross context virtual CPU structure.
3759 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3760 * out-of-sync. Make sure to update the required fields
3761 * before using them.
3762 *
3763 * @remarks No-long-jump zone!!!
3764 */
3765static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3766{
3767 /*
3768 * Guest CR0.
3769 * Guest FPU.
3770 */
3771 int rc = VINF_SUCCESS;
3772 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3773 {
3774 Assert(!(pMixedCtx->cr0 >> 32));
3775 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3776 PVM pVM = pVCpu->CTX_SUFF(pVM);
3777
3778 /* The guest's view (read access) of its CR0 is unblemished. */
3779 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3780 AssertRCReturn(rc, rc);
3781 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3782
3783 /* Setup VT-x's view of the guest CR0. */
3784 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3785 if (pVM->hm.s.fNestedPaging)
3786 {
3787 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3788 {
3789 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3790 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3791 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3792 }
3793 else
3794 {
3795 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3796 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3797 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3798 }
3799
3800 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3801 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3802 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3803
3804 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3805 AssertRCReturn(rc, rc);
3806 }
3807 else
3808 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3809
3810 /*
3811 * Guest FPU bits.
3812 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3813 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3814 */
3815 u32GuestCR0 |= X86_CR0_NE;
3816 bool fInterceptNM = false;
3817 if (CPUMIsGuestFPUStateActive(pVCpu))
3818 {
3819 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3820 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3821 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3822 }
3823 else
3824 {
3825 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3826 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3827 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3828 }
3829
3830 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3831 bool fInterceptMF = false;
3832 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3833 fInterceptMF = true;
3834
3835 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3836 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3837 {
3838 Assert(PDMVmmDevHeapIsEnabled(pVM));
3839 Assert(pVM->hm.s.vmx.pRealModeTSS);
3840 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3841 fInterceptNM = true;
3842 fInterceptMF = true;
3843 }
3844 else
3845 {
3846 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3847 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3848 }
3849 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMM_GUEST_XCPT_INTERCEPTS);
3850
3851 if (fInterceptNM)
3852 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3853 else
3854 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3855
3856 if (fInterceptMF)
3857 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3858 else
3859 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3860
3861 /* Additional intercepts for debugging, define these yourself explicitly. */
3862#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3863 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3864 | RT_BIT(X86_XCPT_BP)
3865 | RT_BIT(X86_XCPT_DE)
3866 | RT_BIT(X86_XCPT_NM)
3867 | RT_BIT(X86_XCPT_TS)
3868 | RT_BIT(X86_XCPT_UD)
3869 | RT_BIT(X86_XCPT_NP)
3870 | RT_BIT(X86_XCPT_SS)
3871 | RT_BIT(X86_XCPT_GP)
3872 | RT_BIT(X86_XCPT_PF)
3873 | RT_BIT(X86_XCPT_MF)
3874 ;
3875#elif defined(HMVMX_ALWAYS_TRAP_PF)
3876 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3877#endif
3878
3879 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3880
3881 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3882 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3883 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3884 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3885 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3886 else
3887 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3888
3889 u32GuestCR0 |= uSetCR0;
3890 u32GuestCR0 &= uZapCR0;
3891 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3892
3893 /* Write VT-x's view of the guest CR0 into the VMCS. */
3894 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3895 AssertRCReturn(rc, rc);
3896 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3897 uZapCR0));
3898
3899 /*
3900 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3901 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3902 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3903 */
3904 uint32_t u32CR0Mask = 0;
3905 u32CR0Mask = X86_CR0_PE
3906 | X86_CR0_NE
3907 | X86_CR0_WP
3908 | X86_CR0_PG
3909 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3910 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3911 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3912
3913 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3914 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3915 * and @bugref{6944}. */
3916#if 0
3917 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3918 u32CR0Mask &= ~X86_CR0_PE;
3919#endif
3920 if (pVM->hm.s.fNestedPaging)
3921 u32CR0Mask &= ~X86_CR0_WP;
3922
3923 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3924 if (fInterceptNM)
3925 {
3926 u32CR0Mask |= X86_CR0_TS
3927 | X86_CR0_MP;
3928 }
3929
3930 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3931 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3932 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3933 AssertRCReturn(rc, rc);
3934 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3935
3936 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3937 }
3938 return rc;
3939}
3940
3941
3942/**
3943 * Loads the guest control registers (CR3, CR4) into the guest-state area
3944 * in the VMCS.
3945 *
3946 * @returns VBox strict status code.
3947 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3948 * without unrestricted guest access and the VMMDev is not presently
3949 * mapped (e.g. EFI32).
3950 *
3951 * @param pVCpu The cross context virtual CPU structure.
3952 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3953 * out-of-sync. Make sure to update the required fields
3954 * before using them.
3955 *
3956 * @remarks No-long-jump zone!!!
3957 */
3958static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3959{
3960 int rc = VINF_SUCCESS;
3961 PVM pVM = pVCpu->CTX_SUFF(pVM);
3962
3963 /*
3964 * Guest CR2.
3965 * It's always loaded in the assembler code. Nothing to do here.
3966 */
3967
3968 /*
3969 * Guest CR3.
3970 */
3971 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3972 {
3973 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3974 if (pVM->hm.s.fNestedPaging)
3975 {
3976 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3977
3978 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3979 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3980 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3981 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3982
3983 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3984 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3985 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3986
3987 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3988 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3989 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3990 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3991 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3992 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3993 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3994
3995 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3996 AssertRCReturn(rc, rc);
3997 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3998
3999 if ( pVM->hm.s.vmx.fUnrestrictedGuest
4000 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
4001 {
4002 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
4003 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
4004 {
4005 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
4006 AssertRCReturn(rc, rc);
4007 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
4008 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
4009 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
4010 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
4011 AssertRCReturn(rc, rc);
4012 }
4013
4014 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
4015 have Unrestricted Execution to handle the guest when it's not using paging. */
4016 GCPhysGuestCR3 = pMixedCtx->cr3;
4017 }
4018 else
4019 {
4020 /*
4021 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
4022 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
4023 * EPT takes care of translating it to host-physical addresses.
4024 */
4025 RTGCPHYS GCPhys;
4026 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
4027
4028 /* We obtain it here every time as the guest could have relocated this PCI region. */
4029 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
4030 if (RT_SUCCESS(rc))
4031 { /* likely */ }
4032 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4033 {
4034 Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
4035 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4036 }
4037 else
4038 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4039
4040 GCPhysGuestCR3 = GCPhys;
4041 }
4042
4043 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
4044 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4045 }
4046 else
4047 {
4048 /* Non-nested paging case, just use the hypervisor's CR3. */
4049 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4050
4051 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
4052 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4053 }
4054 AssertRCReturn(rc, rc);
4055
4056 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
4057 }
4058
4059 /*
4060 * Guest CR4.
4061 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
4062 */
4063 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
4064 {
4065 Assert(!(pMixedCtx->cr4 >> 32));
4066 uint32_t u32GuestCR4 = pMixedCtx->cr4;
4067
4068 /* The guest's view of its CR4 is unblemished. */
4069 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4070 AssertRCReturn(rc, rc);
4071 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4072
4073 /* Setup VT-x's view of the guest CR4. */
4074 /*
4075 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4076 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4077 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4078 */
4079 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4080 {
4081 Assert(pVM->hm.s.vmx.pRealModeTSS);
4082 Assert(PDMVmmDevHeapIsEnabled(pVM));
4083 u32GuestCR4 &= ~X86_CR4_VME;
4084 }
4085
4086 if (pVM->hm.s.fNestedPaging)
4087 {
4088 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4089 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4090 {
4091 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4092 u32GuestCR4 |= X86_CR4_PSE;
4093 /* Our identity mapping is a 32-bit page directory. */
4094 u32GuestCR4 &= ~X86_CR4_PAE;
4095 }
4096 /* else use guest CR4.*/
4097 }
4098 else
4099 {
4100 /*
4101 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4102 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4103 */
4104 switch (pVCpu->hm.s.enmShadowMode)
4105 {
4106 case PGMMODE_REAL: /* Real-mode. */
4107 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4108 case PGMMODE_32_BIT: /* 32-bit paging. */
4109 {
4110 u32GuestCR4 &= ~X86_CR4_PAE;
4111 break;
4112 }
4113
4114 case PGMMODE_PAE: /* PAE paging. */
4115 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4116 {
4117 u32GuestCR4 |= X86_CR4_PAE;
4118 break;
4119 }
4120
4121 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4122 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4123#ifdef VBOX_ENABLE_64_BITS_GUESTS
4124 break;
4125#endif
4126 default:
4127 AssertFailed();
4128 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4129 }
4130 }
4131
4132 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4133 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4134 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4135 u32GuestCR4 |= uSetCR4;
4136 u32GuestCR4 &= uZapCR4;
4137
4138 /* Write VT-x's view of the guest CR4 into the VMCS. */
4139 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4140 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4141 AssertRCReturn(rc, rc);
4142
4143 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4144 uint32_t u32CR4Mask = X86_CR4_VME
4145 | X86_CR4_PAE
4146 | X86_CR4_PGE
4147 | X86_CR4_PSE
4148 | X86_CR4_VMXE;
4149 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4150 u32CR4Mask |= X86_CR4_OSXSAVE;
4151 if (pVM->cpum.ro.GuestFeatures.fPcid)
4152 u32CR4Mask |= X86_CR4_PCIDE;
4153 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4154 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4155 AssertRCReturn(rc, rc);
4156
4157 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4158 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4159
4160 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4161 }
4162 return rc;
4163}
4164
4165
4166/**
4167 * Loads the guest debug registers into the guest-state area in the VMCS.
4168 *
4169 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4170 *
4171 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4172 *
4173 * @returns VBox status code.
4174 * @param pVCpu The cross context virtual CPU structure.
4175 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4176 * out-of-sync. Make sure to update the required fields
4177 * before using them.
4178 *
4179 * @remarks No-long-jump zone!!!
4180 */
4181static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4182{
4183 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4184 return VINF_SUCCESS;
4185
4186#ifdef VBOX_STRICT
4187 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4188 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4189 {
4190 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4191 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4192 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4193 }
4194#endif
4195
4196 int rc;
4197 PVM pVM = pVCpu->CTX_SUFF(pVM);
4198 bool fSteppingDB = false;
4199 bool fInterceptMovDRx = false;
4200 if (pVCpu->hm.s.fSingleInstruction)
4201 {
4202 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4203 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4204 {
4205 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4206 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4207 AssertRCReturn(rc, rc);
4208 Assert(fSteppingDB == false);
4209 }
4210 else
4211 {
4212 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4213 pVCpu->hm.s.fClearTrapFlag = true;
4214 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4215 fSteppingDB = true;
4216 }
4217 }
4218
4219 if ( fSteppingDB
4220 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4221 {
4222 /*
4223 * Use the combined guest and host DRx values found in the hypervisor
4224 * register set because the debugger has breakpoints active or someone
4225 * is single stepping on the host side without a monitor trap flag.
4226 *
4227 * Note! DBGF expects a clean DR6 state before executing guest code.
4228 */
4229#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4230 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4231 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4232 {
4233 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4234 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4235 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4236 }
4237 else
4238#endif
4239 if (!CPUMIsHyperDebugStateActive(pVCpu))
4240 {
4241 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4242 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4243 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4244 }
4245
4246 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4247 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4248 AssertRCReturn(rc, rc);
4249
4250 pVCpu->hm.s.fUsingHyperDR7 = true;
4251 fInterceptMovDRx = true;
4252 }
4253 else
4254 {
4255 /*
4256 * If the guest has enabled debug registers, we need to load them prior to
4257 * executing guest code so they'll trigger at the right time.
4258 */
4259 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4260 {
4261#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4262 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4263 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4264 {
4265 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4266 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4267 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4268 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4269 }
4270 else
4271#endif
4272 if (!CPUMIsGuestDebugStateActive(pVCpu))
4273 {
4274 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4275 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4276 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4277 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4278 }
4279 Assert(!fInterceptMovDRx);
4280 }
4281 /*
4282 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4283 * must intercept #DB in order to maintain a correct DR6 guest value, and
4284 * because we need to intercept it to prevent nested #DBs from hanging the
4285 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4286 */
4287#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4288 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4289 && !CPUMIsGuestDebugStateActive(pVCpu))
4290#else
4291 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4292#endif
4293 {
4294 fInterceptMovDRx = true;
4295 }
4296
4297 /* Update guest DR7. */
4298 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4299 AssertRCReturn(rc, rc);
4300
4301 pVCpu->hm.s.fUsingHyperDR7 = false;
4302 }
4303
4304 /*
4305 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4306 */
4307 if (fInterceptMovDRx)
4308 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4309 else
4310 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4311 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4312 AssertRCReturn(rc, rc);
4313
4314 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4315 return VINF_SUCCESS;
4316}
4317
4318
4319#ifdef VBOX_STRICT
4320/**
4321 * Strict function to validate segment registers.
4322 *
4323 * @remarks ASSUMES CR0 is up to date.
4324 */
4325static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4326{
4327 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4328 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4329 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4330 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4331 && ( !CPUMIsGuestInRealModeEx(pCtx)
4332 && !CPUMIsGuestInV86ModeEx(pCtx)))
4333 {
4334 /* Protected mode checks */
4335 /* CS */
4336 Assert(pCtx->cs.Attr.n.u1Present);
4337 Assert(!(pCtx->cs.Attr.u & 0xf00));
4338 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4339 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4340 || !(pCtx->cs.Attr.n.u1Granularity));
4341 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4342 || (pCtx->cs.Attr.n.u1Granularity));
4343 /* CS cannot be loaded with NULL in protected mode. */
4344 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4345 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4346 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4347 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4348 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4349 else
4350 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4351 /* SS */
4352 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4353 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4354 if ( !(pCtx->cr0 & X86_CR0_PE)
4355 || pCtx->cs.Attr.n.u4Type == 3)
4356 {
4357 Assert(!pCtx->ss.Attr.n.u2Dpl);
4358 }
4359 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4360 {
4361 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4362 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4363 Assert(pCtx->ss.Attr.n.u1Present);
4364 Assert(!(pCtx->ss.Attr.u & 0xf00));
4365 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4366 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4367 || !(pCtx->ss.Attr.n.u1Granularity));
4368 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4369 || (pCtx->ss.Attr.n.u1Granularity));
4370 }
4371 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4372 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4373 {
4374 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4375 Assert(pCtx->ds.Attr.n.u1Present);
4376 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4377 Assert(!(pCtx->ds.Attr.u & 0xf00));
4378 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4379 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4380 || !(pCtx->ds.Attr.n.u1Granularity));
4381 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4382 || (pCtx->ds.Attr.n.u1Granularity));
4383 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4384 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4385 }
4386 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4387 {
4388 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4389 Assert(pCtx->es.Attr.n.u1Present);
4390 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4391 Assert(!(pCtx->es.Attr.u & 0xf00));
4392 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4393 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4394 || !(pCtx->es.Attr.n.u1Granularity));
4395 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4396 || (pCtx->es.Attr.n.u1Granularity));
4397 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4398 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4399 }
4400 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4401 {
4402 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4403 Assert(pCtx->fs.Attr.n.u1Present);
4404 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4405 Assert(!(pCtx->fs.Attr.u & 0xf00));
4406 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4407 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4408 || !(pCtx->fs.Attr.n.u1Granularity));
4409 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4410 || (pCtx->fs.Attr.n.u1Granularity));
4411 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4412 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4413 }
4414 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4415 {
4416 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4417 Assert(pCtx->gs.Attr.n.u1Present);
4418 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4419 Assert(!(pCtx->gs.Attr.u & 0xf00));
4420 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4421 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4422 || !(pCtx->gs.Attr.n.u1Granularity));
4423 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4424 || (pCtx->gs.Attr.n.u1Granularity));
4425 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4426 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4427 }
4428 /* 64-bit capable CPUs. */
4429# if HC_ARCH_BITS == 64
4430 Assert(!(pCtx->cs.u64Base >> 32));
4431 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4432 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4433 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4434# endif
4435 }
4436 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4437 || ( CPUMIsGuestInRealModeEx(pCtx)
4438 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4439 {
4440 /* Real and v86 mode checks. */
4441 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4442 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4443 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4444 {
4445 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4446 }
4447 else
4448 {
4449 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4450 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4451 }
4452
4453 /* CS */
4454 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4455 Assert(pCtx->cs.u32Limit == 0xffff);
4456 Assert(u32CSAttr == 0xf3);
4457 /* SS */
4458 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4459 Assert(pCtx->ss.u32Limit == 0xffff);
4460 Assert(u32SSAttr == 0xf3);
4461 /* DS */
4462 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4463 Assert(pCtx->ds.u32Limit == 0xffff);
4464 Assert(u32DSAttr == 0xf3);
4465 /* ES */
4466 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4467 Assert(pCtx->es.u32Limit == 0xffff);
4468 Assert(u32ESAttr == 0xf3);
4469 /* FS */
4470 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4471 Assert(pCtx->fs.u32Limit == 0xffff);
4472 Assert(u32FSAttr == 0xf3);
4473 /* GS */
4474 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4475 Assert(pCtx->gs.u32Limit == 0xffff);
4476 Assert(u32GSAttr == 0xf3);
4477 /* 64-bit capable CPUs. */
4478# if HC_ARCH_BITS == 64
4479 Assert(!(pCtx->cs.u64Base >> 32));
4480 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4481 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4482 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4483# endif
4484 }
4485}
4486#endif /* VBOX_STRICT */
4487
4488
4489/**
4490 * Writes a guest segment register into the guest-state area in the VMCS.
4491 *
4492 * @returns VBox status code.
4493 * @param pVCpu The cross context virtual CPU structure.
4494 * @param idxSel Index of the selector in the VMCS.
4495 * @param idxLimit Index of the segment limit in the VMCS.
4496 * @param idxBase Index of the segment base in the VMCS.
4497 * @param idxAccess Index of the access rights of the segment in the VMCS.
4498 * @param pSelReg Pointer to the segment selector.
4499 *
4500 * @remarks No-long-jump zone!!!
4501 */
4502static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4503 uint32_t idxAccess, PCPUMSELREG pSelReg)
4504{
4505 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4506 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4507 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4508 AssertRCReturn(rc, rc);
4509
4510 uint32_t u32Access = pSelReg->Attr.u;
4511 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4512 {
4513 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4514 u32Access = 0xf3;
4515 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4516 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4517 }
4518 else
4519 {
4520 /*
4521 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4522 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4523 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4524 * loaded in protected-mode have their attribute as 0.
4525 */
4526 if (!u32Access)
4527 u32Access = X86DESCATTR_UNUSABLE;
4528 }
4529
4530 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4531 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4532 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4533
4534 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4535 AssertRCReturn(rc, rc);
4536 return rc;
4537}
4538
4539
4540/**
4541 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4542 * into the guest-state area in the VMCS.
4543 *
4544 * @returns VBox status code.
4545 * @param pVCpu The cross context virtual CPU structure.
4546 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4547 * out-of-sync. Make sure to update the required fields
4548 * before using them.
4549 *
4550 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4551 * @remarks No-long-jump zone!!!
4552 */
4553static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4554{
4555 int rc = VERR_INTERNAL_ERROR_5;
4556 PVM pVM = pVCpu->CTX_SUFF(pVM);
4557
4558 /*
4559 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4560 */
4561 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4562 {
4563 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4564 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4565 {
4566 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4567 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4568 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4569 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4570 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4571 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4572 }
4573
4574#ifdef VBOX_WITH_REM
4575 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4576 {
4577 Assert(pVM->hm.s.vmx.pRealModeTSS);
4578 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4579 if ( pVCpu->hm.s.vmx.fWasInRealMode
4580 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4581 {
4582 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4583 in real-mode (e.g. OpenBSD 4.0) */
4584 REMFlushTBs(pVM);
4585 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4586 pVCpu->hm.s.vmx.fWasInRealMode = false;
4587 }
4588 }
4589#endif
4590 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4591 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4592 AssertRCReturn(rc, rc);
4593 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4594 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4595 AssertRCReturn(rc, rc);
4596 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4597 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4598 AssertRCReturn(rc, rc);
4599 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4600 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4601 AssertRCReturn(rc, rc);
4602 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4603 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4604 AssertRCReturn(rc, rc);
4605 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4606 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4607 AssertRCReturn(rc, rc);
4608
4609#ifdef VBOX_STRICT
4610 /* Validate. */
4611 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4612#endif
4613
4614 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4615 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4616 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4617 }
4618
4619 /*
4620 * Guest TR.
4621 */
4622 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4623 {
4624 /*
4625 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4626 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4627 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4628 */
4629 uint16_t u16Sel = 0;
4630 uint32_t u32Limit = 0;
4631 uint64_t u64Base = 0;
4632 uint32_t u32AccessRights = 0;
4633
4634 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4635 {
4636 u16Sel = pMixedCtx->tr.Sel;
4637 u32Limit = pMixedCtx->tr.u32Limit;
4638 u64Base = pMixedCtx->tr.u64Base;
4639 u32AccessRights = pMixedCtx->tr.Attr.u;
4640 }
4641 else
4642 {
4643 Assert(pVM->hm.s.vmx.pRealModeTSS);
4644 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4645
4646 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4647 RTGCPHYS GCPhys;
4648 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4649 AssertRCReturn(rc, rc);
4650
4651 X86DESCATTR DescAttr;
4652 DescAttr.u = 0;
4653 DescAttr.n.u1Present = 1;
4654 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4655
4656 u16Sel = 0;
4657 u32Limit = HM_VTX_TSS_SIZE;
4658 u64Base = GCPhys; /* in real-mode phys = virt. */
4659 u32AccessRights = DescAttr.u;
4660 }
4661
4662 /* Validate. */
4663 Assert(!(u16Sel & RT_BIT(2)));
4664 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4665 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4666 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4667 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4668 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4669 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4670 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4671 Assert( (u32Limit & 0xfff) == 0xfff
4672 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4673 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4674 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4675
4676 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4677 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4678 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4679 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4680 AssertRCReturn(rc, rc);
4681
4682 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4683 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4684 }
4685
4686 /*
4687 * Guest GDTR.
4688 */
4689 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4690 {
4691 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4692 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4693 AssertRCReturn(rc, rc);
4694
4695 /* Validate. */
4696 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4697
4698 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4699 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4700 }
4701
4702 /*
4703 * Guest LDTR.
4704 */
4705 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4706 {
4707 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4708 uint32_t u32Access = 0;
4709 if (!pMixedCtx->ldtr.Attr.u)
4710 u32Access = X86DESCATTR_UNUSABLE;
4711 else
4712 u32Access = pMixedCtx->ldtr.Attr.u;
4713
4714 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4715 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4716 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4717 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4718 AssertRCReturn(rc, rc);
4719
4720 /* Validate. */
4721 if (!(u32Access & X86DESCATTR_UNUSABLE))
4722 {
4723 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4724 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4725 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4726 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4727 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4728 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4729 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4730 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4731 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4732 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4733 }
4734
4735 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4736 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4737 }
4738
4739 /*
4740 * Guest IDTR.
4741 */
4742 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4743 {
4744 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4745 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4746 AssertRCReturn(rc, rc);
4747
4748 /* Validate. */
4749 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4750
4751 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4752 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4753 }
4754
4755 return VINF_SUCCESS;
4756}
4757
4758
4759/**
4760 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4761 * areas.
4762 *
4763 * These MSRs will automatically be loaded to the host CPU on every successful
4764 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4765 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4766 * -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4767 *
4768 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4769 *
4770 * @returns VBox status code.
4771 * @param pVCpu The cross context virtual CPU structure.
4772 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4773 * out-of-sync. Make sure to update the required fields
4774 * before using them.
4775 *
4776 * @remarks No-long-jump zone!!!
4777 */
4778static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4779{
4780 AssertPtr(pVCpu);
4781 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4782
4783 /*
4784 * MSRs that we use the auto-load/store MSR area in the VMCS.
4785 */
4786 PVM pVM = pVCpu->CTX_SUFF(pVM);
4787 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4788 {
4789 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4790#if HC_ARCH_BITS == 32
4791 if (pVM->hm.s.fAllow64BitGuests)
4792 {
4793 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4794 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4795 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4796 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4797 AssertRCReturn(rc, rc);
4798# ifdef LOG_ENABLED
4799 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4800 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4801 {
4802 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4803 pMsr->u64Value));
4804 }
4805# endif
4806 }
4807#endif
4808 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4809 }
4810
4811 /*
4812 * Guest Sysenter MSRs.
4813 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4814 * VM-exits on WRMSRs for these MSRs.
4815 */
4816 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4817 {
4818 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4819 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4820 }
4821
4822 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4823 {
4824 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4825 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4826 }
4827
4828 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4829 {
4830 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4831 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4832 }
4833
4834 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4835 {
4836 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4837 {
4838 /*
4839 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4840 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4841 */
4842 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4843 {
4844 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4845 AssertRCReturn(rc,rc);
4846 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4847 }
4848 else
4849 {
4850 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4851 NULL /* pfAddedAndUpdated */);
4852 AssertRCReturn(rc, rc);
4853
4854 /* We need to intercept reads too, see @bugref{7386#c16}. */
4855 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4856 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4857 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4858 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4859 }
4860 }
4861 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4862 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4863 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4864 }
4865
4866 return VINF_SUCCESS;
4867}
4868
4869
4870/**
4871 * Loads the guest activity state into the guest-state area in the VMCS.
4872 *
4873 * @returns VBox status code.
4874 * @param pVCpu The cross context virtual CPU structure.
4875 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4876 * out-of-sync. Make sure to update the required fields
4877 * before using them.
4878 *
4879 * @remarks No-long-jump zone!!!
4880 */
4881static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4882{
4883 NOREF(pMixedCtx);
4884 /** @todo See if we can make use of other states, e.g.
4885 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4886 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4887 {
4888 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4889 AssertRCReturn(rc, rc);
4890
4891 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4892 }
4893 return VINF_SUCCESS;
4894}
4895
4896
4897#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4898/**
4899 * Check if guest state allows safe use of 32-bit switcher again.
4900 *
4901 * Segment bases and protected mode structures must be 32-bit addressable
4902 * because the 32-bit switcher will ignore high dword when writing these VMCS
4903 * fields. See @bugref{8432} for details.
4904 *
4905 * @returns true if safe, false if must continue to use the 64-bit switcher.
4906 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4907 * out-of-sync. Make sure to update the required fields
4908 * before using them.
4909 *
4910 * @remarks No-long-jump zone!!!
4911 */
4912static bool hmR0VmxIs32BitSwitcherSafe(PCPUMCTX pMixedCtx)
4913{
4914 if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000))
4915 return false;
4916 if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000))
4917 return false;
4918 if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000))
4919 return false;
4920 if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000))
4921 return false;
4922 if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000))
4923 return false;
4924 if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000))
4925 return false;
4926 if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000))
4927 return false;
4928 if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000))
4929 return false;
4930 if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000))
4931 return false;
4932 if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000))
4933 return false;
4934 /* All good, bases are 32-bit. */
4935 return true;
4936}
4937#endif
4938
4939
4940/**
4941 * Sets up the appropriate function to run guest code.
4942 *
4943 * @returns VBox status code.
4944 * @param pVCpu The cross context virtual CPU structure.
4945 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4946 * out-of-sync. Make sure to update the required fields
4947 * before using them.
4948 *
4949 * @remarks No-long-jump zone!!!
4950 */
4951static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4952{
4953 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4954 {
4955#ifndef VBOX_ENABLE_64_BITS_GUESTS
4956 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4957#endif
4958 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4959#if HC_ARCH_BITS == 32
4960 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4961 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4962 {
4963 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4964 {
4965 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4966 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4967 | HM_CHANGED_VMX_ENTRY_CTLS
4968 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4969 }
4970 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4971
4972 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4973 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4974 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4975 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 64-bit switcher\n", pVCpu->idCpu));
4976 }
4977#else
4978 /* 64-bit host. */
4979 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4980#endif
4981 }
4982 else
4983 {
4984 /* Guest is not in long mode, use the 32-bit handler. */
4985#if HC_ARCH_BITS == 32
4986 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4987 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4988 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4989 {
4990 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4991 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4992 | HM_CHANGED_VMX_ENTRY_CTLS
4993 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4994 }
4995# ifdef VBOX_ENABLE_64_BITS_GUESTS
4996 /*
4997 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel design, see @bugref{8432#c7}.
4998 * If real-on-v86 mode is active, clear the 64-bit switcher flag because now we know the guest is in a sane
4999 * state where it's safe to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
5000 * the much faster 32-bit switcher again.
5001 */
5002 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
5003 {
5004 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
5005 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher\n", pVCpu->idCpu));
5006 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5007 }
5008 else
5009 {
5010 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
5011 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
5012 || hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
5013 {
5014 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
5015 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5016 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR
5017 | HM_CHANGED_VMX_ENTRY_CTLS
5018 | HM_CHANGED_VMX_EXIT_CTLS
5019 | HM_CHANGED_HOST_CONTEXT);
5020 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher (safe)\n", pVCpu->idCpu));
5021 }
5022 }
5023# else
5024 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5025# endif
5026#else
5027 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5028#endif
5029 }
5030 Assert(pVCpu->hm.s.vmx.pfnStartVM);
5031 return VINF_SUCCESS;
5032}
5033
5034
5035/**
5036 * Wrapper for running the guest code in VT-x.
5037 *
5038 * @returns VBox status code, no informational status codes.
5039 * @param pVM The cross context VM structure.
5040 * @param pVCpu The cross context virtual CPU structure.
5041 * @param pCtx Pointer to the guest-CPU context.
5042 *
5043 * @remarks No-long-jump zone!!!
5044 */
5045DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5046{
5047 /*
5048 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
5049 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
5050 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
5051 */
5052 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
5053 /** @todo Add stats for resume vs launch. */
5054#ifdef VBOX_WITH_KERNEL_USING_XMM
5055 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
5056#else
5057 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
5058#endif
5059 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
5060 return rc;
5061}
5062
5063
5064/**
5065 * Reports world-switch error and dumps some useful debug info.
5066 *
5067 * @param pVM The cross context VM structure.
5068 * @param pVCpu The cross context virtual CPU structure.
5069 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5070 * @param pCtx Pointer to the guest-CPU context.
5071 * @param pVmxTransient Pointer to the VMX transient structure (only
5072 * exitReason updated).
5073 */
5074static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
5075{
5076 Assert(pVM);
5077 Assert(pVCpu);
5078 Assert(pCtx);
5079 Assert(pVmxTransient);
5080 HMVMX_ASSERT_PREEMPT_SAFE();
5081
5082 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
5083 switch (rcVMRun)
5084 {
5085 case VERR_VMX_INVALID_VMXON_PTR:
5086 AssertFailed();
5087 break;
5088 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5089 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5090 {
5091 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5092 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5093 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5094 AssertRC(rc);
5095
5096 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5097 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5098 Cannot do it here as we may have been long preempted. */
5099
5100#ifdef VBOX_STRICT
5101 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5102 pVmxTransient->uExitReason));
5103 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5104 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5105 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5106 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5107 else
5108 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5109 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5110 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5111
5112 /* VMX control bits. */
5113 uint32_t u32Val;
5114 uint64_t u64Val;
5115 RTHCUINTREG uHCReg;
5116 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5117 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5118 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5119 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5120 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5121 {
5122 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5123 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5124 }
5125 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5126 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5127 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5128 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5129 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5130 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5131 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5132 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5133 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5134 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5135 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5136 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5137 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5138 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5139 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5140 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5141 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5142 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5143 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5144 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5145 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5146 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5147 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5148 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5149 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5150 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5151 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5152 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5153 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5154 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5155 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5156 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5157 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5158 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5159 if (pVM->hm.s.fNestedPaging)
5160 {
5161 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5162 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5163 }
5164
5165 /* Guest bits. */
5166 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5167 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5168 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5169 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5170 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5171 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5172 if (pVM->hm.s.vmx.fVpid)
5173 {
5174 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5175 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5176 }
5177
5178 /* Host bits. */
5179 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5180 Log4(("Host CR0 %#RHr\n", uHCReg));
5181 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5182 Log4(("Host CR3 %#RHr\n", uHCReg));
5183 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5184 Log4(("Host CR4 %#RHr\n", uHCReg));
5185
5186 RTGDTR HostGdtr;
5187 PCX86DESCHC pDesc;
5188 ASMGetGDTR(&HostGdtr);
5189 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5190 Log4(("Host CS %#08x\n", u32Val));
5191 if (u32Val < HostGdtr.cbGdt)
5192 {
5193 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5194 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5195 }
5196
5197 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5198 Log4(("Host DS %#08x\n", u32Val));
5199 if (u32Val < HostGdtr.cbGdt)
5200 {
5201 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5202 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5203 }
5204
5205 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5206 Log4(("Host ES %#08x\n", u32Val));
5207 if (u32Val < HostGdtr.cbGdt)
5208 {
5209 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5210 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5211 }
5212
5213 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5214 Log4(("Host FS %#08x\n", u32Val));
5215 if (u32Val < HostGdtr.cbGdt)
5216 {
5217 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5218 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5219 }
5220
5221 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5222 Log4(("Host GS %#08x\n", u32Val));
5223 if (u32Val < HostGdtr.cbGdt)
5224 {
5225 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5226 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5227 }
5228
5229 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5230 Log4(("Host SS %#08x\n", u32Val));
5231 if (u32Val < HostGdtr.cbGdt)
5232 {
5233 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5234 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5235 }
5236
5237 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5238 Log4(("Host TR %#08x\n", u32Val));
5239 if (u32Val < HostGdtr.cbGdt)
5240 {
5241 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5242 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5243 }
5244
5245 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5246 Log4(("Host TR Base %#RHv\n", uHCReg));
5247 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5248 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5249 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5250 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5251 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5252 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5253 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5254 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5255 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5256 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5257 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5258 Log4(("Host RSP %#RHv\n", uHCReg));
5259 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5260 Log4(("Host RIP %#RHv\n", uHCReg));
5261# if HC_ARCH_BITS == 64
5262 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5263 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5264 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5265 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5266 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5267 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5268# endif
5269#endif /* VBOX_STRICT */
5270 break;
5271 }
5272
5273 default:
5274 /* Impossible */
5275 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5276 break;
5277 }
5278 NOREF(pVM); NOREF(pCtx);
5279}
5280
5281
5282#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5283#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5284# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5285#endif
5286#ifdef VBOX_STRICT
5287static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5288{
5289 switch (idxField)
5290 {
5291 case VMX_VMCS_GUEST_RIP:
5292 case VMX_VMCS_GUEST_RSP:
5293 case VMX_VMCS_GUEST_SYSENTER_EIP:
5294 case VMX_VMCS_GUEST_SYSENTER_ESP:
5295 case VMX_VMCS_GUEST_GDTR_BASE:
5296 case VMX_VMCS_GUEST_IDTR_BASE:
5297 case VMX_VMCS_GUEST_CS_BASE:
5298 case VMX_VMCS_GUEST_DS_BASE:
5299 case VMX_VMCS_GUEST_ES_BASE:
5300 case VMX_VMCS_GUEST_FS_BASE:
5301 case VMX_VMCS_GUEST_GS_BASE:
5302 case VMX_VMCS_GUEST_SS_BASE:
5303 case VMX_VMCS_GUEST_LDTR_BASE:
5304 case VMX_VMCS_GUEST_TR_BASE:
5305 case VMX_VMCS_GUEST_CR3:
5306 return true;
5307 }
5308 return false;
5309}
5310
5311static bool hmR0VmxIsValidReadField(uint32_t idxField)
5312{
5313 switch (idxField)
5314 {
5315 /* Read-only fields. */
5316 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5317 return true;
5318 }
5319 /* Remaining readable fields should also be writable. */
5320 return hmR0VmxIsValidWriteField(idxField);
5321}
5322#endif /* VBOX_STRICT */
5323
5324
5325/**
5326 * Executes the specified handler in 64-bit mode.
5327 *
5328 * @returns VBox status code (no informational status codes).
5329 * @param pVM The cross context VM structure.
5330 * @param pVCpu The cross context virtual CPU structure.
5331 * @param pCtx Pointer to the guest CPU context.
5332 * @param enmOp The operation to perform.
5333 * @param cParams Number of parameters.
5334 * @param paParam Array of 32-bit parameters.
5335 */
5336VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5337 uint32_t cParams, uint32_t *paParam)
5338{
5339 NOREF(pCtx);
5340
5341 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5342 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5343 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5344 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5345
5346#ifdef VBOX_STRICT
5347 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5348 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5349
5350 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5351 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5352#endif
5353
5354 /* Disable interrupts. */
5355 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5356
5357#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5358 RTCPUID idHostCpu = RTMpCpuId();
5359 CPUMR0SetLApic(pVCpu, idHostCpu);
5360#endif
5361
5362 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5363 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5364
5365 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5366 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5367 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5368
5369 /* Leave VMX Root Mode. */
5370 VMXDisable();
5371
5372 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5373
5374 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5375 CPUMSetHyperEIP(pVCpu, enmOp);
5376 for (int i = (int)cParams - 1; i >= 0; i--)
5377 CPUMPushHyper(pVCpu, paParam[i]);
5378
5379 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5380
5381 /* Call the switcher. */
5382 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5383 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5384
5385 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5386 /* Make sure the VMX instructions don't cause #UD faults. */
5387 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5388
5389 /* Re-enter VMX Root Mode */
5390 int rc2 = VMXEnable(HCPhysCpuPage);
5391 if (RT_FAILURE(rc2))
5392 {
5393 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5394 ASMSetFlags(fOldEFlags);
5395 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5396 return rc2;
5397 }
5398
5399 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5400 AssertRC(rc2);
5401 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5402 Assert(!(ASMGetFlags() & X86_EFL_IF));
5403 ASMSetFlags(fOldEFlags);
5404 return rc;
5405}
5406
5407
5408/**
5409 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5410 * supporting 64-bit guests.
5411 *
5412 * @returns VBox status code.
5413 * @param fResume Whether to VMLAUNCH or VMRESUME.
5414 * @param pCtx Pointer to the guest-CPU context.
5415 * @param pCache Pointer to the VMCS cache.
5416 * @param pVM The cross context VM structure.
5417 * @param pVCpu The cross context virtual CPU structure.
5418 */
5419DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5420{
5421 NOREF(fResume);
5422
5423 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5424 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5425
5426#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5427 pCache->uPos = 1;
5428 pCache->interPD = PGMGetInterPaeCR3(pVM);
5429 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5430#endif
5431
5432#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5433 pCache->TestIn.HCPhysCpuPage = 0;
5434 pCache->TestIn.HCPhysVmcs = 0;
5435 pCache->TestIn.pCache = 0;
5436 pCache->TestOut.HCPhysVmcs = 0;
5437 pCache->TestOut.pCache = 0;
5438 pCache->TestOut.pCtx = 0;
5439 pCache->TestOut.eflags = 0;
5440#else
5441 NOREF(pCache);
5442#endif
5443
5444 uint32_t aParam[10];
5445 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5446 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5447 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5448 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5449 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5450 aParam[5] = 0;
5451 aParam[6] = VM_RC_ADDR(pVM, pVM);
5452 aParam[7] = 0;
5453 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5454 aParam[9] = 0;
5455
5456#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5457 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5458 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5459#endif
5460 int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5461
5462#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5463 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5464 Assert(pCtx->dr[4] == 10);
5465 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5466#endif
5467
5468#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5469 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5470 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5471 pVCpu->hm.s.vmx.HCPhysVmcs));
5472 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5473 pCache->TestOut.HCPhysVmcs));
5474 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5475 pCache->TestOut.pCache));
5476 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5477 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5478 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5479 pCache->TestOut.pCtx));
5480 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5481#endif
5482 return rc;
5483}
5484
5485
5486/**
5487 * Initialize the VMCS-Read cache.
5488 *
5489 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5490 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5491 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5492 * (those that have a 32-bit FULL & HIGH part).
5493 *
5494 * @returns VBox status code.
5495 * @param pVM The cross context VM structure.
5496 * @param pVCpu The cross context virtual CPU structure.
5497 */
5498static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5499{
5500#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5501{ \
5502 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5503 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5504 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5505 ++cReadFields; \
5506}
5507
5508 AssertPtr(pVM);
5509 AssertPtr(pVCpu);
5510 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5511 uint32_t cReadFields = 0;
5512
5513 /*
5514 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5515 * and serve to indicate exceptions to the rules.
5516 */
5517
5518 /* Guest-natural selector base fields. */
5519#if 0
5520 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5521 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5522 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5523#endif
5524 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5525 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5526 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5527 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5528 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5529 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5530 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5531 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5532 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5533 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5534 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5535 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5536#if 0
5537 /* Unused natural width guest-state fields. */
5538 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5539 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5540#endif
5541 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5542 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5543
5544 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5545#if 0
5546 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5547 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5548 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5549 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5550 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5551 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5552 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5553 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5554 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5555#endif
5556
5557 /* Natural width guest-state fields. */
5558 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5559#if 0
5560 /* Currently unused field. */
5561 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5562#endif
5563
5564 if (pVM->hm.s.fNestedPaging)
5565 {
5566 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5567 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5568 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5569 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5570 }
5571 else
5572 {
5573 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5574 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5575 }
5576
5577#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5578 return VINF_SUCCESS;
5579}
5580
5581
5582/**
5583 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5584 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5585 * darwin, running 64-bit guests).
5586 *
5587 * @returns VBox status code.
5588 * @param pVCpu The cross context virtual CPU structure.
5589 * @param idxField The VMCS field encoding.
5590 * @param u64Val 16, 32 or 64-bit value.
5591 */
5592VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5593{
5594 int rc;
5595 switch (idxField)
5596 {
5597 /*
5598 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5599 */
5600 /* 64-bit Control fields. */
5601 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5602 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5603 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5604 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5605 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5606 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5607 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5608 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5609 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5610 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5611 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5612 case VMX_VMCS64_CTRL_EPTP_FULL:
5613 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5614 /* 64-bit Guest-state fields. */
5615 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5616 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5617 case VMX_VMCS64_GUEST_PAT_FULL:
5618 case VMX_VMCS64_GUEST_EFER_FULL:
5619 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5620 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5621 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5622 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5623 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5624 /* 64-bit Host-state fields. */
5625 case VMX_VMCS64_HOST_PAT_FULL:
5626 case VMX_VMCS64_HOST_EFER_FULL:
5627 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5628 {
5629 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5630 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5631 break;
5632 }
5633
5634 /*
5635 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5636 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5637 */
5638 /* Natural-width Guest-state fields. */
5639 case VMX_VMCS_GUEST_CR3:
5640 case VMX_VMCS_GUEST_ES_BASE:
5641 case VMX_VMCS_GUEST_CS_BASE:
5642 case VMX_VMCS_GUEST_SS_BASE:
5643 case VMX_VMCS_GUEST_DS_BASE:
5644 case VMX_VMCS_GUEST_FS_BASE:
5645 case VMX_VMCS_GUEST_GS_BASE:
5646 case VMX_VMCS_GUEST_LDTR_BASE:
5647 case VMX_VMCS_GUEST_TR_BASE:
5648 case VMX_VMCS_GUEST_GDTR_BASE:
5649 case VMX_VMCS_GUEST_IDTR_BASE:
5650 case VMX_VMCS_GUEST_RSP:
5651 case VMX_VMCS_GUEST_RIP:
5652 case VMX_VMCS_GUEST_SYSENTER_ESP:
5653 case VMX_VMCS_GUEST_SYSENTER_EIP:
5654 {
5655 if (!(RT_HI_U32(u64Val)))
5656 {
5657 /* If this field is 64-bit, VT-x will zero out the top bits. */
5658 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5659 }
5660 else
5661 {
5662 /* Assert that only the 32->64 switcher case should ever come here. */
5663 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5664 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5665 }
5666 break;
5667 }
5668
5669 default:
5670 {
5671 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5672 rc = VERR_INVALID_PARAMETER;
5673 break;
5674 }
5675 }
5676 AssertRCReturn(rc, rc);
5677 return rc;
5678}
5679
5680
5681/**
5682 * Queue up a VMWRITE by using the VMCS write cache.
5683 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5684 *
5685 * @param pVCpu The cross context virtual CPU structure.
5686 * @param idxField The VMCS field encoding.
5687 * @param u64Val 16, 32 or 64-bit value.
5688 */
5689VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5690{
5691 AssertPtr(pVCpu);
5692 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5693
5694 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5695 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5696
5697 /* Make sure there are no duplicates. */
5698 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5699 {
5700 if (pCache->Write.aField[i] == idxField)
5701 {
5702 pCache->Write.aFieldVal[i] = u64Val;
5703 return VINF_SUCCESS;
5704 }
5705 }
5706
5707 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5708 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5709 pCache->Write.cValidEntries++;
5710 return VINF_SUCCESS;
5711}
5712#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5713
5714
5715/**
5716 * Sets up the usage of TSC-offsetting and updates the VMCS.
5717 *
5718 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5719 * VMX preemption timer.
5720 *
5721 * @returns VBox status code.
5722 * @param pVM The cross context VM structure.
5723 * @param pVCpu The cross context virtual CPU structure.
5724 *
5725 * @remarks No-long-jump zone!!!
5726 */
5727static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5728{
5729 int rc;
5730 bool fOffsettedTsc;
5731 bool fParavirtTsc;
5732 if (pVM->hm.s.vmx.fUsePreemptTimer)
5733 {
5734 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5735 &fOffsettedTsc, &fParavirtTsc);
5736
5737 /* Make sure the returned values have sane upper and lower boundaries. */
5738 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5739 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5740 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5741 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5742
5743 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5744 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5745 }
5746 else
5747 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5748
5749 /** @todo later optimize this to be done elsewhere and not before every
5750 * VM-entry. */
5751 if (fParavirtTsc)
5752 {
5753 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5754 information before every VM-entry, hence disable it for performance sake. */
5755#if 0
5756 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5757 AssertRC(rc);
5758#endif
5759 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5760 }
5761
5762 if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5763 {
5764 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5765 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5766
5767 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5768 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5769 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5770 }
5771 else
5772 {
5773 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5774 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5775 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5776 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5777 }
5778}
5779
5780
5781#ifdef HMVMX_USE_IEM_EVENT_REFLECTION
5782/**
5783 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5784 * VM-exit interruption info type.
5785 *
5786 * @returns The IEM exception flags.
5787 * @param uVector The event vector.
5788 * @param uVmxVectorType The VMX event type.
5789 *
5790 * @remarks This function currently only constructs flags required for
5791 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5792 * and CR2 aspects of an exception are not included).
5793 */
5794static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5795{
5796 uint32_t fIemXcptFlags;
5797 switch (uVmxVectorType)
5798 {
5799 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5800 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5801 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5802 break;
5803
5804 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5805 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5806 break;
5807
5808 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5809 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5810 break;
5811
5812 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5813 {
5814 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5815 if (uVector == X86_XCPT_BP)
5816 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5817 else if (uVector == X86_XCPT_OF)
5818 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5819 else
5820 {
5821 fIemXcptFlags = 0;
5822 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5823 }
5824 break;
5825 }
5826
5827 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5828 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5829 break;
5830
5831 default:
5832 fIemXcptFlags = 0;
5833 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5834 break;
5835 }
5836 return fIemXcptFlags;
5837}
5838
5839#else
5840/**
5841 * Determines if an exception is a contributory exception.
5842 *
5843 * Contributory exceptions are ones which can cause double-faults unless the
5844 * original exception was a benign exception. Page-fault is intentionally not
5845 * included here as it's a conditional contributory exception.
5846 *
5847 * @returns true if the exception is contributory, false otherwise.
5848 * @param uVector The exception vector.
5849 */
5850DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5851{
5852 switch (uVector)
5853 {
5854 case X86_XCPT_GP:
5855 case X86_XCPT_SS:
5856 case X86_XCPT_NP:
5857 case X86_XCPT_TS:
5858 case X86_XCPT_DE:
5859 return true;
5860 default:
5861 break;
5862 }
5863 return false;
5864}
5865#endif /* HMVMX_USE_IEM_EVENT_REFLECTION */
5866
5867
5868/**
5869 * Sets an event as a pending event to be injected into the guest.
5870 *
5871 * @param pVCpu The cross context virtual CPU structure.
5872 * @param u32IntInfo The VM-entry interruption-information field.
5873 * @param cbInstr The VM-entry instruction length in bytes (for software
5874 * interrupts, exceptions and privileged software
5875 * exceptions).
5876 * @param u32ErrCode The VM-entry exception error code.
5877 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5878 * page-fault.
5879 *
5880 * @remarks Statistics counter assumes this is a guest event being injected or
5881 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5882 * always incremented.
5883 */
5884DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5885 RTGCUINTPTR GCPtrFaultAddress)
5886{
5887 Assert(!pVCpu->hm.s.Event.fPending);
5888 pVCpu->hm.s.Event.fPending = true;
5889 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5890 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5891 pVCpu->hm.s.Event.cbInstr = cbInstr;
5892 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5893}
5894
5895
5896/**
5897 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5898 *
5899 * @param pVCpu The cross context virtual CPU structure.
5900 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5901 * out-of-sync. Make sure to update the required fields
5902 * before using them.
5903 */
5904DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5905{
5906 NOREF(pMixedCtx);
5907 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5908 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5909 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5910 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5911}
5912
5913
5914/**
5915 * Handle a condition that occurred while delivering an event through the guest
5916 * IDT.
5917 *
5918 * @returns Strict VBox status code (i.e. informational status codes too).
5919 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5920 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5921 * to continue execution of the guest which will delivery the \#DF.
5922 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5923 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5924 *
5925 * @param pVCpu The cross context virtual CPU structure.
5926 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5927 * out-of-sync. Make sure to update the required fields
5928 * before using them.
5929 * @param pVmxTransient Pointer to the VMX transient structure.
5930 *
5931 * @remarks No-long-jump zone!!!
5932 */
5933static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5934{
5935 uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5936
5937 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5938 rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5939
5940 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5941 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5942 {
5943 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5944 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5945#ifdef HMVMX_USE_IEM_EVENT_REFLECTION
5946 /*
5947 * If the event was a software interrupt (generated with INT n) or a software exception (generated
5948 * by INT3/INTO) or a privileged software exception (generated by INT1), we can handle the VM-exit
5949 * and continue guest execution which will re-execute the instruction rather than re-injecting the
5950 * exception, as that can cause premature trips to ring-3 before injection and involve TRPM which
5951 * currently has no way of storing that these exceptions were caused by these instructions
5952 * (ICEBP's #DB poses the problem).
5953 */
5954 IEMXCPTRAISE enmRaise;
5955 IEMXCPTRAISEINFO fRaiseInfo;
5956 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5957 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5958 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5959 {
5960 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5961 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5962 }
5963 else if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5964 {
5965 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5966 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5967 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5968 /** @todo Make AssertMsgReturn as just AssertMsg later. */
5969 AssertMsgReturn(uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT,
5970 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5971 uExitVectorType), VERR_VMX_IPE_5);
5972 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5973
5974 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5975 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
5976 {
5977 pVmxTransient->fVectoringPF = true;
5978 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5979 }
5980 }
5981 else
5982 {
5983 /*
5984 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5985 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5986 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
5987 */
5988 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5989 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5990 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5991 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5992 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5993 }
5994
5995 /*
5996 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5997 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5998 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5999 * subsequent VM-entry would fail.
6000 *
6001 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
6002 */
6003 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
6004 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6005 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
6006 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
6007 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6008 {
6009 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6010 }
6011
6012 switch (enmRaise)
6013 {
6014 case IEMXCPTRAISE_CURRENT_XCPT:
6015 {
6016 Log4(("IDT: vcpu[%RU32] Pending secondary xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n", pVCpu->idCpu,
6017 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
6018 Assert(rcStrict == VINF_SUCCESS);
6019 break;
6020 }
6021
6022 case IEMXCPTRAISE_PREV_EVENT:
6023 {
6024 uint32_t u32ErrCode;
6025 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6026 {
6027 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6028 AssertRCReturn(rc2, rc2);
6029 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6030 }
6031 else
6032 u32ErrCode = 0;
6033
6034 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
6035 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6036 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6037 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
6038
6039 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
6040 pVCpu->hm.s.Event.u32ErrCode));
6041 Assert(rcStrict == VINF_SUCCESS);
6042 break;
6043 }
6044
6045 case IEMXCPTRAISE_REEXEC_INSTR:
6046 Assert(rcStrict == VINF_SUCCESS);
6047 break;
6048
6049 case IEMXCPTRAISE_DOUBLE_FAULT:
6050 {
6051 /*
6052 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
6053 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
6054 */
6055 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6056 {
6057 pVmxTransient->fVectoringDoublePF = true;
6058 Log4(("IDT: vcpu[%RU32] Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
6059 pMixedCtx->cr2));
6060 rcStrict = VINF_SUCCESS;
6061 }
6062 else
6063 {
6064 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6065 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6066 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
6067 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
6068 rcStrict = VINF_HM_DOUBLE_FAULT;
6069 }
6070 break;
6071 }
6072
6073 case IEMXCPTRAISE_TRIPLE_FAULT:
6074 {
6075 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6076 uExitVector));
6077 rcStrict = VINF_EM_RESET;
6078 break;
6079 }
6080
6081 case IEMXCPTRAISE_CPU_HANG:
6082 {
6083 Log4(("IDT: vcpu[%RU32] Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", pVCpu->idCpu, fRaiseInfo));
6084 rcStrict = VERR_EM_GUEST_CPU_HANG;
6085 break;
6086 }
6087
6088 default:
6089 {
6090 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6091 rcStrict = VERR_VMX_IPE_2;
6092 break;
6093 }
6094 }
6095#else
6096 typedef enum
6097 {
6098 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
6099 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
6100 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
6101 VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
6102 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
6103 } VMXREFLECTXCPT;
6104
6105 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
6106 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
6107 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
6108 {
6109 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
6110 {
6111 enmReflect = VMXREFLECTXCPT_XCPT;
6112#ifdef VBOX_STRICT
6113 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
6114 && uExitVector == X86_XCPT_PF)
6115 {
6116 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6117 }
6118#endif
6119 if ( uExitVector == X86_XCPT_PF
6120 && uIdtVector == X86_XCPT_PF)
6121 {
6122 pVmxTransient->fVectoringDoublePF = true;
6123 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6124 }
6125 else if ( uExitVector == X86_XCPT_AC
6126 && uIdtVector == X86_XCPT_AC)
6127 {
6128 enmReflect = VMXREFLECTXCPT_HANG;
6129 Log4(("IDT: Nested #AC - Bad guest\n"));
6130 }
6131 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
6132 && hmR0VmxIsContributoryXcpt(uExitVector)
6133 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
6134 || uIdtVector == X86_XCPT_PF))
6135 {
6136 enmReflect = VMXREFLECTXCPT_DF;
6137 }
6138 else if (uIdtVector == X86_XCPT_DF)
6139 enmReflect = VMXREFLECTXCPT_TF;
6140 }
6141 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
6142 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
6143 {
6144 /*
6145 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
6146 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
6147 */
6148 enmReflect = VMXREFLECTXCPT_XCPT;
6149
6150 if (uExitVector == X86_XCPT_PF)
6151 {
6152 pVmxTransient->fVectoringPF = true;
6153 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6154 }
6155 }
6156 }
6157 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6158 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
6159 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
6160 {
6161 /*
6162 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
6163 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
6164 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
6165 */
6166 enmReflect = VMXREFLECTXCPT_XCPT;
6167 }
6168
6169 /*
6170 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
6171 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
6172 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
6173 *
6174 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
6175 */
6176 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6177 && enmReflect == VMXREFLECTXCPT_XCPT
6178 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
6179 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6180 {
6181 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6182 }
6183
6184 switch (enmReflect)
6185 {
6186 case VMXREFLECTXCPT_XCPT:
6187 {
6188 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6189 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6190 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
6191
6192 uint32_t u32ErrCode = 0;
6193 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6194 {
6195 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6196 AssertRCReturn(rc2, rc2);
6197 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6198 }
6199
6200 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
6201 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6202 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6203 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
6204 rcStrict = VINF_SUCCESS;
6205 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
6206 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
6207
6208 break;
6209 }
6210
6211 case VMXREFLECTXCPT_DF:
6212 {
6213 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6214 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6215 rcStrict = VINF_HM_DOUBLE_FAULT;
6216 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
6217 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
6218
6219 break;
6220 }
6221
6222 case VMXREFLECTXCPT_TF:
6223 {
6224 rcStrict = VINF_EM_RESET;
6225 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6226 uExitVector));
6227 break;
6228 }
6229
6230 case VMXREFLECTXCPT_HANG:
6231 {
6232 rcStrict = VERR_EM_GUEST_CPU_HANG;
6233 break;
6234 }
6235
6236 default:
6237 Assert(rcStrict == VINF_SUCCESS);
6238 break;
6239 }
6240#endif /* HMVMX_USE_IEM_EVENT_REFLECTION */
6241 }
6242 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6243 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6244 && uExitVector != X86_XCPT_DF
6245 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6246 {
6247 /*
6248 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6249 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6250 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6251 */
6252 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6253 {
6254 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
6255 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6256 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6257 }
6258 }
6259
6260 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
6261 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
6262 return rcStrict;
6263}
6264
6265
6266/**
6267 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
6268 *
6269 * @returns VBox status code.
6270 * @param pVCpu The cross context virtual CPU structure.
6271 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6272 * out-of-sync. Make sure to update the required fields
6273 * before using them.
6274 *
6275 * @remarks No-long-jump zone!!!
6276 */
6277static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6278{
6279 NOREF(pMixedCtx);
6280
6281 /*
6282 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
6283 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
6284 */
6285 VMMRZCallRing3Disable(pVCpu);
6286 HM_DISABLE_PREEMPT();
6287
6288 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
6289 {
6290#ifndef DEBUG_bird /** @todo this triggers running bs3-cpu-generated-1.img with --debug-command-line
6291 * and 'dbgc-init' containing:
6292 * sxe "xcpt_de"
6293 * sxe "xcpt_bp"
6294 * sxi "xcpt_gp"
6295 * sxi "xcpt_ss"
6296 * sxi "xcpt_np"
6297 */
6298 /** @todo r=ramshankar: Should be fixed after r119291. */
6299 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
6300#endif
6301 uint32_t uVal = 0;
6302 uint32_t uShadow = 0;
6303 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
6304 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
6305 AssertRCReturn(rc, rc);
6306
6307 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
6308 CPUMSetGuestCR0(pVCpu, uVal);
6309 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
6310 }
6311
6312 HM_RESTORE_PREEMPT();
6313 VMMRZCallRing3Enable(pVCpu);
6314 return VINF_SUCCESS;
6315}
6316
6317
6318/**
6319 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
6320 *
6321 * @returns VBox status code.
6322 * @param pVCpu The cross context virtual CPU structure.
6323 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6324 * out-of-sync. Make sure to update the required fields
6325 * before using them.
6326 *
6327 * @remarks No-long-jump zone!!!
6328 */
6329static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6330{
6331 NOREF(pMixedCtx);
6332
6333 int rc = VINF_SUCCESS;
6334 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
6335 {
6336 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4));
6337 uint32_t uVal = 0;
6338 uint32_t uShadow = 0;
6339 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
6340 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
6341 AssertRCReturn(rc, rc);
6342
6343 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
6344 CPUMSetGuestCR4(pVCpu, uVal);
6345 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6346 }
6347 return rc;
6348}
6349
6350
6351/**
6352 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
6353 *
6354 * @returns VBox status code.
6355 * @param pVCpu The cross context virtual CPU structure.
6356 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6357 * out-of-sync. Make sure to update the required fields
6358 * before using them.
6359 *
6360 * @remarks No-long-jump zone!!!
6361 */
6362static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6363{
6364 int rc = VINF_SUCCESS;
6365 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6366 {
6367 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP));
6368 uint64_t u64Val = 0;
6369 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6370 AssertRCReturn(rc, rc);
6371
6372 pMixedCtx->rip = u64Val;
6373 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6374 }
6375 return rc;
6376}
6377
6378
6379/**
6380 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6381 *
6382 * @returns VBox status code.
6383 * @param pVCpu The cross context virtual CPU structure.
6384 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6385 * out-of-sync. Make sure to update the required fields
6386 * before using them.
6387 *
6388 * @remarks No-long-jump zone!!!
6389 */
6390static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6391{
6392 int rc = VINF_SUCCESS;
6393 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6394 {
6395 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP));
6396 uint64_t u64Val = 0;
6397 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6398 AssertRCReturn(rc, rc);
6399
6400 pMixedCtx->rsp = u64Val;
6401 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6402 }
6403 return rc;
6404}
6405
6406
6407/**
6408 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6409 *
6410 * @returns VBox status code.
6411 * @param pVCpu The cross context virtual CPU structure.
6412 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6413 * out-of-sync. Make sure to update the required fields
6414 * before using them.
6415 *
6416 * @remarks No-long-jump zone!!!
6417 */
6418static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6419{
6420 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6421 {
6422 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS));
6423 uint32_t uVal = 0;
6424 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6425 AssertRCReturn(rc, rc);
6426
6427 pMixedCtx->eflags.u32 = uVal;
6428 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6429 {
6430 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6431 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6432
6433 pMixedCtx->eflags.Bits.u1VM = 0;
6434 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6435 }
6436
6437 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6438 }
6439 return VINF_SUCCESS;
6440}
6441
6442
6443/**
6444 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6445 * guest-CPU context.
6446 */
6447DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6448{
6449 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6450 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6451 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6452 return rc;
6453}
6454
6455
6456/**
6457 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6458 * from the guest-state area in the VMCS.
6459 *
6460 * @param pVCpu The cross context virtual CPU structure.
6461 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6462 * out-of-sync. Make sure to update the required fields
6463 * before using them.
6464 *
6465 * @remarks No-long-jump zone!!!
6466 */
6467static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6468{
6469 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6470 {
6471 uint32_t uIntrState = 0;
6472 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6473 AssertRC(rc);
6474
6475 if (!uIntrState)
6476 {
6477 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6478 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6479
6480 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6481 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6482 }
6483 else
6484 {
6485 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6486 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6487 {
6488 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6489 AssertRC(rc);
6490 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6491 AssertRC(rc);
6492
6493 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6494 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6495 }
6496 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6497 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6498
6499 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6500 {
6501 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6502 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6503 }
6504 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6505 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6506 }
6507
6508 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6509 }
6510}
6511
6512
6513/**
6514 * Saves the guest's activity state.
6515 *
6516 * @returns VBox status code.
6517 * @param pVCpu The cross context virtual CPU structure.
6518 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6519 * out-of-sync. Make sure to update the required fields
6520 * before using them.
6521 *
6522 * @remarks No-long-jump zone!!!
6523 */
6524static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6525{
6526 NOREF(pMixedCtx);
6527 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6528 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6529 return VINF_SUCCESS;
6530}
6531
6532
6533/**
6534 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6535 * the current VMCS into the guest-CPU context.
6536 *
6537 * @returns VBox status code.
6538 * @param pVCpu The cross context virtual CPU structure.
6539 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6540 * out-of-sync. Make sure to update the required fields
6541 * before using them.
6542 *
6543 * @remarks No-long-jump zone!!!
6544 */
6545static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6546{
6547 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6548 {
6549 Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR));
6550 uint32_t u32Val = 0;
6551 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6552 pMixedCtx->SysEnter.cs = u32Val;
6553 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6554 }
6555
6556 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6557 {
6558 Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR));
6559 uint64_t u64Val = 0;
6560 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6561 pMixedCtx->SysEnter.eip = u64Val;
6562 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6563 }
6564 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6565 {
6566 Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR));
6567 uint64_t u64Val = 0;
6568 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6569 pMixedCtx->SysEnter.esp = u64Val;
6570 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6571 }
6572 return VINF_SUCCESS;
6573}
6574
6575
6576/**
6577 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6578 * the CPU back into the guest-CPU context.
6579 *
6580 * @returns VBox status code.
6581 * @param pVCpu The cross context virtual CPU structure.
6582 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6583 * out-of-sync. Make sure to update the required fields
6584 * before using them.
6585 *
6586 * @remarks No-long-jump zone!!!
6587 */
6588static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6589{
6590 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6591 VMMRZCallRing3Disable(pVCpu);
6592 HM_DISABLE_PREEMPT();
6593
6594 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6595 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6596 {
6597 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMM_GUEST_LAZY_MSRS));
6598 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6599 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6600 }
6601
6602 HM_RESTORE_PREEMPT();
6603 VMMRZCallRing3Enable(pVCpu);
6604
6605 return VINF_SUCCESS;
6606}
6607
6608
6609/**
6610 * Saves the auto load/store'd guest MSRs from the current VMCS into
6611 * the guest-CPU context.
6612 *
6613 * @returns VBox status code.
6614 * @param pVCpu The cross context virtual CPU structure.
6615 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6616 * out-of-sync. Make sure to update the required fields
6617 * before using them.
6618 *
6619 * @remarks No-long-jump zone!!!
6620 */
6621static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6622{
6623 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6624 return VINF_SUCCESS;
6625
6626 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS));
6627 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6628 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6629 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6630 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6631 {
6632 switch (pMsr->u32Msr)
6633 {
6634 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6635 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6636 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6637 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6638 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6639 case MSR_IA32_SPEC_CTRL: CPUMR0SetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6640 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6641 break;
6642
6643 default:
6644 {
6645 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6646 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6647 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6648 }
6649 }
6650 }
6651
6652 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6653 return VINF_SUCCESS;
6654}
6655
6656
6657/**
6658 * Saves the guest control registers from the current VMCS into the guest-CPU
6659 * context.
6660 *
6661 * @returns VBox status code.
6662 * @param pVCpu The cross context virtual CPU structure.
6663 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6664 * out-of-sync. Make sure to update the required fields
6665 * before using them.
6666 *
6667 * @remarks No-long-jump zone!!!
6668 */
6669static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6670{
6671 /* Guest CR0. Guest FPU. */
6672 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6673 AssertRCReturn(rc, rc);
6674
6675 /* Guest CR4. */
6676 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6677 AssertRCReturn(rc, rc);
6678
6679 /* Guest CR2 - updated always during the world-switch or in #PF. */
6680 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6681 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6682 {
6683 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3));
6684 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6685 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6686
6687 PVM pVM = pVCpu->CTX_SUFF(pVM);
6688 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6689 || ( pVM->hm.s.fNestedPaging
6690 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6691 {
6692 uint64_t u64Val = 0;
6693 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6694 if (pMixedCtx->cr3 != u64Val)
6695 {
6696 CPUMSetGuestCR3(pVCpu, u64Val);
6697 if (VMMRZCallRing3IsEnabled(pVCpu))
6698 {
6699 PGMUpdateCR3(pVCpu, u64Val);
6700 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6701 }
6702 else
6703 {
6704 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6705 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6706 }
6707 }
6708
6709 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6710 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6711 {
6712 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6713 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6714 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6715 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6716 AssertRCReturn(rc, rc);
6717
6718 if (VMMRZCallRing3IsEnabled(pVCpu))
6719 {
6720 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6721 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6722 }
6723 else
6724 {
6725 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6726 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6727 }
6728 }
6729 }
6730
6731 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6732 }
6733
6734 /*
6735 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6736 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6737 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6738 *
6739 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6740 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6741 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6742 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6743 *
6744 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6745 */
6746 if (VMMRZCallRing3IsEnabled(pVCpu))
6747 {
6748 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6749 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6750
6751 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6752 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6753
6754 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6755 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6756 }
6757
6758 return rc;
6759}
6760
6761
6762/**
6763 * Reads a guest segment register from the current VMCS into the guest-CPU
6764 * context.
6765 *
6766 * @returns VBox status code.
6767 * @param pVCpu The cross context virtual CPU structure.
6768 * @param idxSel Index of the selector in the VMCS.
6769 * @param idxLimit Index of the segment limit in the VMCS.
6770 * @param idxBase Index of the segment base in the VMCS.
6771 * @param idxAccess Index of the access rights of the segment in the VMCS.
6772 * @param pSelReg Pointer to the segment selector.
6773 *
6774 * @remarks No-long-jump zone!!!
6775 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6776 * macro as that takes care of whether to read from the VMCS cache or
6777 * not.
6778 */
6779DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6780 PCPUMSELREG pSelReg)
6781{
6782 NOREF(pVCpu);
6783
6784 uint32_t u32Val = 0;
6785 int rc = VMXReadVmcs32(idxSel, &u32Val);
6786 AssertRCReturn(rc, rc);
6787 pSelReg->Sel = (uint16_t)u32Val;
6788 pSelReg->ValidSel = (uint16_t)u32Val;
6789 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6790
6791 rc = VMXReadVmcs32(idxLimit, &u32Val);
6792 AssertRCReturn(rc, rc);
6793 pSelReg->u32Limit = u32Val;
6794
6795 uint64_t u64Val = 0;
6796 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6797 AssertRCReturn(rc, rc);
6798 pSelReg->u64Base = u64Val;
6799
6800 rc = VMXReadVmcs32(idxAccess, &u32Val);
6801 AssertRCReturn(rc, rc);
6802 pSelReg->Attr.u = u32Val;
6803
6804 /*
6805 * If VT-x marks the segment as unusable, most other bits remain undefined:
6806 * - For CS the L, D and G bits have meaning.
6807 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6808 * - For the remaining data segments no bits are defined.
6809 *
6810 * The present bit and the unusable bit has been observed to be set at the
6811 * same time (the selector was supposed to be invalid as we started executing
6812 * a V8086 interrupt in ring-0).
6813 *
6814 * What should be important for the rest of the VBox code, is that the P bit is
6815 * cleared. Some of the other VBox code recognizes the unusable bit, but
6816 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6817 * safe side here, we'll strip off P and other bits we don't care about. If
6818 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6819 *
6820 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6821 */
6822 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6823 {
6824 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6825
6826 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6827 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6828 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6829
6830 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6831#ifdef DEBUG_bird
6832 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6833 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6834 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6835#endif
6836 }
6837 return VINF_SUCCESS;
6838}
6839
6840
6841#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6842# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6843 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6844 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6845#else
6846# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6847 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6848 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6849#endif
6850
6851
6852/**
6853 * Saves the guest segment registers from the current VMCS into the guest-CPU
6854 * context.
6855 *
6856 * @returns VBox status code.
6857 * @param pVCpu The cross context virtual CPU structure.
6858 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6859 * out-of-sync. Make sure to update the required fields
6860 * before using them.
6861 *
6862 * @remarks No-long-jump zone!!!
6863 */
6864static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6865{
6866 /* Guest segment registers. */
6867 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6868 {
6869 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS));
6870 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6871 AssertRCReturn(rc, rc);
6872
6873 rc = VMXLOCAL_READ_SEG(CS, cs);
6874 rc |= VMXLOCAL_READ_SEG(SS, ss);
6875 rc |= VMXLOCAL_READ_SEG(DS, ds);
6876 rc |= VMXLOCAL_READ_SEG(ES, es);
6877 rc |= VMXLOCAL_READ_SEG(FS, fs);
6878 rc |= VMXLOCAL_READ_SEG(GS, gs);
6879 AssertRCReturn(rc, rc);
6880
6881 /* Restore segment attributes for real-on-v86 mode hack. */
6882 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6883 {
6884 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6885 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6886 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6887 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6888 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6889 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6890 }
6891 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6892 }
6893
6894 return VINF_SUCCESS;
6895}
6896
6897
6898/**
6899 * Saves the guest descriptor table registers and task register from the current
6900 * VMCS into the guest-CPU context.
6901 *
6902 * @returns VBox status code.
6903 * @param pVCpu The cross context virtual CPU structure.
6904 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6905 * out-of-sync. Make sure to update the required fields
6906 * before using them.
6907 *
6908 * @remarks No-long-jump zone!!!
6909 */
6910static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6911{
6912 int rc = VINF_SUCCESS;
6913
6914 /* Guest LDTR. */
6915 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6916 {
6917 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR));
6918 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6919 AssertRCReturn(rc, rc);
6920 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6921 }
6922
6923 /* Guest GDTR. */
6924 uint64_t u64Val = 0;
6925 uint32_t u32Val = 0;
6926 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6927 {
6928 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR));
6929 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6930 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6931 pMixedCtx->gdtr.pGdt = u64Val;
6932 pMixedCtx->gdtr.cbGdt = u32Val;
6933 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6934 }
6935
6936 /* Guest IDTR. */
6937 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6938 {
6939 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR));
6940 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6941 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6942 pMixedCtx->idtr.pIdt = u64Val;
6943 pMixedCtx->idtr.cbIdt = u32Val;
6944 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6945 }
6946
6947 /* Guest TR. */
6948 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6949 {
6950 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR));
6951 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6952 AssertRCReturn(rc, rc);
6953
6954 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6955 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6956 {
6957 rc = VMXLOCAL_READ_SEG(TR, tr);
6958 AssertRCReturn(rc, rc);
6959 }
6960 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6961 }
6962 return rc;
6963}
6964
6965#undef VMXLOCAL_READ_SEG
6966
6967
6968/**
6969 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6970 * context.
6971 *
6972 * @returns VBox status code.
6973 * @param pVCpu The cross context virtual CPU structure.
6974 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6975 * out-of-sync. Make sure to update the required fields
6976 * before using them.
6977 *
6978 * @remarks No-long-jump zone!!!
6979 */
6980static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6981{
6982 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DR7))
6983 {
6984 if (!pVCpu->hm.s.fUsingHyperDR7)
6985 {
6986 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6987 uint32_t u32Val;
6988 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6989 pMixedCtx->dr[7] = u32Val;
6990 }
6991
6992 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DR7);
6993 }
6994 return VINF_SUCCESS;
6995}
6996
6997
6998/**
6999 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
7000 *
7001 * @returns VBox status code.
7002 * @param pVCpu The cross context virtual CPU structure.
7003 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7004 * out-of-sync. Make sure to update the required fields
7005 * before using them.
7006 *
7007 * @remarks No-long-jump zone!!!
7008 */
7009static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7010{
7011 NOREF(pMixedCtx);
7012
7013 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
7014 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
7015 return VINF_SUCCESS;
7016}
7017
7018
7019/**
7020 * Saves the entire guest state from the currently active VMCS into the
7021 * guest-CPU context.
7022 *
7023 * This essentially VMREADs all guest-data.
7024 *
7025 * @returns VBox status code.
7026 * @param pVCpu The cross context virtual CPU structure.
7027 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7028 * out-of-sync. Make sure to update the required fields
7029 * before using them.
7030 */
7031static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7032{
7033 Assert(pVCpu);
7034 Assert(pMixedCtx);
7035
7036 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
7037 return VINF_SUCCESS;
7038
7039 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
7040 again on the ring-3 callback path, there is no real need to. */
7041 if (VMMRZCallRing3IsEnabled(pVCpu))
7042 VMMR0LogFlushDisable(pVCpu);
7043 else
7044 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7045 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
7046
7047 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7048 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7049
7050 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7051 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7052
7053 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7054 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7055
7056 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7057 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7058
7059 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
7060 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7061
7062 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
7063 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7064
7065 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7066 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7067
7068 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7069 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7070
7071 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
7072 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7073
7074 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
7075 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7076
7077 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
7078 ("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
7079 HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
7080
7081 if (VMMRZCallRing3IsEnabled(pVCpu))
7082 VMMR0LogFlushEnable(pVCpu);
7083
7084 return VINF_SUCCESS;
7085}
7086
7087
7088/**
7089 * Saves basic guest registers needed for IEM instruction execution.
7090 *
7091 * @returns VBox status code (OR-able).
7092 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
7093 * @param pMixedCtx Pointer to the CPU context of the guest.
7094 * @param fMemory Whether the instruction being executed operates on
7095 * memory or not. Only CR0 is synced up if clear.
7096 * @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
7097 */
7098static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
7099{
7100 /*
7101 * We assume all general purpose registers other than RSP are available.
7102 *
7103 * - RIP is a must, as it will be incremented or otherwise changed.
7104 * - RFLAGS are always required to figure the CPL.
7105 * - RSP isn't always required, however it's a GPR, so frequently required.
7106 * - SS and CS are the only segment register needed if IEM doesn't do memory
7107 * access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
7108 * - CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
7109 * be required for memory accesses.
7110 *
7111 * Note! Before IEM dispatches an exception, it will call us to sync in everything.
7112 */
7113 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7114 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7115 if (fNeedRsp)
7116 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
7117 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7118 if (!fMemory)
7119 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7120 else
7121 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7122 AssertRCReturn(rc, rc);
7123 return rc;
7124}
7125
7126
7127/**
7128 * Ensures that we've got a complete basic guest-context.
7129 *
7130 * This excludes the FPU, SSE, AVX, and similar extended state. The interface
7131 * is for the interpreter.
7132 *
7133 * @returns VBox status code.
7134 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
7135 * @param pMixedCtx Pointer to the guest-CPU context which may have data
7136 * needing to be synced in.
7137 * @thread EMT(pVCpu)
7138 */
7139VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7140{
7141 /* Note! Since this is only applicable to VT-x, the implementation is placed
7142 in the VT-x part of the sources instead of the generic stuff. */
7143 int rc;
7144 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
7145 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7146 else
7147 rc = VINF_SUCCESS;
7148
7149 /*
7150 * For now, imply that the caller might change everything too. Do this after
7151 * saving the guest state so as to not trigger assertions.
7152 *
7153 * This is required for AMD-V too as it too only selectively re-loads changed
7154 * guest state back in to the VMCB.
7155 */
7156 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7157 return rc;
7158}
7159
7160
7161/**
7162 * Check per-VM and per-VCPU force flag actions that require us to go back to
7163 * ring-3 for one reason or another.
7164 *
7165 * @returns Strict VBox status code (i.e. informational status codes too)
7166 * @retval VINF_SUCCESS if we don't have any actions that require going back to
7167 * ring-3.
7168 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7169 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7170 * interrupts)
7171 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7172 * all EMTs to be in ring-3.
7173 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7174 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7175 * to the EM loop.
7176 *
7177 * @param pVM The cross context VM structure.
7178 * @param pVCpu The cross context virtual CPU structure.
7179 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7180 * out-of-sync. Make sure to update the required fields
7181 * before using them.
7182 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
7183 */
7184static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
7185{
7186 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7187
7188 /*
7189 * Anything pending? Should be more likely than not if we're doing a good job.
7190 */
7191 if ( !fStepping
7192 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7193 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7194 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7195 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7196 return VINF_SUCCESS;
7197
7198 /* We need the control registers now, make sure the guest-CPU context is updated. */
7199 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7200 AssertRCReturn(rc3, rc3);
7201
7202 /* Pending HM CR3 sync. */
7203 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7204 {
7205 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
7206 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
7207 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
7208 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7209 }
7210
7211 /* Pending HM PAE PDPEs. */
7212 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7213 {
7214 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7215 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7216 }
7217
7218 /* Pending PGM C3 sync. */
7219 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7220 {
7221 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
7222 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7223 if (rcStrict2 != VINF_SUCCESS)
7224 {
7225 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7226 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7227 return rcStrict2;
7228 }
7229 }
7230
7231 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7232 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
7233 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7234 {
7235 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7236 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
7237 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7238 return rc2;
7239 }
7240
7241 /* Pending VM request packets, such as hardware interrupts. */
7242 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
7243 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
7244 {
7245 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
7246 return VINF_EM_PENDING_REQUEST;
7247 }
7248
7249 /* Pending PGM pool flushes. */
7250 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7251 {
7252 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
7253 return VINF_PGM_POOL_FLUSH_PENDING;
7254 }
7255
7256 /* Pending DMA requests. */
7257 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
7258 {
7259 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
7260 return VINF_EM_RAW_TO_R3;
7261 }
7262
7263 return VINF_SUCCESS;
7264}
7265
7266
7267/**
7268 * Converts any TRPM trap into a pending HM event. This is typically used when
7269 * entering from ring-3 (not longjmp returns).
7270 *
7271 * @param pVCpu The cross context virtual CPU structure.
7272 */
7273static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7274{
7275 Assert(TRPMHasTrap(pVCpu));
7276 Assert(!pVCpu->hm.s.Event.fPending);
7277
7278 uint8_t uVector;
7279 TRPMEVENT enmTrpmEvent;
7280 RTGCUINT uErrCode;
7281 RTGCUINTPTR GCPtrFaultAddress;
7282 uint8_t cbInstr;
7283
7284 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7285 AssertRC(rc);
7286
7287 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7288 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7289 if (enmTrpmEvent == TRPM_TRAP)
7290 {
7291 switch (uVector)
7292 {
7293 case X86_XCPT_NMI:
7294 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7295 break;
7296
7297 case X86_XCPT_BP:
7298 case X86_XCPT_OF:
7299 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7300 break;
7301
7302 case X86_XCPT_PF:
7303 case X86_XCPT_DF:
7304 case X86_XCPT_TS:
7305 case X86_XCPT_NP:
7306 case X86_XCPT_SS:
7307 case X86_XCPT_GP:
7308 case X86_XCPT_AC:
7309 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7310 RT_FALL_THRU();
7311 default:
7312 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7313 break;
7314 }
7315 }
7316 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7317 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7318 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7319 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7320 else
7321 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7322
7323 rc = TRPMResetTrap(pVCpu);
7324 AssertRC(rc);
7325 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7326 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7327
7328 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7329}
7330
7331
7332/**
7333 * Converts the pending HM event into a TRPM trap.
7334 *
7335 * @param pVCpu The cross context virtual CPU structure.
7336 */
7337static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7338{
7339 Assert(pVCpu->hm.s.Event.fPending);
7340
7341 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7342 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7343 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
7344 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7345
7346 /* If a trap was already pending, we did something wrong! */
7347 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7348
7349 TRPMEVENT enmTrapType;
7350 switch (uVectorType)
7351 {
7352 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7353 enmTrapType = TRPM_HARDWARE_INT;
7354 break;
7355
7356 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7357 enmTrapType = TRPM_SOFTWARE_INT;
7358 break;
7359
7360 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7361 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7362 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7363 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7364 enmTrapType = TRPM_TRAP;
7365 break;
7366
7367 default:
7368 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7369 enmTrapType = TRPM_32BIT_HACK;
7370 break;
7371 }
7372
7373 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7374
7375 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7376 AssertRC(rc);
7377
7378 if (fErrorCodeValid)
7379 TRPMSetErrorCode(pVCpu, uErrorCode);
7380
7381 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7382 && uVector == X86_XCPT_PF)
7383 {
7384 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7385 }
7386 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7387 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7388 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7389 {
7390 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7391 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
7392 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7393 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7394 }
7395
7396 /* Clear any pending events from the VMCS. */
7397 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
7398 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
7399
7400 /* We're now done converting the pending event. */
7401 pVCpu->hm.s.Event.fPending = false;
7402}
7403
7404
7405/**
7406 * Does the necessary state syncing before returning to ring-3 for any reason
7407 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7408 *
7409 * @returns VBox status code.
7410 * @param pVCpu The cross context virtual CPU structure.
7411 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7412 * be out-of-sync. Make sure to update the required
7413 * fields before using them.
7414 * @param fSaveGuestState Whether to save the guest state or not.
7415 *
7416 * @remarks No-long-jmp zone!!!
7417 */
7418static int hmR0VmxLeave(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7419{
7420 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7421 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7422
7423 RTCPUID idCpu = RTMpCpuId();
7424 Log4Func(("HostCpuId=%u\n", idCpu));
7425
7426 /*
7427 * !!! IMPORTANT !!!
7428 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7429 */
7430
7431 /* Save the guest state if necessary. */
7432 if ( fSaveGuestState
7433 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7434 {
7435 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7436 AssertRCReturn(rc, rc);
7437 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7438 }
7439
7440 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
7441 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7442 {
7443 /* We shouldn't reload CR0 without saving it first. */
7444 if (!fSaveGuestState)
7445 {
7446 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7447 AssertRCReturn(rc, rc);
7448 }
7449 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7450 }
7451
7452 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7453#ifdef VBOX_STRICT
7454 if (CPUMIsHyperDebugStateActive(pVCpu))
7455 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7456#endif
7457 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7458 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7459 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7460 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7461
7462#if HC_ARCH_BITS == 64
7463 /* Restore host-state bits that VT-x only restores partially. */
7464 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7465 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7466 {
7467 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7468 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7469 }
7470 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7471#endif
7472
7473 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7474 if (pVCpu->hm.s.vmx.fLazyMsrs)
7475 {
7476 /* We shouldn't reload the guest MSRs without saving it first. */
7477 if (!fSaveGuestState)
7478 {
7479 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7480 AssertRCReturn(rc, rc);
7481 }
7482 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7483 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7484 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7485 }
7486
7487 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7488 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7489
7490 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7491 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7492 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7493 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7494 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7495 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7496 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7497 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7498
7499 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7500
7501 /** @todo This partially defeats the purpose of having preemption hooks.
7502 * The problem is, deregistering the hooks should be moved to a place that
7503 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7504 * context.
7505 */
7506 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7507 {
7508 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7509 AssertRCReturn(rc, rc);
7510
7511 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7512 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7513 }
7514 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7515 NOREF(idCpu);
7516
7517 return VINF_SUCCESS;
7518}
7519
7520
7521/**
7522 * Leaves the VT-x session.
7523 *
7524 * @returns VBox status code.
7525 * @param pVCpu The cross context virtual CPU structure.
7526 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7527 * out-of-sync. Make sure to update the required fields
7528 * before using them.
7529 *
7530 * @remarks No-long-jmp zone!!!
7531 */
7532DECLINLINE(int) hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7533{
7534 HM_DISABLE_PREEMPT();
7535 HMVMX_ASSERT_CPU_SAFE();
7536 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7537 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7538
7539 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7540 and done this from the VMXR0ThreadCtxCallback(). */
7541 if (!pVCpu->hm.s.fLeaveDone)
7542 {
7543 int rc2 = hmR0VmxLeave(pVCpu, pMixedCtx, true /* fSaveGuestState */);
7544 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7545 pVCpu->hm.s.fLeaveDone = true;
7546 }
7547 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7548
7549 /*
7550 * !!! IMPORTANT !!!
7551 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7552 */
7553
7554 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7555 /** @todo Deregistering here means we need to VMCLEAR always
7556 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7557 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7558 VMMR0ThreadCtxHookDisable(pVCpu);
7559
7560 /* Leave HM context. This takes care of local init (term). */
7561 int rc = HMR0LeaveCpu(pVCpu);
7562
7563 HM_RESTORE_PREEMPT();
7564 return rc;
7565}
7566
7567
7568/**
7569 * Does the necessary state syncing before doing a longjmp to ring-3.
7570 *
7571 * @returns VBox status code.
7572 * @param pVCpu The cross context virtual CPU structure.
7573 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7574 * out-of-sync. Make sure to update the required fields
7575 * before using them.
7576 *
7577 * @remarks No-long-jmp zone!!!
7578 */
7579DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7580{
7581 return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7582}
7583
7584
7585/**
7586 * Take necessary actions before going back to ring-3.
7587 *
7588 * An action requires us to go back to ring-3. This function does the necessary
7589 * steps before we can safely return to ring-3. This is not the same as longjmps
7590 * to ring-3, this is voluntary and prepares the guest so it may continue
7591 * executing outside HM (recompiler/IEM).
7592 *
7593 * @returns VBox status code.
7594 * @param pVM The cross context VM structure.
7595 * @param pVCpu The cross context virtual CPU structure.
7596 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7597 * out-of-sync. Make sure to update the required fields
7598 * before using them.
7599 * @param rcExit The reason for exiting to ring-3. Can be
7600 * VINF_VMM_UNKNOWN_RING3_CALL.
7601 */
7602static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7603{
7604 Assert(pVM);
7605 Assert(pVCpu);
7606 Assert(pMixedCtx);
7607 HMVMX_ASSERT_PREEMPT_SAFE();
7608
7609 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7610 {
7611 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7612 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7613 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7614 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7615 }
7616
7617 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7618 VMMRZCallRing3Disable(pVCpu);
7619 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7620
7621 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7622 if (pVCpu->hm.s.Event.fPending)
7623 {
7624 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7625 Assert(!pVCpu->hm.s.Event.fPending);
7626 }
7627
7628 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7629 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7630
7631 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7632 and if we're injecting an event we should have a TRPM trap pending. */
7633 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7634#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7635 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7636#endif
7637
7638 /* Save guest state and restore host state bits. */
7639 int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7640 AssertRCReturn(rc, rc);
7641 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7642 /* Thread-context hooks are unregistered at this point!!! */
7643
7644 /* Sync recompiler state. */
7645 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7646 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7647 | CPUM_CHANGED_LDTR
7648 | CPUM_CHANGED_GDTR
7649 | CPUM_CHANGED_IDTR
7650 | CPUM_CHANGED_TR
7651 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7652 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7653 if ( pVM->hm.s.fNestedPaging
7654 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7655 {
7656 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7657 }
7658
7659 Assert(!pVCpu->hm.s.fClearTrapFlag);
7660
7661 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7662 if (rcExit != VINF_EM_RAW_INTERRUPT)
7663 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7664
7665 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7666
7667 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7668 VMMRZCallRing3RemoveNotification(pVCpu);
7669 VMMRZCallRing3Enable(pVCpu);
7670
7671 return rc;
7672}
7673
7674
7675/**
7676 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7677 * longjump to ring-3 and possibly get preempted.
7678 *
7679 * @returns VBox status code.
7680 * @param pVCpu The cross context virtual CPU structure.
7681 * @param enmOperation The operation causing the ring-3 longjump.
7682 * @param pvUser Opaque pointer to the guest-CPU context. The data
7683 * may be out-of-sync. Make sure to update the required
7684 * fields before using them.
7685 */
7686static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7687{
7688 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7689 {
7690 /*
7691 * !!! IMPORTANT !!!
7692 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7693 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7694 */
7695 VMMRZCallRing3RemoveNotification(pVCpu);
7696 VMMRZCallRing3Disable(pVCpu);
7697 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7698 RTThreadPreemptDisable(&PreemptState);
7699
7700 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7701 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7702
7703#if HC_ARCH_BITS == 64
7704 /* Restore host-state bits that VT-x only restores partially. */
7705 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7706 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7707 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7708 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7709#endif
7710 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7711 if (pVCpu->hm.s.vmx.fLazyMsrs)
7712 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7713
7714 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7715 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7716 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7717 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7718 {
7719 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7720 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7721 }
7722
7723 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7724 VMMR0ThreadCtxHookDisable(pVCpu);
7725 HMR0LeaveCpu(pVCpu);
7726 RTThreadPreemptRestore(&PreemptState);
7727 return VINF_SUCCESS;
7728 }
7729
7730 Assert(pVCpu);
7731 Assert(pvUser);
7732 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7733 HMVMX_ASSERT_PREEMPT_SAFE();
7734
7735 VMMRZCallRing3Disable(pVCpu);
7736 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7737
7738 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7739 enmOperation));
7740
7741 int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7742 AssertRCReturn(rc, rc);
7743
7744 VMMRZCallRing3Enable(pVCpu);
7745 return VINF_SUCCESS;
7746}
7747
7748
7749/**
7750 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7751 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7752 *
7753 * @param pVCpu The cross context virtual CPU structure.
7754 */
7755DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7756{
7757 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7758 {
7759 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7760 {
7761 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7762 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7763 AssertRC(rc);
7764 Log4(("Setup interrupt-window exiting\n"));
7765 }
7766 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7767}
7768
7769
7770/**
7771 * Clears the interrupt-window exiting control in the VMCS.
7772 *
7773 * @param pVCpu The cross context virtual CPU structure.
7774 */
7775DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7776{
7777 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7778 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7779 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7780 AssertRC(rc);
7781 Log4(("Cleared interrupt-window exiting\n"));
7782}
7783
7784
7785/**
7786 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7787 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7788 *
7789 * @param pVCpu The cross context virtual CPU structure.
7790 */
7791DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7792{
7793 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7794 {
7795 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7796 {
7797 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7798 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7799 AssertRC(rc);
7800 Log4(("Setup NMI-window exiting\n"));
7801 }
7802 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7803}
7804
7805
7806/**
7807 * Clears the NMI-window exiting control in the VMCS.
7808 *
7809 * @param pVCpu The cross context virtual CPU structure.
7810 */
7811DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7812{
7813 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7814 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7815 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7816 AssertRC(rc);
7817 Log4(("Cleared NMI-window exiting\n"));
7818}
7819
7820
7821/**
7822 * Evaluates the event to be delivered to the guest and sets it as the pending
7823 * event.
7824 *
7825 * @returns The VT-x guest-interruptibility state.
7826 * @param pVCpu The cross context virtual CPU structure.
7827 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7828 * out-of-sync. Make sure to update the required fields
7829 * before using them.
7830 */
7831static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7832{
7833 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7834 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7835 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7836 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7837 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7838
7839 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7840 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7841 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7842 Assert(!TRPMHasTrap(pVCpu));
7843
7844 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7845 APICUpdatePendingInterrupts(pVCpu);
7846
7847 /*
7848 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7849 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7850 */
7851 /** @todo SMI. SMIs take priority over NMIs. */
7852 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7853 {
7854 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7855 if ( !pVCpu->hm.s.Event.fPending
7856 && !fBlockNmi
7857 && !fBlockSti
7858 && !fBlockMovSS)
7859 {
7860 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7861 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7862 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7863
7864 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7865 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7866 }
7867 else
7868 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7869 }
7870 /*
7871 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7872 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7873 */
7874 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7875 && !pVCpu->hm.s.fSingleInstruction)
7876 {
7877 Assert(!DBGFIsStepping(pVCpu));
7878 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7879 AssertRC(rc);
7880 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7881 if ( !pVCpu->hm.s.Event.fPending
7882 && !fBlockInt
7883 && !fBlockSti
7884 && !fBlockMovSS)
7885 {
7886 uint8_t u8Interrupt;
7887 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7888 if (RT_SUCCESS(rc))
7889 {
7890 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7891 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7892 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7893
7894 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7895 }
7896 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7897 {
7898 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7899 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7900 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7901
7902 /*
7903 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7904 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7905 * need to re-set this force-flag here.
7906 */
7907 }
7908 else
7909 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7910 }
7911 else
7912 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7913 }
7914
7915 return uIntrState;
7916}
7917
7918
7919/**
7920 * Sets a pending-debug exception to be delivered to the guest if the guest is
7921 * single-stepping in the VMCS.
7922 *
7923 * @param pVCpu The cross context virtual CPU structure.
7924 */
7925DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7926{
7927 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7928 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7929 AssertRC(rc);
7930}
7931
7932
7933/**
7934 * Injects any pending events into the guest if the guest is in a state to
7935 * receive them.
7936 *
7937 * @returns Strict VBox status code (i.e. informational status codes too).
7938 * @param pVCpu The cross context virtual CPU structure.
7939 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7940 * out-of-sync. Make sure to update the required fields
7941 * before using them.
7942 * @param uIntrState The VT-x guest-interruptibility state.
7943 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7944 * return VINF_EM_DBG_STEPPED if the event was
7945 * dispatched directly.
7946 */
7947static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7948{
7949 HMVMX_ASSERT_PREEMPT_SAFE();
7950 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7951
7952 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7953 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7954
7955 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7956 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7957 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7958 Assert(!TRPMHasTrap(pVCpu));
7959
7960 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7961 if (pVCpu->hm.s.Event.fPending)
7962 {
7963 /*
7964 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7965 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7966 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7967 *
7968 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7969 */
7970 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7971#ifdef VBOX_STRICT
7972 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7973 {
7974 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7975 Assert(!fBlockInt);
7976 Assert(!fBlockSti);
7977 Assert(!fBlockMovSS);
7978 }
7979 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7980 {
7981 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7982 Assert(!fBlockSti);
7983 Assert(!fBlockMovSS);
7984 Assert(!fBlockNmi);
7985 }
7986#endif
7987 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7988 (uint8_t)uIntType));
7989 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7990 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7991 fStepping, &uIntrState);
7992 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7993
7994 /* Update the interruptibility-state as it could have been changed by
7995 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7996 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7997 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7998
7999 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8000 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
8001 else
8002 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
8003 }
8004
8005 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
8006 if ( fBlockSti
8007 || fBlockMovSS)
8008 {
8009 if (!pVCpu->hm.s.fSingleInstruction)
8010 {
8011 /*
8012 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
8013 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
8014 * See Intel spec. 27.3.4 "Saving Non-Register State".
8015 */
8016 Assert(!DBGFIsStepping(pVCpu));
8017 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8018 AssertRCReturn(rc2, rc2);
8019 if (pMixedCtx->eflags.Bits.u1TF)
8020 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
8021 }
8022 else if (pMixedCtx->eflags.Bits.u1TF)
8023 {
8024 /*
8025 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
8026 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
8027 */
8028 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
8029 uIntrState = 0;
8030 }
8031 }
8032
8033 /*
8034 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
8035 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8036 */
8037 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
8038 AssertRC(rc2);
8039
8040 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8041 NOREF(fBlockMovSS); NOREF(fBlockSti);
8042 return rcStrict;
8043}
8044
8045
8046/**
8047 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
8048 *
8049 * @param pVCpu The cross context virtual CPU structure.
8050 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8051 * out-of-sync. Make sure to update the required fields
8052 * before using them.
8053 */
8054DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8055{
8056 NOREF(pMixedCtx);
8057 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
8058 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
8059}
8060
8061
8062/**
8063 * Injects a double-fault (\#DF) exception into the VM.
8064 *
8065 * @returns Strict VBox status code (i.e. informational status codes too).
8066 * @param pVCpu The cross context virtual CPU structure.
8067 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8068 * out-of-sync. Make sure to update the required fields
8069 * before using them.
8070 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
8071 * and should return VINF_EM_DBG_STEPPED if the event
8072 * is injected directly (register modified by us, not
8073 * by hardware on VM-entry).
8074 * @param puIntrState Pointer to the current guest interruptibility-state.
8075 * This interruptibility-state will be updated if
8076 * necessary. This cannot not be NULL.
8077 */
8078DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
8079{
8080 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
8081 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8082 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8083 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
8084 fStepping, puIntrState);
8085}
8086
8087
8088/**
8089 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
8090 *
8091 * @param pVCpu The cross context virtual CPU structure.
8092 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8093 * out-of-sync. Make sure to update the required fields
8094 * before using them.
8095 */
8096DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8097{
8098 NOREF(pMixedCtx);
8099 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
8100 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8101 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
8102}
8103
8104
8105/**
8106 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
8107 *
8108 * @param pVCpu The cross context virtual CPU structure.
8109 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8110 * out-of-sync. Make sure to update the required fields
8111 * before using them.
8112 * @param cbInstr The value of RIP that is to be pushed on the guest
8113 * stack.
8114 */
8115DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
8116{
8117 NOREF(pMixedCtx);
8118 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
8119 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8120 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
8121}
8122
8123
8124/**
8125 * Injects a general-protection (\#GP) fault into the VM.
8126 *
8127 * @returns Strict VBox status code (i.e. informational status codes too).
8128 * @param pVCpu The cross context virtual CPU structure.
8129 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8130 * out-of-sync. Make sure to update the required fields
8131 * before using them.
8132 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
8133 * mode, i.e. in real-mode it's not valid).
8134 * @param u32ErrorCode The error code associated with the \#GP.
8135 * @param fStepping Whether we're running in
8136 * hmR0VmxRunGuestCodeStep() and should return
8137 * VINF_EM_DBG_STEPPED if the event is injected
8138 * directly (register modified by us, not by
8139 * hardware on VM-entry).
8140 * @param puIntrState Pointer to the current guest interruptibility-state.
8141 * This interruptibility-state will be updated if
8142 * necessary. This cannot not be NULL.
8143 */
8144DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
8145 bool fStepping, uint32_t *puIntrState)
8146{
8147 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
8148 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8149 if (fErrorCodeValid)
8150 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8151 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
8152 fStepping, puIntrState);
8153}
8154
8155
8156#if 0 /* unused */
8157/**
8158 * Sets a general-protection (\#GP) exception as pending-for-injection into the
8159 * VM.
8160 *
8161 * @param pVCpu The cross context virtual CPU structure.
8162 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8163 * out-of-sync. Make sure to update the required fields
8164 * before using them.
8165 * @param u32ErrorCode The error code associated with the \#GP.
8166 */
8167DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
8168{
8169 NOREF(pMixedCtx);
8170 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
8171 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8172 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8173 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
8174}
8175#endif /* unused */
8176
8177
8178/**
8179 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
8180 *
8181 * @param pVCpu The cross context virtual CPU structure.
8182 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8183 * out-of-sync. Make sure to update the required fields
8184 * before using them.
8185 * @param uVector The software interrupt vector number.
8186 * @param cbInstr The value of RIP that is to be pushed on the guest
8187 * stack.
8188 */
8189DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
8190{
8191 NOREF(pMixedCtx);
8192 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
8193 if ( uVector == X86_XCPT_BP
8194 || uVector == X86_XCPT_OF)
8195 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8196 else
8197 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8198 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
8199}
8200
8201
8202/**
8203 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8204 * stack.
8205 *
8206 * @returns Strict VBox status code (i.e. informational status codes too).
8207 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8208 * @param pVM The cross context VM structure.
8209 * @param pMixedCtx Pointer to the guest-CPU context.
8210 * @param uValue The value to push to the guest stack.
8211 */
8212DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
8213{
8214 /*
8215 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8216 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8217 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8218 */
8219 if (pMixedCtx->sp == 1)
8220 return VINF_EM_RESET;
8221 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8222 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
8223 AssertRC(rc);
8224 return rc;
8225}
8226
8227
8228/**
8229 * Injects an event into the guest upon VM-entry by updating the relevant fields
8230 * in the VM-entry area in the VMCS.
8231 *
8232 * @returns Strict VBox status code (i.e. informational status codes too).
8233 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8234 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8235 *
8236 * @param pVCpu The cross context virtual CPU structure.
8237 * @param pMixedCtx Pointer to the guest-CPU context. The data may
8238 * be out-of-sync. Make sure to update the required
8239 * fields before using them.
8240 * @param u64IntInfo The VM-entry interruption-information field.
8241 * @param cbInstr The VM-entry instruction length in bytes (for
8242 * software interrupts, exceptions and privileged
8243 * software exceptions).
8244 * @param u32ErrCode The VM-entry exception error code.
8245 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
8246 * @param puIntrState Pointer to the current guest interruptibility-state.
8247 * This interruptibility-state will be updated if
8248 * necessary. This cannot not be NULL.
8249 * @param fStepping Whether we're running in
8250 * hmR0VmxRunGuestCodeStep() and should return
8251 * VINF_EM_DBG_STEPPED if the event is injected
8252 * directly (register modified by us, not by
8253 * hardware on VM-entry).
8254 *
8255 * @remarks Requires CR0!
8256 */
8257static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
8258 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
8259 uint32_t *puIntrState)
8260{
8261 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8262 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
8263 Assert(puIntrState);
8264 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
8265
8266 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
8267 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
8268
8269#ifdef VBOX_STRICT
8270 /*
8271 * Validate the error-code-valid bit for hardware exceptions.
8272 * No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8273 */
8274 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8275 && !CPUMIsGuestInRealModeEx(pMixedCtx))
8276 {
8277 switch (uVector)
8278 {
8279 case X86_XCPT_PF:
8280 case X86_XCPT_DF:
8281 case X86_XCPT_TS:
8282 case X86_XCPT_NP:
8283 case X86_XCPT_SS:
8284 case X86_XCPT_GP:
8285 case X86_XCPT_AC:
8286 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
8287 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8288 RT_FALL_THRU();
8289 default:
8290 break;
8291 }
8292 }
8293#endif
8294
8295 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
8296 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8297 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
8298
8299 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8300
8301 /* We require CR0 to check if the guest is in real-mode. */
8302 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8303 AssertRCReturn(rc, rc);
8304
8305 /*
8306 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
8307 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
8308 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
8309 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8310 */
8311 if (CPUMIsGuestInRealModeEx(pMixedCtx))
8312 {
8313 PVM pVM = pVCpu->CTX_SUFF(pVM);
8314 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
8315 {
8316 Assert(PDMVmmDevHeapIsEnabled(pVM));
8317 Assert(pVM->hm.s.vmx.pRealModeTSS);
8318
8319 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
8320 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8321 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
8322 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8323 AssertRCReturn(rc, rc);
8324 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
8325
8326 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8327 size_t const cbIdtEntry = sizeof(X86IDTR16);
8328 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
8329 {
8330 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8331 if (uVector == X86_XCPT_DF)
8332 return VINF_EM_RESET;
8333
8334 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
8335 if (uVector == X86_XCPT_GP)
8336 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
8337
8338 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
8339 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
8340 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
8341 fStepping, puIntrState);
8342 }
8343
8344 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8345 uint16_t uGuestIp = pMixedCtx->ip;
8346 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
8347 {
8348 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
8349 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
8350 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8351 }
8352 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
8353 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8354
8355 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8356 X86IDTR16 IdtEntry;
8357 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
8358 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8359 AssertRCReturn(rc, rc);
8360
8361 /* Construct the stack frame for the interrupt/exception handler. */
8362 VBOXSTRICTRC rcStrict;
8363 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
8364 if (rcStrict == VINF_SUCCESS)
8365 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
8366 if (rcStrict == VINF_SUCCESS)
8367 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
8368
8369 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
8370 if (rcStrict == VINF_SUCCESS)
8371 {
8372 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
8373 pMixedCtx->rip = IdtEntry.offSel;
8374 pMixedCtx->cs.Sel = IdtEntry.uSel;
8375 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
8376 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8377 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8378 && uVector == X86_XCPT_PF)
8379 pMixedCtx->cr2 = GCPtrFaultAddress;
8380
8381 /* If any other guest-state bits are changed here, make sure to update
8382 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
8383 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
8384 | HM_CHANGED_GUEST_RIP
8385 | HM_CHANGED_GUEST_RFLAGS
8386 | HM_CHANGED_GUEST_RSP);
8387
8388 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8389 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8390 {
8391 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8392 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8393 Log4(("Clearing inhibition due to STI.\n"));
8394 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
8395 }
8396 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8397 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
8398
8399 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8400 it, if we are returning to ring-3 before executing guest code. */
8401 pVCpu->hm.s.Event.fPending = false;
8402
8403 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8404 if (fStepping)
8405 rcStrict = VINF_EM_DBG_STEPPED;
8406 }
8407 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8408 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8409 return rcStrict;
8410 }
8411
8412 /*
8413 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8414 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8415 */
8416 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8417 }
8418
8419 /* Validate. */
8420 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8421 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8422 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8423
8424 /* Inject. */
8425 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8426 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8427 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8428 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8429
8430 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8431 && uVector == X86_XCPT_PF)
8432 pMixedCtx->cr2 = GCPtrFaultAddress;
8433
8434 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8435 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8436
8437 AssertRCReturn(rc, rc);
8438 return VINF_SUCCESS;
8439}
8440
8441
8442/**
8443 * Clears the interrupt-window exiting control in the VMCS and if necessary
8444 * clears the current event in the VMCS as well.
8445 *
8446 * @returns VBox status code.
8447 * @param pVCpu The cross context virtual CPU structure.
8448 *
8449 * @remarks Use this function only to clear events that have not yet been
8450 * delivered to the guest but are injected in the VMCS!
8451 * @remarks No-long-jump zone!!!
8452 */
8453static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8454{
8455 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8456
8457 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8458 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8459
8460 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8461 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8462}
8463
8464
8465/**
8466 * Enters the VT-x session.
8467 *
8468 * @returns VBox status code.
8469 * @param pVM The cross context VM structure.
8470 * @param pVCpu The cross context virtual CPU structure.
8471 * @param pCpu Pointer to the CPU info struct.
8472 */
8473VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8474{
8475 AssertPtr(pVM);
8476 AssertPtr(pVCpu);
8477 Assert(pVM->hm.s.vmx.fSupported);
8478 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8479 NOREF(pCpu); NOREF(pVM);
8480
8481 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8482 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8483
8484#ifdef VBOX_STRICT
8485 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8486 RTCCUINTREG uHostCR4 = ASMGetCR4();
8487 if (!(uHostCR4 & X86_CR4_VMXE))
8488 {
8489 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8490 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8491 }
8492#endif
8493
8494 /*
8495 * Load the VCPU's VMCS as the current (and active) one.
8496 */
8497 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8498 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8499 if (RT_FAILURE(rc))
8500 return rc;
8501
8502 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8503 pVCpu->hm.s.fLeaveDone = false;
8504 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8505
8506 return VINF_SUCCESS;
8507}
8508
8509
8510/**
8511 * The thread-context callback (only on platforms which support it).
8512 *
8513 * @param enmEvent The thread-context event.
8514 * @param pVCpu The cross context virtual CPU structure.
8515 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8516 * @thread EMT(pVCpu)
8517 */
8518VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8519{
8520 NOREF(fGlobalInit);
8521
8522 switch (enmEvent)
8523 {
8524 case RTTHREADCTXEVENT_OUT:
8525 {
8526 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8527 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8528 VMCPU_ASSERT_EMT(pVCpu);
8529
8530 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8531
8532 /* No longjmps (logger flushes, locks) in this fragile context. */
8533 VMMRZCallRing3Disable(pVCpu);
8534 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8535
8536 /*
8537 * Restore host-state (FPU, debug etc.)
8538 */
8539 if (!pVCpu->hm.s.fLeaveDone)
8540 {
8541 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8542 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8543 hmR0VmxLeave(pVCpu, pMixedCtx, false /* fSaveGuestState */);
8544 pVCpu->hm.s.fLeaveDone = true;
8545 }
8546
8547 /* Leave HM context, takes care of local init (term). */
8548 int rc = HMR0LeaveCpu(pVCpu);
8549 AssertRC(rc); NOREF(rc);
8550
8551 /* Restore longjmp state. */
8552 VMMRZCallRing3Enable(pVCpu);
8553 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8554 break;
8555 }
8556
8557 case RTTHREADCTXEVENT_IN:
8558 {
8559 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8560 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8561 VMCPU_ASSERT_EMT(pVCpu);
8562
8563 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8564 VMMRZCallRing3Disable(pVCpu);
8565 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8566
8567 /* Initialize the bare minimum state required for HM. This takes care of
8568 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8569 int rc = HMR0EnterCpu(pVCpu);
8570 AssertRC(rc);
8571 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8572
8573 /* Load the active VMCS as the current one. */
8574 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8575 {
8576 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8577 AssertRC(rc); NOREF(rc);
8578 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8579 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8580 }
8581 pVCpu->hm.s.fLeaveDone = false;
8582
8583 /* Restore longjmp state. */
8584 VMMRZCallRing3Enable(pVCpu);
8585 break;
8586 }
8587
8588 default:
8589 break;
8590 }
8591}
8592
8593
8594/**
8595 * Saves the host state in the VMCS host-state.
8596 * Sets up the VM-exit MSR-load area.
8597 *
8598 * The CPU state will be loaded from these fields on every successful VM-exit.
8599 *
8600 * @returns VBox status code.
8601 * @param pVM The cross context VM structure.
8602 * @param pVCpu The cross context virtual CPU structure.
8603 *
8604 * @remarks No-long-jump zone!!!
8605 */
8606static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8607{
8608 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8609
8610 int rc = VINF_SUCCESS;
8611 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8612 {
8613 rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8614 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8615
8616 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8617 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8618
8619 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8620 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8621
8622 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8623 }
8624 return rc;
8625}
8626
8627
8628/**
8629 * Saves the host state in the VMCS host-state.
8630 *
8631 * @returns VBox status code.
8632 * @param pVM The cross context VM structure.
8633 * @param pVCpu The cross context virtual CPU structure.
8634 *
8635 * @remarks No-long-jump zone!!!
8636 */
8637VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8638{
8639 AssertPtr(pVM);
8640 AssertPtr(pVCpu);
8641
8642 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8643
8644 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8645 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8646 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8647 return hmR0VmxSaveHostState(pVM, pVCpu);
8648}
8649
8650
8651/**
8652 * Loads the guest state into the VMCS guest-state area.
8653 *
8654 * The will typically be done before VM-entry when the guest-CPU state and the
8655 * VMCS state may potentially be out of sync.
8656 *
8657 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8658 * VM-entry controls.
8659 * Sets up the appropriate VMX non-root function to execute guest code based on
8660 * the guest CPU mode.
8661 *
8662 * @returns VBox strict status code.
8663 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8664 * without unrestricted guest access and the VMMDev is not presently
8665 * mapped (e.g. EFI32).
8666 *
8667 * @param pVM The cross context VM structure.
8668 * @param pVCpu The cross context virtual CPU structure.
8669 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8670 * out-of-sync. Make sure to update the required fields
8671 * before using them.
8672 *
8673 * @remarks No-long-jump zone!!!
8674 */
8675static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8676{
8677 AssertPtr(pVM);
8678 AssertPtr(pVCpu);
8679 AssertPtr(pMixedCtx);
8680 HMVMX_ASSERT_PREEMPT_SAFE();
8681
8682 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8683
8684 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8685
8686 /* Determine real-on-v86 mode. */
8687 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8688 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8689 && CPUMIsGuestInRealModeEx(pMixedCtx))
8690 {
8691 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8692 }
8693
8694 /*
8695 * Load the guest-state into the VMCS.
8696 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8697 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8698 */
8699 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8700 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8701
8702 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8703 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8704 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8705
8706 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8707 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8708 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8709
8710 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8711 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8712
8713 VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8714 if (rcStrict == VINF_SUCCESS)
8715 { /* likely */ }
8716 else
8717 {
8718 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8719 return rcStrict;
8720 }
8721
8722 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8723 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8724 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8725
8726 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8727 determine we don't have to swap EFER after all. */
8728 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8729 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8730
8731 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8732 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8733
8734 rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8735 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8736
8737 /*
8738 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8739 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8740 */
8741 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8742 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8743
8744 /* Clear any unused and reserved bits. */
8745 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2
8746 | HM_CHANGED_GUEST_HWVIRT);
8747
8748 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8749 return rc;
8750}
8751
8752
8753/**
8754 * Loads the state shared between the host and guest into the VMCS.
8755 *
8756 * @param pVM The cross context VM structure.
8757 * @param pVCpu The cross context virtual CPU structure.
8758 * @param pCtx Pointer to the guest-CPU context.
8759 *
8760 * @remarks No-long-jump zone!!!
8761 */
8762static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8763{
8764 NOREF(pVM);
8765
8766 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8767 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8768
8769 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8770 {
8771 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8772 AssertRC(rc);
8773 }
8774
8775 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8776 {
8777 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8778 AssertRC(rc);
8779
8780 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8781 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8782 {
8783 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8784 AssertRC(rc);
8785 }
8786 }
8787
8788 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMM_GUEST_LAZY_MSRS))
8789 {
8790 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8791 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMM_GUEST_LAZY_MSRS);
8792 }
8793
8794 /* Loading CR0, debug state might have changed intercepts, update VMCS. */
8795 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMM_GUEST_XCPT_INTERCEPTS))
8796 {
8797 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8798 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8799 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8800 AssertRC(rc);
8801 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMM_GUEST_XCPT_INTERCEPTS);
8802 }
8803
8804 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8805 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8806}
8807
8808
8809/**
8810 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8811 *
8812 * @returns Strict VBox status code (i.e. informational status codes too).
8813 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8814 * without unrestricted guest access and the VMMDev is not presently
8815 * mapped (e.g. EFI32).
8816 *
8817 * @param pVM The cross context VM structure.
8818 * @param pVCpu The cross context virtual CPU structure.
8819 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8820 * out-of-sync. Make sure to update the required fields
8821 * before using them.
8822 *
8823 * @remarks No-long-jump zone!!!
8824 */
8825static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8826{
8827 HMVMX_ASSERT_PREEMPT_SAFE();
8828 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8829 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8830
8831 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8832#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8833 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8834#endif
8835
8836 /*
8837 * RIP is what changes the most often and hence if it's the only bit needing to be
8838 * updated, we shall handle it early for performance reasons.
8839 */
8840 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8841 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8842 {
8843 rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8844 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8845 { /* likely */}
8846 else
8847 {
8848 AssertMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8849 VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8850 }
8851 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8852 }
8853 else if (HMCPU_CF_VALUE(pVCpu))
8854 {
8855 rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8856 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8857 { /* likely */}
8858 else
8859 {
8860 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8861 ("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8862 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8863 return rcStrict;
8864 }
8865 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8866 }
8867
8868 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8869 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8870 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8871 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8872 return rcStrict;
8873}
8874
8875
8876/**
8877 * Does the preparations before executing guest code in VT-x.
8878 *
8879 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8880 * recompiler/IEM. We must be cautious what we do here regarding committing
8881 * guest-state information into the VMCS assuming we assuredly execute the
8882 * guest in VT-x mode.
8883 *
8884 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8885 * the common-state (TRPM/forceflags), we must undo those changes so that the
8886 * recompiler/IEM can (and should) use them when it resumes guest execution.
8887 * Otherwise such operations must be done when we can no longer exit to ring-3.
8888 *
8889 * @returns Strict VBox status code (i.e. informational status codes too).
8890 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8891 * have been disabled.
8892 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8893 * double-fault into the guest.
8894 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8895 * dispatched directly.
8896 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8897 *
8898 * @param pVM The cross context VM structure.
8899 * @param pVCpu The cross context virtual CPU structure.
8900 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8901 * out-of-sync. Make sure to update the required fields
8902 * before using them.
8903 * @param pVmxTransient Pointer to the VMX transient structure.
8904 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8905 * us ignore some of the reasons for returning to
8906 * ring-3, and return VINF_EM_DBG_STEPPED if event
8907 * dispatching took place.
8908 */
8909static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8910{
8911 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8912
8913#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8914 PGMRZDynMapFlushAutoSet(pVCpu);
8915#endif
8916
8917 /* Check force flag actions that might require us to go back to ring-3. */
8918 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8919 if (rcStrict == VINF_SUCCESS)
8920 { /* FFs doesn't get set all the time. */ }
8921 else
8922 return rcStrict;
8923
8924#ifndef IEM_VERIFICATION_MODE_FULL
8925 /*
8926 * Setup the virtualized-APIC accesses.
8927 *
8928 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8929 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8930 *
8931 * This is the reason we do it here and not in hmR0VmxLoadGuestState().
8932 */
8933 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8934 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8935 && PDMHasApic(pVM))
8936 {
8937 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8938 Assert(u64MsrApicBase);
8939 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8940
8941 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8942
8943 /* Unalias any existing mapping. */
8944 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8945 AssertRCReturn(rc, rc);
8946
8947 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8948 Log4(("hmR0VmxPreRunGuest: VCPU%u: Mapped HC APIC-access page at %#RGp\n", pVCpu->idCpu, GCPhysApicBase));
8949 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8950 AssertRCReturn(rc, rc);
8951
8952 /* Update the per-VCPU cache of the APIC base MSR. */
8953 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8954 }
8955#endif /* !IEM_VERIFICATION_MODE_FULL */
8956
8957 if (TRPMHasTrap(pVCpu))
8958 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8959 uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8960
8961 /*
8962 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8963 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8964 */
8965 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
8966 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8967 { /* likely */ }
8968 else
8969 {
8970 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8971 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8972 return rcStrict;
8973 }
8974
8975 /*
8976 * No longjmps to ring-3 from this point on!!!
8977 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8978 * This also disables flushing of the R0-logger instance (if any).
8979 */
8980 VMMRZCallRing3Disable(pVCpu);
8981
8982 /*
8983 * Load the guest state bits.
8984 *
8985 * We cannot perform longjmps while loading the guest state because we do not preserve the
8986 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8987 * CPU migration.
8988 *
8989 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8990 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8991 * Hence, loading of the guest state needs to be done -after- injection of events.
8992 */
8993 rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8994 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8995 { /* likely */ }
8996 else
8997 {
8998 VMMRZCallRing3Enable(pVCpu);
8999 return rcStrict;
9000 }
9001
9002 /*
9003 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
9004 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
9005 *
9006 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
9007 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
9008 *
9009 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
9010 * executing guest code.
9011 */
9012 pVmxTransient->fEFlags = ASMIntDisableFlags();
9013
9014 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
9015 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
9016 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
9017 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
9018 {
9019 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
9020 {
9021 pVCpu->hm.s.Event.fPending = false;
9022
9023 /*
9024 * We've injected any pending events. This is really the point of no return (to ring-3).
9025 *
9026 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
9027 * returns from this function, so don't enable them here.
9028 */
9029 return VINF_SUCCESS;
9030 }
9031
9032 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
9033 rcStrict = VINF_EM_RAW_INTERRUPT;
9034 }
9035 else
9036 {
9037 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
9038 rcStrict = VINF_EM_RAW_TO_R3;
9039 }
9040
9041 ASMSetFlags(pVmxTransient->fEFlags);
9042 VMMRZCallRing3Enable(pVCpu);
9043
9044 return rcStrict;
9045}
9046
9047
9048/**
9049 * Prepares to run guest code in VT-x and we've committed to doing so. This
9050 * means there is no backing out to ring-3 or anywhere else at this
9051 * point.
9052 *
9053 * @param pVM The cross context VM structure.
9054 * @param pVCpu The cross context virtual CPU structure.
9055 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9056 * out-of-sync. Make sure to update the required fields
9057 * before using them.
9058 * @param pVmxTransient Pointer to the VMX transient structure.
9059 *
9060 * @remarks Called with preemption disabled.
9061 * @remarks No-long-jump zone!!!
9062 */
9063static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9064{
9065 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9066 Assert(VMMR0IsLogFlushDisabled(pVCpu));
9067 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9068
9069 /*
9070 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
9071 */
9072 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
9073 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
9074
9075#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
9076 if (!CPUMIsGuestFPUStateActive(pVCpu))
9077 {
9078 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
9079 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
9080 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
9081 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
9082 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
9083 }
9084 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9085#endif
9086
9087 if ( pVCpu->hm.s.fPreloadGuestFpu
9088 && !CPUMIsGuestFPUStateActive(pVCpu))
9089 {
9090 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
9091 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
9092 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
9093 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
9094 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
9095 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
9096 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9097 }
9098
9099 /*
9100 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
9101 */
9102 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
9103 && pVCpu->hm.s.vmx.cMsrs > 0)
9104 {
9105 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
9106 }
9107
9108 /*
9109 * Load the host state bits as we may've been preempted (only happens when
9110 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
9111 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
9112 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
9113 * See @bugref{8432}.
9114 */
9115 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
9116 {
9117 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
9118 AssertRC(rc);
9119 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
9120 }
9121 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
9122
9123 /*
9124 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
9125 */
9126 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
9127 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
9128 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
9129
9130 /* Store status of the shared guest-host state at the time of VM-entry. */
9131#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
9132 if (CPUMIsGuestInLongModeEx(pMixedCtx))
9133 {
9134 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
9135 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
9136 }
9137 else
9138#endif
9139 {
9140 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
9141 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
9142 }
9143 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
9144
9145 /*
9146 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
9147 */
9148 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9149 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
9150
9151 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
9152 RTCPUID idCurrentCpu = pCpu->idCpu;
9153 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
9154 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
9155 {
9156 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
9157 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
9158 }
9159
9160 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
9161 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
9162 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
9163 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
9164
9165 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
9166
9167 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
9168 to start executing. */
9169
9170 /*
9171 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
9172 */
9173 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
9174 {
9175 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9176 {
9177 bool fMsrUpdated;
9178 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9179 AssertRC(rc2);
9180 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
9181
9182 rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
9183 &fMsrUpdated);
9184 AssertRC(rc2);
9185 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9186
9187 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
9188 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
9189 }
9190 else
9191 {
9192 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
9193 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9194 }
9195 }
9196
9197 if (pVM->cpum.ro.GuestFeatures.fIbrs)
9198 {
9199 bool fMsrUpdated;
9200 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9201 AssertRC(rc2);
9202 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
9203
9204 rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMR0GetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
9205 &fMsrUpdated);
9206 AssertRC(rc2);
9207 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9208 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
9209 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
9210 }
9211
9212#ifdef VBOX_STRICT
9213 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
9214 hmR0VmxCheckHostEferMsr(pVCpu);
9215 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
9216#endif
9217#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
9218 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
9219 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
9220 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
9221#endif
9222}
9223
9224
9225/**
9226 * Performs some essential restoration of state after running guest code in
9227 * VT-x.
9228 *
9229 * @param pVM The cross context VM structure.
9230 * @param pVCpu The cross context virtual CPU structure.
9231 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9232 * out-of-sync. Make sure to update the required fields
9233 * before using them.
9234 * @param pVmxTransient Pointer to the VMX transient structure.
9235 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
9236 *
9237 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
9238 *
9239 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
9240 * unconditionally when it is safe to do so.
9241 */
9242static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
9243{
9244 NOREF(pVM);
9245
9246 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9247
9248 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
9249 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
9250 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
9251 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
9252 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
9253 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
9254
9255 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9256 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
9257
9258 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
9259 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
9260 Assert(!ASMIntAreEnabled());
9261 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
9262
9263#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
9264 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVM, pVCpu))
9265 {
9266 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9267 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9268 }
9269#endif
9270
9271#if HC_ARCH_BITS == 64
9272 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
9273#endif
9274#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
9275 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
9276 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
9277 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
9278#else
9279 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
9280#endif
9281#ifdef VBOX_STRICT
9282 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
9283#endif
9284 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
9285
9286 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
9287 uint32_t uExitReason;
9288 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
9289 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9290 AssertRC(rc);
9291 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
9292 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
9293
9294 if (rcVMRun == VINF_SUCCESS)
9295 {
9296 /*
9297 * Update the VM-exit history array here even if the VM-entry failed due to:
9298 * - Invalid guest state.
9299 * - MSR loading.
9300 * - Machine-check event.
9301 *
9302 * In any of the above cases we will still have a "valid" VM-exit reason
9303 * despite @a fVMEntryFailed being false.
9304 *
9305 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
9306 */
9307 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
9308
9309 if (!pVmxTransient->fVMEntryFailed)
9310 {
9311 /** @todo We can optimize this by only syncing with our force-flags when
9312 * really needed and keeping the VMCS state as it is for most
9313 * VM-exits. */
9314 /* Update the guest interruptibility-state from the VMCS. */
9315 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
9316
9317 /*
9318 * Allow longjmps to ring-3 -after- saving the guest-interruptibility state
9319 * as it's not part of hmR0VmxSaveGuestState() and thus would trigger an assertion
9320 * on the longjmp path to ring-3 while saving the (rest of) the guest state,
9321 * see @bugref{6208#c63}.
9322 */
9323 VMMRZCallRing3Enable(pVCpu);
9324
9325#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
9326 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9327 AssertRC(rc);
9328#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
9329 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9330 AssertRC(rc);
9331#endif
9332
9333 /*
9334 * Sync the TPR shadow with our APIC state.
9335 */
9336 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9337 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
9338 {
9339 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
9340 AssertRC(rc);
9341 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_APIC_STATE);
9342 }
9343
9344 return;
9345 }
9346 }
9347 else
9348 {
9349 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
9350 pVmxTransient->fVMEntryFailed));
9351 }
9352
9353 VMMRZCallRing3Enable(pVCpu);
9354}
9355
9356
9357/**
9358 * Runs the guest code using VT-x the normal way.
9359 *
9360 * @returns VBox status code.
9361 * @param pVM The cross context VM structure.
9362 * @param pVCpu The cross context virtual CPU structure.
9363 * @param pCtx Pointer to the guest-CPU context.
9364 *
9365 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
9366 */
9367static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9368{
9369 VMXTRANSIENT VmxTransient;
9370 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9371 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9372 uint32_t cLoops = 0;
9373
9374 for (;; cLoops++)
9375 {
9376 Assert(!HMR0SuspendPending());
9377 HMVMX_ASSERT_CPU_SAFE();
9378
9379 /* Preparatory work for running guest code, this may force us to return
9380 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
9381 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9382 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
9383 if (rcStrict != VINF_SUCCESS)
9384 break;
9385
9386 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9387 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9388 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9389
9390 /* Restore any residual host-state and save any bits shared between host
9391 and guest into the guest-CPU state. Re-enables interrupts! */
9392 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
9393
9394 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9395 if (RT_SUCCESS(rcRun))
9396 { /* very likely */ }
9397 else
9398 {
9399 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9400 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9401 return rcRun;
9402 }
9403
9404 /* Profile the VM-exit. */
9405 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9406 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9407 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9408 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9409 HMVMX_START_EXIT_DISPATCH_PROF();
9410
9411 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9412
9413 /* Handle the VM-exit. */
9414#ifdef HMVMX_USE_FUNCTION_TABLE
9415 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
9416#else
9417 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
9418#endif
9419 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9420 if (rcStrict == VINF_SUCCESS)
9421 {
9422 if (cLoops <= pVM->hm.s.cMaxResumeLoops)
9423 continue; /* likely */
9424 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9425 rcStrict = VINF_EM_RAW_INTERRUPT;
9426 }
9427 break;
9428 }
9429
9430 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9431 return rcStrict;
9432}
9433
9434
9435
9436/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
9437 * probes.
9438 *
9439 * The following few functions and associated structure contains the bloat
9440 * necessary for providing detailed debug events and dtrace probes as well as
9441 * reliable host side single stepping. This works on the principle of
9442 * "subclassing" the normal execution loop and workers. We replace the loop
9443 * method completely and override selected helpers to add necessary adjustments
9444 * to their core operation.
9445 *
9446 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
9447 * any performance for debug and analysis features.
9448 *
9449 * @{
9450 */
9451
9452/**
9453 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
9454 * the debug run loop.
9455 */
9456typedef struct VMXRUNDBGSTATE
9457{
9458 /** The RIP we started executing at. This is for detecting that we stepped. */
9459 uint64_t uRipStart;
9460 /** The CS we started executing with. */
9461 uint16_t uCsStart;
9462
9463 /** Whether we've actually modified the 1st execution control field. */
9464 bool fModifiedProcCtls : 1;
9465 /** Whether we've actually modified the 2nd execution control field. */
9466 bool fModifiedProcCtls2 : 1;
9467 /** Whether we've actually modified the exception bitmap. */
9468 bool fModifiedXcptBitmap : 1;
9469
9470 /** We desire the modified the CR0 mask to be cleared. */
9471 bool fClearCr0Mask : 1;
9472 /** We desire the modified the CR4 mask to be cleared. */
9473 bool fClearCr4Mask : 1;
9474 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9475 uint32_t fCpe1Extra;
9476 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9477 uint32_t fCpe1Unwanted;
9478 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9479 uint32_t fCpe2Extra;
9480 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9481 uint32_t bmXcptExtra;
9482 /** The sequence number of the Dtrace provider settings the state was
9483 * configured against. */
9484 uint32_t uDtraceSettingsSeqNo;
9485 /** VM-exits to check (one bit per VM-exit). */
9486 uint32_t bmExitsToCheck[3];
9487
9488 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9489 uint32_t fProcCtlsInitial;
9490 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9491 uint32_t fProcCtls2Initial;
9492 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9493 uint32_t bmXcptInitial;
9494} VMXRUNDBGSTATE;
9495AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9496typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9497
9498
9499/**
9500 * Initializes the VMXRUNDBGSTATE structure.
9501 *
9502 * @param pVCpu The cross context virtual CPU structure of the
9503 * calling EMT.
9504 * @param pCtx The CPU register context to go with @a pVCpu.
9505 * @param pDbgState The structure to initialize.
9506 */
9507DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9508{
9509 pDbgState->uRipStart = pCtx->rip;
9510 pDbgState->uCsStart = pCtx->cs.Sel;
9511
9512 pDbgState->fModifiedProcCtls = false;
9513 pDbgState->fModifiedProcCtls2 = false;
9514 pDbgState->fModifiedXcptBitmap = false;
9515 pDbgState->fClearCr0Mask = false;
9516 pDbgState->fClearCr4Mask = false;
9517 pDbgState->fCpe1Extra = 0;
9518 pDbgState->fCpe1Unwanted = 0;
9519 pDbgState->fCpe2Extra = 0;
9520 pDbgState->bmXcptExtra = 0;
9521 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9522 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9523 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9524}
9525
9526
9527/**
9528 * Updates the VMSC fields with changes requested by @a pDbgState.
9529 *
9530 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9531 * immediately before executing guest code, i.e. when interrupts are disabled.
9532 * We don't check status codes here as we cannot easily assert or return in the
9533 * latter case.
9534 *
9535 * @param pVCpu The cross context virtual CPU structure.
9536 * @param pDbgState The debug state.
9537 */
9538DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9539{
9540 /*
9541 * Ensure desired flags in VMCS control fields are set.
9542 * (Ignoring write failure here, as we're committed and it's just debug extras.)
9543 *
9544 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9545 * there should be no stale data in pCtx at this point.
9546 */
9547 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9548 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9549 {
9550 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
9551 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9552 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9553 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9554 pDbgState->fModifiedProcCtls = true;
9555 }
9556
9557 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9558 {
9559 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
9560 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9561 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9562 pDbgState->fModifiedProcCtls2 = true;
9563 }
9564
9565 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9566 {
9567 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
9568 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9569 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9570 pDbgState->fModifiedXcptBitmap = true;
9571 }
9572
9573 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9574 {
9575 pVCpu->hm.s.vmx.u32CR0Mask = 0;
9576 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9577 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9578 }
9579
9580 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9581 {
9582 pVCpu->hm.s.vmx.u32CR4Mask = 0;
9583 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9584 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9585 }
9586}
9587
9588
9589DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9590{
9591 /*
9592 * Restore VM-exit control settings as we may not reenter this function the
9593 * next time around.
9594 */
9595 /* We reload the initial value, trigger what we can of recalculations the
9596 next time around. From the looks of things, that's all that's required atm. */
9597 if (pDbgState->fModifiedProcCtls)
9598 {
9599 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9600 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9601 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9602 AssertRCReturn(rc2, rc2);
9603 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9604 }
9605
9606 /* We're currently the only ones messing with this one, so just restore the
9607 cached value and reload the field. */
9608 if ( pDbgState->fModifiedProcCtls2
9609 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9610 {
9611 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9612 AssertRCReturn(rc2, rc2);
9613 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9614 }
9615
9616 /* If we've modified the exception bitmap, we restore it and trigger
9617 reloading and partial recalculation the next time around. */
9618 if (pDbgState->fModifiedXcptBitmap)
9619 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9620
9621 return rcStrict;
9622}
9623
9624
9625/**
9626 * Configures VM-exit controls for current DBGF and DTrace settings.
9627 *
9628 * This updates @a pDbgState and the VMCS execution control fields to reflect
9629 * the necessary VM-exits demanded by DBGF and DTrace.
9630 *
9631 * @param pVM The cross context VM structure.
9632 * @param pVCpu The cross context virtual CPU structure.
9633 * @param pCtx Pointer to the guest-CPU context.
9634 * @param pDbgState The debug state.
9635 * @param pVmxTransient Pointer to the VMX transient structure. May update
9636 * fUpdateTscOffsettingAndPreemptTimer.
9637 */
9638static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9639 PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9640{
9641 /*
9642 * Take down the dtrace serial number so we can spot changes.
9643 */
9644 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9645 ASMCompilerBarrier();
9646
9647 /*
9648 * We'll rebuild most of the middle block of data members (holding the
9649 * current settings) as we go along here, so start by clearing it all.
9650 */
9651 pDbgState->bmXcptExtra = 0;
9652 pDbgState->fCpe1Extra = 0;
9653 pDbgState->fCpe1Unwanted = 0;
9654 pDbgState->fCpe2Extra = 0;
9655 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9656 pDbgState->bmExitsToCheck[i] = 0;
9657
9658 /*
9659 * Software interrupts (INT XXh) - no idea how to trigger these...
9660 */
9661 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9662 || VBOXVMM_INT_SOFTWARE_ENABLED())
9663 {
9664 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9665 }
9666
9667 /*
9668 * INT3 breakpoints - triggered by #BP exceptions.
9669 */
9670 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9671 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9672
9673 /*
9674 * Exception bitmap and XCPT events+probes.
9675 */
9676 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9677 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9678 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9679
9680 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9681 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9682 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9683 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9684 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9685 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9686 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9687 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9688 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9689 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9690 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9691 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9692 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9693 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9694 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9695 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9696 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9697 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9698
9699 if (pDbgState->bmXcptExtra)
9700 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9701
9702 /*
9703 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9704 *
9705 * Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9706 * So, when adding/changing/removing please don't forget to update it.
9707 *
9708 * Some of the macros are picking up local variables to save horizontal space,
9709 * (being able to see it in a table is the lesser evil here).
9710 */
9711#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9712 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9713 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9714#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9715 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9716 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9717 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9718 } else do { } while (0)
9719#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9720 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9721 { \
9722 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9723 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9724 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9725 } else do { } while (0)
9726#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9727 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9728 { \
9729 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9730 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9731 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9732 } else do { } while (0)
9733#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9734 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9735 { \
9736 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9737 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9738 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9739 } else do { } while (0)
9740
9741 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9742 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9743 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9744 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9745 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9746
9747 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9748 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9749 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9750 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9751 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9752 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9753 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9754 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9755 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9756 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9757 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9758 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9759 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9760 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9761 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9762 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9763 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9764 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9765 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9766 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9767 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9768 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9769 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9770 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9771 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9772 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9773 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9774 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9775 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9776 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9777 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9778 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9779 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9780 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9781 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9782 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9783
9784 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9785 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9786 {
9787 int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9788 rc2 |= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9789 rc2 |= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9790 AssertRC(rc2);
9791
9792#if 0 /** @todo fix me */
9793 pDbgState->fClearCr0Mask = true;
9794 pDbgState->fClearCr4Mask = true;
9795#endif
9796 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9797 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9798 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9799 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9800 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9801 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9802 require clearing here and in the loop if we start using it. */
9803 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9804 }
9805 else
9806 {
9807 if (pDbgState->fClearCr0Mask)
9808 {
9809 pDbgState->fClearCr0Mask = false;
9810 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9811 }
9812 if (pDbgState->fClearCr4Mask)
9813 {
9814 pDbgState->fClearCr4Mask = false;
9815 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9816 }
9817 }
9818 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9819 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9820
9821 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9822 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9823 {
9824 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9825 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9826 }
9827 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9828 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9829
9830 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9831 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9832 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9833 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9834 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9835 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9836 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9837 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9838#if 0 /** @todo too slow, fix handler. */
9839 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9840#endif
9841 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9842
9843 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9844 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9845 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9846 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9847 {
9848 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9849 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9850 }
9851 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9852 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9853 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9854 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9855
9856 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9857 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9858 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9859 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9860 {
9861 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9862 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9863 }
9864 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9865 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9866 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9867 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9868
9869 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9870 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9871 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9872 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9873 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9874 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9875 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9876 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9877 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9878 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9879 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9880 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9881 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9882 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9883 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9884 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9885 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9886 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9887 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9888 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9889 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9890 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9891
9892#undef IS_EITHER_ENABLED
9893#undef SET_ONLY_XBM_IF_EITHER_EN
9894#undef SET_CPE1_XBM_IF_EITHER_EN
9895#undef SET_CPEU_XBM_IF_EITHER_EN
9896#undef SET_CPE2_XBM_IF_EITHER_EN
9897
9898 /*
9899 * Sanitize the control stuff.
9900 */
9901 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9902 if (pDbgState->fCpe2Extra)
9903 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9904 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9905 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9906 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9907 {
9908 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9909 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9910 }
9911
9912 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9913 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9914 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9915 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9916}
9917
9918
9919/**
9920 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9921 * appropriate.
9922 *
9923 * The caller has checked the VM-exit against the
9924 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9925 * already, so we don't have to do that either.
9926 *
9927 * @returns Strict VBox status code (i.e. informational status codes too).
9928 * @param pVM The cross context VM structure.
9929 * @param pVCpu The cross context virtual CPU structure.
9930 * @param pMixedCtx Pointer to the guest-CPU context.
9931 * @param pVmxTransient Pointer to the VMX-transient structure.
9932 * @param uExitReason The VM-exit reason.
9933 *
9934 * @remarks The name of this function is displayed by dtrace, so keep it short
9935 * and to the point. No longer than 33 chars long, please.
9936 */
9937static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9938 PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9939{
9940 /*
9941 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9942 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9943 *
9944 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9945 * does. Must add/change/remove both places. Same ordering, please.
9946 *
9947 * Added/removed events must also be reflected in the next section
9948 * where we dispatch dtrace events.
9949 */
9950 bool fDtrace1 = false;
9951 bool fDtrace2 = false;
9952 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9953 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9954 uint32_t uEventArg = 0;
9955#define SET_EXIT(a_EventSubName) \
9956 do { \
9957 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9958 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9959 } while (0)
9960#define SET_BOTH(a_EventSubName) \
9961 do { \
9962 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9963 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9964 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9965 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9966 } while (0)
9967 switch (uExitReason)
9968 {
9969 case VMX_EXIT_MTF:
9970 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9971
9972 case VMX_EXIT_XCPT_OR_NMI:
9973 {
9974 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9975 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9976 {
9977 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9978 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9979 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9980 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9981 {
9982 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9983 {
9984 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9985 uEventArg = pVmxTransient->uExitIntErrorCode;
9986 }
9987 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9988 switch (enmEvent1)
9989 {
9990 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9991 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9992 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9993 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9994 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9995 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9996 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9997 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9998 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9999 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
10000 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
10001 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
10002 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
10003 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
10004 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
10005 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
10006 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
10007 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
10008 default: break;
10009 }
10010 }
10011 else
10012 AssertFailed();
10013 break;
10014
10015 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
10016 uEventArg = idxVector;
10017 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
10018 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
10019 break;
10020 }
10021 break;
10022 }
10023
10024 case VMX_EXIT_TRIPLE_FAULT:
10025 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
10026 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
10027 break;
10028 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
10029 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
10030 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
10031 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
10032 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
10033
10034 /* Instruction specific VM-exits: */
10035 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
10036 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
10037 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
10038 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
10039 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
10040 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
10041 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
10042 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
10043 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
10044 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
10045 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
10046 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
10047 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
10048 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
10049 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
10050 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
10051 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
10052 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
10053 case VMX_EXIT_MOV_CRX:
10054 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10055/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
10056* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
10057 if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
10058 == VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
10059 SET_BOTH(CRX_READ);
10060 else
10061 SET_BOTH(CRX_WRITE);
10062 uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
10063 break;
10064 case VMX_EXIT_MOV_DRX:
10065 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10066 if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
10067 == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
10068 SET_BOTH(DRX_READ);
10069 else
10070 SET_BOTH(DRX_WRITE);
10071 uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
10072 break;
10073 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
10074 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
10075 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
10076 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
10077 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
10078 case VMX_EXIT_XDTR_ACCESS:
10079 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10080 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
10081 {
10082 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
10083 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
10084 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
10085 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
10086 }
10087 break;
10088
10089 case VMX_EXIT_TR_ACCESS:
10090 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10091 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
10092 {
10093 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
10094 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
10095 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
10096 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
10097 }
10098 break;
10099
10100 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
10101 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
10102 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
10103 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
10104 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
10105 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
10106 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
10107 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
10108 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
10109 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
10110 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
10111
10112 /* Events that aren't relevant at this point. */
10113 case VMX_EXIT_EXT_INT:
10114 case VMX_EXIT_INT_WINDOW:
10115 case VMX_EXIT_NMI_WINDOW:
10116 case VMX_EXIT_TPR_BELOW_THRESHOLD:
10117 case VMX_EXIT_PREEMPT_TIMER:
10118 case VMX_EXIT_IO_INSTR:
10119 break;
10120
10121 /* Errors and unexpected events. */
10122 case VMX_EXIT_INIT_SIGNAL:
10123 case VMX_EXIT_SIPI:
10124 case VMX_EXIT_IO_SMI:
10125 case VMX_EXIT_SMI:
10126 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10127 case VMX_EXIT_ERR_MSR_LOAD:
10128 case VMX_EXIT_ERR_MACHINE_CHECK:
10129 break;
10130
10131 default:
10132 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10133 break;
10134 }
10135#undef SET_BOTH
10136#undef SET_EXIT
10137
10138 /*
10139 * Dtrace tracepoints go first. We do them here at once so we don't
10140 * have to copy the guest state saving and stuff a few dozen times.
10141 * Down side is that we've got to repeat the switch, though this time
10142 * we use enmEvent since the probes are a subset of what DBGF does.
10143 */
10144 if (fDtrace1 || fDtrace2)
10145 {
10146 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10147 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10148 switch (enmEvent1)
10149 {
10150 /** @todo consider which extra parameters would be helpful for each probe. */
10151 case DBGFEVENT_END: break;
10152 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
10153 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
10154 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
10155 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
10156 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
10157 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
10158 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
10159 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
10160 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
10161 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
10162 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
10163 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
10164 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
10165 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
10166 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
10167 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
10168 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
10169 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
10170 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10171 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
10172 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
10173 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
10174 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
10175 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
10176 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
10177 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
10178 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
10179 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10180 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10181 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10182 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10183 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
10184 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
10185 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
10186 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
10187 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
10188 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
10189 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
10190 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
10191 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
10192 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
10193 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
10194 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
10195 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
10196 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
10197 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
10198 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
10199 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
10200 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
10201 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
10202 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
10203 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
10204 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
10205 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
10206 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
10207 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
10208 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
10209 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
10210 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
10211 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
10212 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
10213 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
10214 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
10215 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
10216 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
10217 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
10218 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
10219 }
10220 switch (enmEvent2)
10221 {
10222 /** @todo consider which extra parameters would be helpful for each probe. */
10223 case DBGFEVENT_END: break;
10224 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
10225 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
10226 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
10227 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
10228 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
10229 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
10230 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
10231 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
10232 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
10233 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10234 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10235 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10236 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10237 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
10238 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
10239 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
10240 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
10241 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
10242 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
10243 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
10244 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
10245 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
10246 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
10247 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
10248 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
10249 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
10250 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
10251 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
10252 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
10253 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
10254 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
10255 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
10256 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
10257 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
10258 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
10259 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
10260 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
10261 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
10262 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
10263 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
10264 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
10265 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
10266 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
10267 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
10268 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
10269 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
10270 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
10271 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
10272 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
10273 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
10274 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
10275 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
10276 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
10277 }
10278 }
10279
10280 /*
10281 * Fire of the DBGF event, if enabled (our check here is just a quick one,
10282 * the DBGF call will do a full check).
10283 *
10284 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
10285 * Note! If we have to events, we prioritize the first, i.e. the instruction
10286 * one, in order to avoid event nesting.
10287 */
10288 if ( enmEvent1 != DBGFEVENT_END
10289 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
10290 {
10291 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
10292 if (rcStrict != VINF_SUCCESS)
10293 return rcStrict;
10294 }
10295 else if ( enmEvent2 != DBGFEVENT_END
10296 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
10297 {
10298 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
10299 if (rcStrict != VINF_SUCCESS)
10300 return rcStrict;
10301 }
10302
10303 return VINF_SUCCESS;
10304}
10305
10306
10307/**
10308 * Single-stepping VM-exit filtering.
10309 *
10310 * This is preprocessing the VM-exits and deciding whether we've gotten far
10311 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
10312 * handling is performed.
10313 *
10314 * @returns Strict VBox status code (i.e. informational status codes too).
10315 * @param pVM The cross context VM structure.
10316 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
10317 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
10318 * out-of-sync. Make sure to update the required
10319 * fields before using them.
10320 * @param pVmxTransient Pointer to the VMX-transient structure.
10321 * @param uExitReason The VM-exit reason.
10322 * @param pDbgState The debug state.
10323 */
10324DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
10325 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
10326{
10327 /*
10328 * Expensive (saves context) generic dtrace VM-exit probe.
10329 */
10330 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
10331 { /* more likely */ }
10332 else
10333 {
10334 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10335 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10336 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
10337 }
10338
10339 /*
10340 * Check for host NMI, just to get that out of the way.
10341 */
10342 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
10343 { /* normally likely */ }
10344 else
10345 {
10346 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10347 AssertRCReturn(rc2, rc2);
10348 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10349 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10350 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
10351 }
10352
10353 /*
10354 * Check for single stepping event if we're stepping.
10355 */
10356 if (pVCpu->hm.s.fSingleInstruction)
10357 {
10358 switch (uExitReason)
10359 {
10360 case VMX_EXIT_MTF:
10361 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
10362
10363 /* Various events: */
10364 case VMX_EXIT_XCPT_OR_NMI:
10365 case VMX_EXIT_EXT_INT:
10366 case VMX_EXIT_TRIPLE_FAULT:
10367 case VMX_EXIT_INT_WINDOW:
10368 case VMX_EXIT_NMI_WINDOW:
10369 case VMX_EXIT_TASK_SWITCH:
10370 case VMX_EXIT_TPR_BELOW_THRESHOLD:
10371 case VMX_EXIT_APIC_ACCESS:
10372 case VMX_EXIT_EPT_VIOLATION:
10373 case VMX_EXIT_EPT_MISCONFIG:
10374 case VMX_EXIT_PREEMPT_TIMER:
10375
10376 /* Instruction specific VM-exits: */
10377 case VMX_EXIT_CPUID:
10378 case VMX_EXIT_GETSEC:
10379 case VMX_EXIT_HLT:
10380 case VMX_EXIT_INVD:
10381 case VMX_EXIT_INVLPG:
10382 case VMX_EXIT_RDPMC:
10383 case VMX_EXIT_RDTSC:
10384 case VMX_EXIT_RSM:
10385 case VMX_EXIT_VMCALL:
10386 case VMX_EXIT_VMCLEAR:
10387 case VMX_EXIT_VMLAUNCH:
10388 case VMX_EXIT_VMPTRLD:
10389 case VMX_EXIT_VMPTRST:
10390 case VMX_EXIT_VMREAD:
10391 case VMX_EXIT_VMRESUME:
10392 case VMX_EXIT_VMWRITE:
10393 case VMX_EXIT_VMXOFF:
10394 case VMX_EXIT_VMXON:
10395 case VMX_EXIT_MOV_CRX:
10396 case VMX_EXIT_MOV_DRX:
10397 case VMX_EXIT_IO_INSTR:
10398 case VMX_EXIT_RDMSR:
10399 case VMX_EXIT_WRMSR:
10400 case VMX_EXIT_MWAIT:
10401 case VMX_EXIT_MONITOR:
10402 case VMX_EXIT_PAUSE:
10403 case VMX_EXIT_XDTR_ACCESS:
10404 case VMX_EXIT_TR_ACCESS:
10405 case VMX_EXIT_INVEPT:
10406 case VMX_EXIT_RDTSCP:
10407 case VMX_EXIT_INVVPID:
10408 case VMX_EXIT_WBINVD:
10409 case VMX_EXIT_XSETBV:
10410 case VMX_EXIT_RDRAND:
10411 case VMX_EXIT_INVPCID:
10412 case VMX_EXIT_VMFUNC:
10413 case VMX_EXIT_RDSEED:
10414 case VMX_EXIT_XSAVES:
10415 case VMX_EXIT_XRSTORS:
10416 {
10417 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10418 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10419 AssertRCReturn(rc2, rc2);
10420 if ( pMixedCtx->rip != pDbgState->uRipStart
10421 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
10422 return VINF_EM_DBG_STEPPED;
10423 break;
10424 }
10425
10426 /* Errors and unexpected events: */
10427 case VMX_EXIT_INIT_SIGNAL:
10428 case VMX_EXIT_SIPI:
10429 case VMX_EXIT_IO_SMI:
10430 case VMX_EXIT_SMI:
10431 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10432 case VMX_EXIT_ERR_MSR_LOAD:
10433 case VMX_EXIT_ERR_MACHINE_CHECK:
10434 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
10435 break;
10436
10437 default:
10438 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10439 break;
10440 }
10441 }
10442
10443 /*
10444 * Check for debugger event breakpoints and dtrace probes.
10445 */
10446 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
10447 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
10448 {
10449 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10450 if (rcStrict != VINF_SUCCESS)
10451 return rcStrict;
10452 }
10453
10454 /*
10455 * Normal processing.
10456 */
10457#ifdef HMVMX_USE_FUNCTION_TABLE
10458 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
10459#else
10460 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10461#endif
10462}
10463
10464
10465/**
10466 * Single steps guest code using VT-x.
10467 *
10468 * @returns Strict VBox status code (i.e. informational status codes too).
10469 * @param pVM The cross context VM structure.
10470 * @param pVCpu The cross context virtual CPU structure.
10471 * @param pCtx Pointer to the guest-CPU context.
10472 *
10473 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10474 */
10475static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10476{
10477 VMXTRANSIENT VmxTransient;
10478 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10479
10480 /* Set HMCPU indicators. */
10481 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10482 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
10483 pVCpu->hm.s.fDebugWantRdTscExit = false;
10484 pVCpu->hm.s.fUsingDebugLoop = true;
10485
10486 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10487 VMXRUNDBGSTATE DbgState;
10488 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10489 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10490
10491 /*
10492 * The loop.
10493 */
10494 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10495 for (uint32_t cLoops = 0; ; cLoops++)
10496 {
10497 Assert(!HMR0SuspendPending());
10498 HMVMX_ASSERT_CPU_SAFE();
10499 bool fStepping = pVCpu->hm.s.fSingleInstruction;
10500
10501 /*
10502 * Preparatory work for running guest code, this may force us to return
10503 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10504 */
10505 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10506 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10507 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10508 if (rcStrict != VINF_SUCCESS)
10509 break;
10510
10511 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10512 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10513
10514 /*
10515 * Now we can run the guest code.
10516 */
10517 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10518
10519 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10520
10521 /*
10522 * Restore any residual host-state and save any bits shared between host
10523 * and guest into the guest-CPU state. Re-enables interrupts!
10524 */
10525 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
10526
10527 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10528 if (RT_SUCCESS(rcRun))
10529 { /* very likely */ }
10530 else
10531 {
10532 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10533 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10534 return rcRun;
10535 }
10536
10537 /* Profile the VM-exit. */
10538 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10539 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10540 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10541 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10542 HMVMX_START_EXIT_DISPATCH_PROF();
10543
10544 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10545
10546 /*
10547 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10548 */
10549 rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10550 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10551 if (rcStrict != VINF_SUCCESS)
10552 break;
10553 if (cLoops > pVM->hm.s.cMaxResumeLoops)
10554 {
10555 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10556 rcStrict = VINF_EM_RAW_INTERRUPT;
10557 break;
10558 }
10559
10560 /*
10561 * Stepping: Did the RIP change, if so, consider it a single step.
10562 * Otherwise, make sure one of the TFs gets set.
10563 */
10564 if (fStepping)
10565 {
10566 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10567 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10568 AssertRCReturn(rc2, rc2);
10569 if ( pCtx->rip != DbgState.uRipStart
10570 || pCtx->cs.Sel != DbgState.uCsStart)
10571 {
10572 rcStrict = VINF_EM_DBG_STEPPED;
10573 break;
10574 }
10575 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10576 }
10577
10578 /*
10579 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
10580 */
10581 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10582 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10583 }
10584
10585 /*
10586 * Clear the X86_EFL_TF if necessary.
10587 */
10588 if (pVCpu->hm.s.fClearTrapFlag)
10589 {
10590 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10591 AssertRCReturn(rc2, rc2);
10592 pVCpu->hm.s.fClearTrapFlag = false;
10593 pCtx->eflags.Bits.u1TF = 0;
10594 }
10595 /** @todo there seems to be issues with the resume flag when the monitor trap
10596 * flag is pending without being used. Seen early in bios init when
10597 * accessing APIC page in protected mode. */
10598
10599 /*
10600 * Restore VM-exit control settings as we may not reenter this function the
10601 * next time around.
10602 */
10603 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10604
10605 /* Restore HMCPU indicators. */
10606 pVCpu->hm.s.fUsingDebugLoop = false;
10607 pVCpu->hm.s.fDebugWantRdTscExit = false;
10608 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10609
10610 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10611 return rcStrict;
10612}
10613
10614
10615/** @} */
10616
10617
10618/**
10619 * Checks if any expensive dtrace probes are enabled and we should go to the
10620 * debug loop.
10621 *
10622 * @returns true if we should use debug loop, false if not.
10623 */
10624static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10625{
10626 /* It's probably faster to OR the raw 32-bit counter variables together.
10627 Since the variables are in an array and the probes are next to one
10628 another (more or less), we have good locality. So, better read
10629 eight-nine cache lines ever time and only have one conditional, than
10630 128+ conditionals, right? */
10631 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10632 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10633 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10634 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10635 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10636 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10637 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10638 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10639 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10640 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10641 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10642 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10643 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10644 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10645 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10646 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10647 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10648 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10649 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10650 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10651 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10652 ) != 0
10653 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10654 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10655 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10656 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10657 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10658 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10659 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10660 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10661 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10662 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10663 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10664 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10665 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10666 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10667 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10668 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10669 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10670 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10671 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10672 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10673 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10674 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10675 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10676 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10677 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10678 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10679 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10680 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10681 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10682 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10683 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10684 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10685 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10686 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10687 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10688 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10689 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10690 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10691 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10692 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10693 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10694 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10695 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10696 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10697 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10698 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10699 ) != 0
10700 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10701 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10702 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10703 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10704 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10705 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10706 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10707 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10708 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10709 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10710 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10711 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10712 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10713 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10714 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10715 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10716 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10717 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10718 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10719 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10720 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10721 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10722 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10723 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10724 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10725 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10726 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10727 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10728 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10729 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10730 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10731 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10732 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10733 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10734 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10735 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10736 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10737 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10738 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10739 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10740 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10741 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10742 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10743 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10744 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10745 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10746 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10747 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10748 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10749 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10750 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10751 ) != 0;
10752}
10753
10754
10755/**
10756 * Runs the guest code using VT-x.
10757 *
10758 * @returns Strict VBox status code (i.e. informational status codes too).
10759 * @param pVM The cross context VM structure.
10760 * @param pVCpu The cross context virtual CPU structure.
10761 * @param pCtx Pointer to the guest-CPU context.
10762 */
10763VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10764{
10765 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10766 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10767 HMVMX_ASSERT_PREEMPT_SAFE();
10768
10769 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10770
10771 VBOXSTRICTRC rcStrict;
10772 if ( !pVCpu->hm.s.fUseDebugLoop
10773 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10774 && !DBGFIsStepping(pVCpu)
10775 && !pVM->dbgf.ro.cEnabledInt3Breakpoints)
10776 rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10777 else
10778 rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10779
10780 if (rcStrict == VERR_EM_INTERPRETER)
10781 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10782 else if (rcStrict == VINF_EM_RESET)
10783 rcStrict = VINF_EM_TRIPLE_FAULT;
10784
10785 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10786 if (RT_FAILURE(rc2))
10787 {
10788 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10789 rcStrict = rc2;
10790 }
10791 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10792 return rcStrict;
10793}
10794
10795
10796#ifndef HMVMX_USE_FUNCTION_TABLE
10797DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10798{
10799# ifdef DEBUG_ramshankar
10800# define RETURN_EXIT_CALL(a_CallExpr) \
10801 do { \
10802 int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10803 VBOXSTRICTRC rcStrict = a_CallExpr; \
10804 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10805 return rcStrict; \
10806 } while (0)
10807# else
10808# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10809# endif
10810 switch (rcReason)
10811 {
10812 case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10813 case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10814 case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10815 case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10816 case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10817 case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10818 case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10819 case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10820 case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10821 case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10822 case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10823 case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10824 case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10825 case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10826 case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10827 case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10828 case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10829 case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10830 case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10831 case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10832 case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10833 case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10834 case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10835 case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10836 case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10837 case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10838 case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10839 case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10840 case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10841 case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10842 case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10843 case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10844 case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10845 case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10846
10847 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10848 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10849 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10850 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10851 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10852 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10853 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10854 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10855 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10856
10857 case VMX_EXIT_VMCLEAR:
10858 case VMX_EXIT_VMLAUNCH:
10859 case VMX_EXIT_VMPTRLD:
10860 case VMX_EXIT_VMPTRST:
10861 case VMX_EXIT_VMREAD:
10862 case VMX_EXIT_VMRESUME:
10863 case VMX_EXIT_VMWRITE:
10864 case VMX_EXIT_VMXOFF:
10865 case VMX_EXIT_VMXON:
10866 case VMX_EXIT_INVEPT:
10867 case VMX_EXIT_INVVPID:
10868 case VMX_EXIT_VMFUNC:
10869 case VMX_EXIT_XSAVES:
10870 case VMX_EXIT_XRSTORS:
10871 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10872 case VMX_EXIT_ENCLS:
10873 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10874 case VMX_EXIT_PML_FULL:
10875 default:
10876 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10877 }
10878#undef RETURN_EXIT_CALL
10879}
10880#endif /* !HMVMX_USE_FUNCTION_TABLE */
10881
10882
10883#ifdef VBOX_STRICT
10884/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10885# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10886 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10887
10888# define HMVMX_ASSERT_PREEMPT_CPUID() \
10889 do { \
10890 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10891 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10892 } while (0)
10893
10894# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10895 do { \
10896 AssertPtr(pVCpu); \
10897 AssertPtr(pMixedCtx); \
10898 AssertPtr(pVmxTransient); \
10899 Assert(pVmxTransient->fVMEntryFailed == false); \
10900 Assert(ASMIntAreEnabled()); \
10901 HMVMX_ASSERT_PREEMPT_SAFE(); \
10902 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10903 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10904 HMVMX_ASSERT_PREEMPT_SAFE(); \
10905 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10906 HMVMX_ASSERT_PREEMPT_CPUID(); \
10907 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10908 } while (0)
10909
10910# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10911 do { \
10912 Log4Func(("\n")); \
10913 } while (0)
10914#else /* nonstrict builds: */
10915# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10916 do { \
10917 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10918 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10919 } while (0)
10920# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10921#endif
10922
10923
10924/**
10925 * Advances the guest RIP by the specified number of bytes.
10926 *
10927 * @param pVCpu The cross context virtual CPU structure.
10928 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10929 * out-of-sync. Make sure to update the required fields
10930 * before using them.
10931 * @param cbInstr Number of bytes to advance the RIP by.
10932 *
10933 * @remarks No-long-jump zone!!!
10934 */
10935DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10936{
10937 /* Advance the RIP. */
10938 pMixedCtx->rip += cbInstr;
10939 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10940
10941 /* Update interrupt inhibition. */
10942 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10943 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10944 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10945}
10946
10947
10948/**
10949 * Advances the guest RIP after reading it from the VMCS.
10950 *
10951 * @returns VBox status code, no informational status codes.
10952 * @param pVCpu The cross context virtual CPU structure.
10953 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10954 * out-of-sync. Make sure to update the required fields
10955 * before using them.
10956 * @param pVmxTransient Pointer to the VMX transient structure.
10957 *
10958 * @remarks No-long-jump zone!!!
10959 */
10960static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10961{
10962 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10963 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10964 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10965 AssertRCReturn(rc, rc);
10966
10967 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10968
10969 /*
10970 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10971 * pending debug exception field as it takes care of priority of events.
10972 *
10973 * See Intel spec. 32.2.1 "Debug Exceptions".
10974 */
10975 if ( !pVCpu->hm.s.fSingleInstruction
10976 && pMixedCtx->eflags.Bits.u1TF)
10977 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10978
10979 return VINF_SUCCESS;
10980}
10981
10982
10983/**
10984 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10985 * and update error record fields accordingly.
10986 *
10987 * @return VMX_IGS_* return codes.
10988 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10989 * wrong with the guest state.
10990 *
10991 * @param pVM The cross context VM structure.
10992 * @param pVCpu The cross context virtual CPU structure.
10993 * @param pCtx Pointer to the guest-CPU state.
10994 *
10995 * @remarks This function assumes our cache of the VMCS controls
10996 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10997 */
10998static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10999{
11000#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
11001#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
11002 uError = (err); \
11003 break; \
11004 } else do { } while (0)
11005
11006 int rc;
11007 uint32_t uError = VMX_IGS_ERROR;
11008 uint32_t u32Val;
11009 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
11010
11011 do
11012 {
11013 /*
11014 * CR0.
11015 */
11016 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
11017 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
11018 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
11019 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
11020 if (fUnrestrictedGuest)
11021 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
11022
11023 uint32_t u32GuestCR0;
11024 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
11025 AssertRCBreak(rc);
11026 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
11027 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
11028 if ( !fUnrestrictedGuest
11029 && (u32GuestCR0 & X86_CR0_PG)
11030 && !(u32GuestCR0 & X86_CR0_PE))
11031 {
11032 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
11033 }
11034
11035 /*
11036 * CR4.
11037 */
11038 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
11039 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
11040
11041 uint32_t u32GuestCR4;
11042 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
11043 AssertRCBreak(rc);
11044 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
11045 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
11046
11047 /*
11048 * IA32_DEBUGCTL MSR.
11049 */
11050 uint64_t u64Val;
11051 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
11052 AssertRCBreak(rc);
11053 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
11054 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
11055 {
11056 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
11057 }
11058 uint64_t u64DebugCtlMsr = u64Val;
11059
11060#ifdef VBOX_STRICT
11061 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
11062 AssertRCBreak(rc);
11063 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
11064#endif
11065 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
11066
11067 /*
11068 * RIP and RFLAGS.
11069 */
11070 uint32_t u32Eflags;
11071#if HC_ARCH_BITS == 64
11072 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
11073 AssertRCBreak(rc);
11074 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
11075 if ( !fLongModeGuest
11076 || !pCtx->cs.Attr.n.u1Long)
11077 {
11078 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
11079 }
11080 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
11081 * must be identical if the "IA-32e mode guest" VM-entry
11082 * control is 1 and CS.L is 1. No check applies if the
11083 * CPU supports 64 linear-address bits. */
11084
11085 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
11086 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
11087 AssertRCBreak(rc);
11088 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
11089 VMX_IGS_RFLAGS_RESERVED);
11090 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
11091 u32Eflags = u64Val;
11092#else
11093 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
11094 AssertRCBreak(rc);
11095 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
11096 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
11097#endif
11098
11099 if ( fLongModeGuest
11100 || ( fUnrestrictedGuest
11101 && !(u32GuestCR0 & X86_CR0_PE)))
11102 {
11103 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
11104 }
11105
11106 uint32_t u32EntryInfo;
11107 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
11108 AssertRCBreak(rc);
11109 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11110 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11111 {
11112 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
11113 }
11114
11115 /*
11116 * 64-bit checks.
11117 */
11118#if HC_ARCH_BITS == 64
11119 if (fLongModeGuest)
11120 {
11121 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
11122 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
11123 }
11124
11125 if ( !fLongModeGuest
11126 && (u32GuestCR4 & X86_CR4_PCIDE))
11127 {
11128 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
11129 }
11130
11131 /** @todo CR3 field must be such that bits 63:52 and bits in the range
11132 * 51:32 beyond the processor's physical-address width are 0. */
11133
11134 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
11135 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
11136 {
11137 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
11138 }
11139
11140 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
11141 AssertRCBreak(rc);
11142 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
11143
11144 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
11145 AssertRCBreak(rc);
11146 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
11147#endif
11148
11149 /*
11150 * PERF_GLOBAL MSR.
11151 */
11152 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
11153 {
11154 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
11155 AssertRCBreak(rc);
11156 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
11157 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
11158 }
11159
11160 /*
11161 * PAT MSR.
11162 */
11163 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
11164 {
11165 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
11166 AssertRCBreak(rc);
11167 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
11168 for (unsigned i = 0; i < 8; i++)
11169 {
11170 uint8_t u8Val = (u64Val & 0xff);
11171 if ( u8Val != 0 /* UC */
11172 && u8Val != 1 /* WC */
11173 && u8Val != 4 /* WT */
11174 && u8Val != 5 /* WP */
11175 && u8Val != 6 /* WB */
11176 && u8Val != 7 /* UC- */)
11177 {
11178 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
11179 }
11180 u64Val >>= 8;
11181 }
11182 }
11183
11184 /*
11185 * EFER MSR.
11186 */
11187 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
11188 {
11189 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
11190 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
11191 AssertRCBreak(rc);
11192 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
11193 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
11194 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
11195 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
11196 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
11197 HMVMX_CHECK_BREAK( fUnrestrictedGuest
11198 || !(u32GuestCR0 & X86_CR0_PG)
11199 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
11200 VMX_IGS_EFER_LMA_LME_MISMATCH);
11201 }
11202
11203 /*
11204 * Segment registers.
11205 */
11206 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11207 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
11208 if (!(u32Eflags & X86_EFL_VM))
11209 {
11210 /* CS */
11211 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
11212 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
11213 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
11214 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
11215 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
11216 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
11217 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
11218 /* CS cannot be loaded with NULL in protected mode. */
11219 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
11220 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
11221 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
11222 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
11223 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
11224 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
11225 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
11226 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
11227 else
11228 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
11229
11230 /* SS */
11231 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11232 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
11233 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
11234 if ( !(pCtx->cr0 & X86_CR0_PE)
11235 || pCtx->cs.Attr.n.u4Type == 3)
11236 {
11237 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
11238 }
11239 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
11240 {
11241 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
11242 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
11243 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
11244 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
11245 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
11246 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
11247 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
11248 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
11249 }
11250
11251 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
11252 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
11253 {
11254 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
11255 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
11256 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11257 || pCtx->ds.Attr.n.u4Type > 11
11258 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
11259 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
11260 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
11261 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
11262 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
11263 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
11264 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
11265 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11266 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
11267 }
11268 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
11269 {
11270 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
11271 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
11272 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11273 || pCtx->es.Attr.n.u4Type > 11
11274 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
11275 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
11276 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
11277 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
11278 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
11279 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
11280 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
11281 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11282 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
11283 }
11284 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
11285 {
11286 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
11287 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
11288 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11289 || pCtx->fs.Attr.n.u4Type > 11
11290 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
11291 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
11292 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
11293 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
11294 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
11295 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
11296 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
11297 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11298 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
11299 }
11300 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
11301 {
11302 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
11303 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
11304 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11305 || pCtx->gs.Attr.n.u4Type > 11
11306 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
11307 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
11308 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
11309 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
11310 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
11311 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
11312 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
11313 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11314 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
11315 }
11316 /* 64-bit capable CPUs. */
11317#if HC_ARCH_BITS == 64
11318 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11319 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11320 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11321 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11322 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11323 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
11324 VMX_IGS_LONGMODE_SS_BASE_INVALID);
11325 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
11326 VMX_IGS_LONGMODE_DS_BASE_INVALID);
11327 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
11328 VMX_IGS_LONGMODE_ES_BASE_INVALID);
11329#endif
11330 }
11331 else
11332 {
11333 /* V86 mode checks. */
11334 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
11335 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11336 {
11337 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
11338 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
11339 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
11340 }
11341 else
11342 {
11343 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
11344 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
11345 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
11346 }
11347
11348 /* CS */
11349 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
11350 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
11351 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
11352 /* SS */
11353 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
11354 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
11355 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
11356 /* DS */
11357 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
11358 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
11359 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
11360 /* ES */
11361 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
11362 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
11363 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
11364 /* FS */
11365 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
11366 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
11367 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
11368 /* GS */
11369 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
11370 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
11371 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
11372 /* 64-bit capable CPUs. */
11373#if HC_ARCH_BITS == 64
11374 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11375 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11376 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11377 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11378 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11379 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
11380 VMX_IGS_LONGMODE_SS_BASE_INVALID);
11381 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
11382 VMX_IGS_LONGMODE_DS_BASE_INVALID);
11383 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
11384 VMX_IGS_LONGMODE_ES_BASE_INVALID);
11385#endif
11386 }
11387
11388 /*
11389 * TR.
11390 */
11391 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
11392 /* 64-bit capable CPUs. */
11393#if HC_ARCH_BITS == 64
11394 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
11395#endif
11396 if (fLongModeGuest)
11397 {
11398 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
11399 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
11400 }
11401 else
11402 {
11403 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
11404 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
11405 VMX_IGS_TR_ATTR_TYPE_INVALID);
11406 }
11407 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
11408 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
11409 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
11410 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
11411 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11412 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
11413 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11414 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
11415
11416 /*
11417 * GDTR and IDTR.
11418 */
11419#if HC_ARCH_BITS == 64
11420 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
11421 AssertRCBreak(rc);
11422 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
11423
11424 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
11425 AssertRCBreak(rc);
11426 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
11427#endif
11428
11429 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
11430 AssertRCBreak(rc);
11431 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11432
11433 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
11434 AssertRCBreak(rc);
11435 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11436
11437 /*
11438 * Guest Non-Register State.
11439 */
11440 /* Activity State. */
11441 uint32_t u32ActivityState;
11442 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
11443 AssertRCBreak(rc);
11444 HMVMX_CHECK_BREAK( !u32ActivityState
11445 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
11446 VMX_IGS_ACTIVITY_STATE_INVALID);
11447 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
11448 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
11449 uint32_t u32IntrState;
11450 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
11451 AssertRCBreak(rc);
11452 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
11453 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11454 {
11455 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11456 }
11457
11458 /** @todo Activity state and injecting interrupts. Left as a todo since we
11459 * currently don't use activity states but ACTIVE. */
11460
11461 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11462 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11463
11464 /* Guest interruptibility-state. */
11465 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11466 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11467 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
11468 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11469 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11470 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11471 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11472 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11473 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11474 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
11475 {
11476 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11477 {
11478 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11479 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11480 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11481 }
11482 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11483 {
11484 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11485 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11486 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11487 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11488 }
11489 }
11490 /** @todo Assumes the processor is not in SMM. */
11491 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11492 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11493 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11494 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11495 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11496 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11497 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11498 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11499 {
11500 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11501 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11502 }
11503
11504 /* Pending debug exceptions. */
11505#if HC_ARCH_BITS == 64
11506 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11507 AssertRCBreak(rc);
11508 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11509 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11510 u32Val = u64Val; /* For pending debug exceptions checks below. */
11511#else
11512 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11513 AssertRCBreak(rc);
11514 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11515 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11516#endif
11517
11518 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11519 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11520 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11521 {
11522 if ( (u32Eflags & X86_EFL_TF)
11523 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11524 {
11525 /* Bit 14 is PendingDebug.BS. */
11526 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11527 }
11528 if ( !(u32Eflags & X86_EFL_TF)
11529 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11530 {
11531 /* Bit 14 is PendingDebug.BS. */
11532 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11533 }
11534 }
11535
11536 /* VMCS link pointer. */
11537 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11538 AssertRCBreak(rc);
11539 if (u64Val != UINT64_C(0xffffffffffffffff))
11540 {
11541 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11542 /** @todo Bits beyond the processor's physical-address width MBZ. */
11543 /** @todo 32-bit located in memory referenced by value of this field (as a
11544 * physical address) must contain the processor's VMCS revision ID. */
11545 /** @todo SMM checks. */
11546 }
11547
11548 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11549 * not using Nested Paging? */
11550 if ( pVM->hm.s.fNestedPaging
11551 && !fLongModeGuest
11552 && CPUMIsGuestInPAEModeEx(pCtx))
11553 {
11554 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11555 AssertRCBreak(rc);
11556 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11557
11558 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11559 AssertRCBreak(rc);
11560 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11561
11562 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11563 AssertRCBreak(rc);
11564 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11565
11566 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11567 AssertRCBreak(rc);
11568 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11569 }
11570
11571 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11572 if (uError == VMX_IGS_ERROR)
11573 uError = VMX_IGS_REASON_NOT_FOUND;
11574 } while (0);
11575
11576 pVCpu->hm.s.u32HMError = uError;
11577 return uError;
11578
11579#undef HMVMX_ERROR_BREAK
11580#undef HMVMX_CHECK_BREAK
11581}
11582
11583/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11584/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11585/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11586
11587/** @name VM-exit handlers.
11588 * @{
11589 */
11590
11591/**
11592 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11593 */
11594HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11595{
11596 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11597 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11598 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11599 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11600 return VINF_SUCCESS;
11601 return VINF_EM_RAW_INTERRUPT;
11602}
11603
11604
11605/**
11606 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11607 */
11608HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11609{
11610 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11611 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11612
11613 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11614 AssertRCReturn(rc, rc);
11615
11616 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11617 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11618 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11619 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11620
11621 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11622 {
11623 /*
11624 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11625 * anything we inject is not going to cause a VM-exit directly for the event being injected.
11626 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11627 *
11628 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11629 */
11630 VMXDispatchHostNmi();
11631 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11632 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11633 return VINF_SUCCESS;
11634 }
11635
11636 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11637 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11638 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11639 { /* likely */ }
11640 else
11641 {
11642 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11643 rcStrictRc1 = VINF_SUCCESS;
11644 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11645 return rcStrictRc1;
11646 }
11647
11648 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11649 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11650 switch (uIntType)
11651 {
11652 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11653 Assert(uVector == X86_XCPT_DB);
11654 RT_FALL_THRU();
11655 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11656 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11657 RT_FALL_THRU();
11658 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11659 {
11660 /*
11661 * If there's any exception caused as a result of event injection, the resulting
11662 * secondary/final execption will be pending, we shall continue guest execution
11663 * after injecting the event. The page-fault case is complicated and we manually
11664 * handle any currently pending event in hmR0VmxExitXcptPF.
11665 */
11666 if (!pVCpu->hm.s.Event.fPending)
11667 { /* likely */ }
11668 else if (uVector != X86_XCPT_PF)
11669 {
11670 rc = VINF_SUCCESS;
11671 break;
11672 }
11673
11674 switch (uVector)
11675 {
11676 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11677 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11678 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11679 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11680 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11681 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11682 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11683
11684 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11685 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11686 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11687 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11688 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11689 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11690 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11691 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11692 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11693 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11694 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11695 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11696 default:
11697 {
11698 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11699 AssertRCReturn(rc, rc);
11700
11701 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11702 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11703 {
11704 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11705 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11706 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11707
11708 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11709 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11710 AssertRCReturn(rc, rc);
11711 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11712 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11713 0 /* GCPtrFaultAddress */);
11714 AssertRCReturn(rc, rc);
11715 }
11716 else
11717 {
11718 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11719 pVCpu->hm.s.u32HMError = uVector;
11720 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11721 }
11722 break;
11723 }
11724 }
11725 break;
11726 }
11727
11728 default:
11729 {
11730 pVCpu->hm.s.u32HMError = uExitIntInfo;
11731 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11732 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11733 break;
11734 }
11735 }
11736 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11737 return rc;
11738}
11739
11740
11741/**
11742 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11743 */
11744HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11745{
11746 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11747
11748 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11749 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11750
11751 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11752 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11753 return VINF_SUCCESS;
11754}
11755
11756
11757/**
11758 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11759 */
11760HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11761{
11762 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11763 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11764 {
11765 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11766 HMVMX_RETURN_UNEXPECTED_EXIT();
11767 }
11768
11769 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11770
11771 /*
11772 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11773 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11774 */
11775 uint32_t uIntrState = 0;
11776 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11777 AssertRCReturn(rc, rc);
11778
11779 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11780 if ( fBlockSti
11781 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11782 {
11783 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11784 }
11785
11786 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11787 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11788
11789 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11790 return VINF_SUCCESS;
11791}
11792
11793
11794/**
11795 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11796 */
11797HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11798{
11799 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11800 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11801 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11802}
11803
11804
11805/**
11806 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11807 */
11808HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11809{
11810 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11811 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11812 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11813}
11814
11815
11816/**
11817 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11818 */
11819HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11820{
11821 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11822 PVM pVM = pVCpu->CTX_SUFF(pVM);
11823 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11824 if (RT_LIKELY(rc == VINF_SUCCESS))
11825 {
11826 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11827 Assert(pVmxTransient->cbInstr == 2);
11828 }
11829 else
11830 {
11831 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11832 rc = VERR_EM_INTERPRETER;
11833 }
11834 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11835 return rc;
11836}
11837
11838
11839/**
11840 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11841 */
11842HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11843{
11844 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11845 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11846 AssertRCReturn(rc, rc);
11847
11848 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11849 return VINF_EM_RAW_EMULATE_INSTR;
11850
11851 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11852 HMVMX_RETURN_UNEXPECTED_EXIT();
11853}
11854
11855
11856/**
11857 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11858 */
11859HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11860{
11861 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11862 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11863 AssertRCReturn(rc, rc);
11864
11865 PVM pVM = pVCpu->CTX_SUFF(pVM);
11866 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11867 if (RT_LIKELY(rc == VINF_SUCCESS))
11868 {
11869 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11870 Assert(pVmxTransient->cbInstr == 2);
11871 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11872 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11873 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11874 }
11875 else
11876 rc = VERR_EM_INTERPRETER;
11877 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11878 return rc;
11879}
11880
11881
11882/**
11883 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11884 */
11885HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11886{
11887 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11888 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11889 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11890 AssertRCReturn(rc, rc);
11891
11892 PVM pVM = pVCpu->CTX_SUFF(pVM);
11893 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11894 if (RT_SUCCESS(rc))
11895 {
11896 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11897 Assert(pVmxTransient->cbInstr == 3);
11898 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11899 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11900 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11901 }
11902 else
11903 {
11904 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11905 rc = VERR_EM_INTERPRETER;
11906 }
11907 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11908 return rc;
11909}
11910
11911
11912/**
11913 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11914 */
11915HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11916{
11917 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11918 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11919 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11920 AssertRCReturn(rc, rc);
11921
11922 PVM pVM = pVCpu->CTX_SUFF(pVM);
11923 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11924 if (RT_LIKELY(rc == VINF_SUCCESS))
11925 {
11926 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11927 Assert(pVmxTransient->cbInstr == 2);
11928 }
11929 else
11930 {
11931 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11932 rc = VERR_EM_INTERPRETER;
11933 }
11934 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11935 return rc;
11936}
11937
11938
11939/**
11940 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11941 */
11942HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11943{
11944 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11945 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11946
11947 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11948 if (pVCpu->hm.s.fHypercallsEnabled)
11949 {
11950#if 0
11951 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11952#else
11953 /* Aggressive state sync. for now. */
11954 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11955 rc |= hmR0VmxSaveGuestRflags(pVCpu,pMixedCtx); /* For CPL checks in gimHvHypercall() & gimKvmHypercall() */
11956 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11957 AssertRCReturn(rc, rc);
11958#endif
11959
11960 /* Perform the hypercall. */
11961 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11962 if (rcStrict == VINF_SUCCESS)
11963 {
11964 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11965 AssertRCReturn(rc, rc);
11966 }
11967 else
11968 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11969 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11970 || RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
11971
11972 /* If the hypercall changes anything other than guest's general-purpose registers,
11973 we would need to reload the guest changed bits here before VM-entry. */
11974 }
11975 else
11976 Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11977
11978 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11979 if (RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)))
11980 {
11981 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11982 rcStrict = VINF_SUCCESS;
11983 }
11984
11985 return rcStrict;
11986}
11987
11988
11989/**
11990 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11991 */
11992HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11993{
11994 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11995 PVM pVM = pVCpu->CTX_SUFF(pVM);
11996 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11997
11998 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11999 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12000 AssertRCReturn(rc, rc);
12001
12002 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
12003 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
12004 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12005 else
12006 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
12007 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
12008 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
12009 return rcStrict;
12010}
12011
12012
12013/**
12014 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
12015 */
12016HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12017{
12018 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12019 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12020 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12021 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12022 AssertRCReturn(rc, rc);
12023
12024 PVM pVM = pVCpu->CTX_SUFF(pVM);
12025 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12026 if (RT_LIKELY(rc == VINF_SUCCESS))
12027 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12028 else
12029 {
12030 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
12031 rc = VERR_EM_INTERPRETER;
12032 }
12033 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
12034 return rc;
12035}
12036
12037
12038/**
12039 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
12040 */
12041HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12042{
12043 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12044 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12045 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12046 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12047 AssertRCReturn(rc, rc);
12048
12049 PVM pVM = pVCpu->CTX_SUFF(pVM);
12050 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12051 rc = VBOXSTRICTRC_VAL(rc2);
12052 if (RT_LIKELY( rc == VINF_SUCCESS
12053 || rc == VINF_EM_HALT))
12054 {
12055 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12056 AssertRCReturn(rc3, rc3);
12057
12058 if ( rc == VINF_EM_HALT
12059 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
12060 {
12061 rc = VINF_SUCCESS;
12062 }
12063 }
12064 else
12065 {
12066 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
12067 rc = VERR_EM_INTERPRETER;
12068 }
12069 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
12070 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
12071 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
12072 return rc;
12073}
12074
12075
12076/**
12077 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
12078 */
12079HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12080{
12081 /*
12082 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
12083 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
12084 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
12085 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
12086 */
12087 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12088 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12089 HMVMX_RETURN_UNEXPECTED_EXIT();
12090}
12091
12092
12093/**
12094 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
12095 */
12096HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12097{
12098 /*
12099 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
12100 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
12101 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
12102 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
12103 */
12104 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12105 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12106 HMVMX_RETURN_UNEXPECTED_EXIT();
12107}
12108
12109
12110/**
12111 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
12112 */
12113HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12114{
12115 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
12116 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12117 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12118 HMVMX_RETURN_UNEXPECTED_EXIT();
12119}
12120
12121
12122/**
12123 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
12124 */
12125HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12126{
12127 /*
12128 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
12129 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
12130 * See Intel spec. 25.3 "Other Causes of VM-exits".
12131 */
12132 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12133 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12134 HMVMX_RETURN_UNEXPECTED_EXIT();
12135}
12136
12137
12138/**
12139 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
12140 * VM-exit.
12141 */
12142HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12143{
12144 /*
12145 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
12146 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
12147 *
12148 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
12149 * See Intel spec. "23.8 Restrictions on VMX operation".
12150 */
12151 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12152 return VINF_SUCCESS;
12153}
12154
12155
12156/**
12157 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
12158 * VM-exit.
12159 */
12160HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12161{
12162 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12163 return VINF_EM_RESET;
12164}
12165
12166
12167/**
12168 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
12169 */
12170HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12171{
12172 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12173 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
12174
12175 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12176 AssertRCReturn(rc, rc);
12177
12178 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
12179 rc = VINF_SUCCESS;
12180 else
12181 rc = VINF_EM_HALT;
12182
12183 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
12184 if (rc != VINF_SUCCESS)
12185 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
12186 return rc;
12187}
12188
12189
12190/**
12191 * VM-exit handler for instructions that result in a \#UD exception delivered to
12192 * the guest.
12193 */
12194HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12195{
12196 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12197 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
12198 return VINF_SUCCESS;
12199}
12200
12201
12202/**
12203 * VM-exit handler for expiry of the VMX preemption timer.
12204 */
12205HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12206{
12207 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12208
12209 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
12210 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12211
12212 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
12213 PVM pVM = pVCpu->CTX_SUFF(pVM);
12214 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
12215 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
12216 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
12217}
12218
12219
12220/**
12221 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
12222 */
12223HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12224{
12225 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12226
12227 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12228 rc |= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, false /*fNeedRsp*/);
12229 rc |= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
12230 AssertRCReturn(rc, rc);
12231
12232 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
12233 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12234
12235 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
12236
12237 return rcStrict;
12238}
12239
12240
12241/**
12242 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
12243 */
12244HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12245{
12246 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12247 /** @todo Use VM-exit instruction information. */
12248 return VERR_EM_INTERPRETER;
12249}
12250
12251
12252/**
12253 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
12254 * Error VM-exit.
12255 */
12256HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12257{
12258 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12259 AssertRCReturn(rc, rc);
12260
12261 rc = hmR0VmxCheckVmcsCtls(pVCpu);
12262 AssertRCReturn(rc, rc);
12263
12264 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12265 NOREF(uInvalidReason);
12266
12267#ifdef VBOX_STRICT
12268 uint32_t uIntrState;
12269 RTHCUINTREG uHCReg;
12270 uint64_t u64Val;
12271 uint32_t u32Val;
12272
12273 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
12274 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
12275 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
12276 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
12277 AssertRCReturn(rc, rc);
12278
12279 Log4(("uInvalidReason %u\n", uInvalidReason));
12280 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
12281 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
12282 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
12283 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
12284
12285 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
12286 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
12287 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
12288 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
12289 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
12290 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
12291 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
12292 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
12293 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
12294 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
12295 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
12296 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
12297#else
12298 NOREF(pVmxTransient);
12299#endif
12300
12301 hmR0DumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12302 return VERR_VMX_INVALID_GUEST_STATE;
12303}
12304
12305
12306/**
12307 * VM-exit handler for VM-entry failure due to an MSR-load
12308 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
12309 */
12310HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12311{
12312 NOREF(pVmxTransient);
12313 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12314 HMVMX_RETURN_UNEXPECTED_EXIT();
12315}
12316
12317
12318/**
12319 * VM-exit handler for VM-entry failure due to a machine-check event
12320 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
12321 */
12322HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12323{
12324 NOREF(pVmxTransient);
12325 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12326 HMVMX_RETURN_UNEXPECTED_EXIT();
12327}
12328
12329
12330/**
12331 * VM-exit handler for all undefined reasons. Should never ever happen.. in
12332 * theory.
12333 */
12334HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12335{
12336 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
12337 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
12338 return VERR_VMX_UNDEFINED_EXIT_CODE;
12339}
12340
12341
12342/**
12343 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
12344 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
12345 * Conditional VM-exit.
12346 */
12347HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12348{
12349 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12350
12351 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
12352 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
12353 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
12354 return VERR_EM_INTERPRETER;
12355 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12356 HMVMX_RETURN_UNEXPECTED_EXIT();
12357}
12358
12359
12360/**
12361 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
12362 */
12363HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12364{
12365 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12366
12367 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
12368 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
12369 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
12370 return VERR_EM_INTERPRETER;
12371 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12372 HMVMX_RETURN_UNEXPECTED_EXIT();
12373}
12374
12375
12376/**
12377 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
12378 */
12379HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12380{
12381 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12382
12383 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
12384 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12385 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12386 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12387 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12388 {
12389 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12390 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12391 }
12392 AssertRCReturn(rc, rc);
12393 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
12394
12395#ifdef VBOX_STRICT
12396 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
12397 {
12398 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
12399 && pMixedCtx->ecx != MSR_K6_EFER)
12400 {
12401 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12402 pMixedCtx->ecx));
12403 HMVMX_RETURN_UNEXPECTED_EXIT();
12404 }
12405 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12406 {
12407 VMXMSREXITREAD enmRead;
12408 VMXMSREXITWRITE enmWrite;
12409 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12410 AssertRCReturn(rc2, rc2);
12411 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
12412 {
12413 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12414 HMVMX_RETURN_UNEXPECTED_EXIT();
12415 }
12416 }
12417 }
12418#endif
12419
12420 PVM pVM = pVCpu->CTX_SUFF(pVM);
12421 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12422 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
12423 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
12424 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
12425 if (RT_SUCCESS(rc))
12426 {
12427 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12428 Assert(pVmxTransient->cbInstr == 2);
12429 }
12430 return rc;
12431}
12432
12433
12434/**
12435 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12436 */
12437HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12438{
12439 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12440 PVM pVM = pVCpu->CTX_SUFF(pVM);
12441 int rc = VINF_SUCCESS;
12442
12443 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
12444 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12445 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12446 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12447 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12448 {
12449 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12450 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12451 }
12452 AssertRCReturn(rc, rc);
12453 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
12454
12455 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12456 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
12457 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12458
12459 if (RT_SUCCESS(rc))
12460 {
12461 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12462
12463 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
12464 if ( pMixedCtx->ecx == MSR_IA32_APICBASE
12465 || ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
12466 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
12467 {
12468 /*
12469 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12470 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
12471 * EMInterpretWrmsr() changes it.
12472 */
12473 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_APIC_STATE);
12474 }
12475 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12476 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12477 else if (pMixedCtx->ecx == MSR_K6_EFER)
12478 {
12479 /*
12480 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12481 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12482 * the other bits as well, SCE and NXE. See @bugref{7368}.
12483 */
12484 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
12485 }
12486
12487 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12488 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12489 {
12490 switch (pMixedCtx->ecx)
12491 {
12492 /*
12493 * For SYSENTER CS, EIP, ESP MSRs, we set both the flags here so we don't accidentally
12494 * overwrite the changed guest-CPU context value while going to ring-3, see @bufref{8745}.
12495 */
12496 case MSR_IA32_SYSENTER_CS:
12497 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
12498 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
12499 break;
12500 case MSR_IA32_SYSENTER_EIP:
12501 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
12502 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
12503 break;
12504 case MSR_IA32_SYSENTER_ESP:
12505 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
12506 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
12507 break;
12508 case MSR_K8_FS_BASE: RT_FALL_THRU();
12509 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
12510 case MSR_K6_EFER: /* already handled above */ break;
12511 default:
12512 {
12513 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12514 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12515 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12516 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMM_GUEST_LAZY_MSRS);
12517 break;
12518 }
12519 }
12520 }
12521#ifdef VBOX_STRICT
12522 else
12523 {
12524 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12525 switch (pMixedCtx->ecx)
12526 {
12527 case MSR_IA32_SYSENTER_CS:
12528 case MSR_IA32_SYSENTER_EIP:
12529 case MSR_IA32_SYSENTER_ESP:
12530 case MSR_K8_FS_BASE:
12531 case MSR_K8_GS_BASE:
12532 {
12533 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12534 HMVMX_RETURN_UNEXPECTED_EXIT();
12535 }
12536
12537 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12538 default:
12539 {
12540 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12541 {
12542 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12543 if (pMixedCtx->ecx != MSR_K6_EFER)
12544 {
12545 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12546 pMixedCtx->ecx));
12547 HMVMX_RETURN_UNEXPECTED_EXIT();
12548 }
12549 }
12550
12551 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12552 {
12553 VMXMSREXITREAD enmRead;
12554 VMXMSREXITWRITE enmWrite;
12555 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12556 AssertRCReturn(rc2, rc2);
12557 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12558 {
12559 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12560 HMVMX_RETURN_UNEXPECTED_EXIT();
12561 }
12562 }
12563 break;
12564 }
12565 }
12566 }
12567#endif /* VBOX_STRICT */
12568 }
12569 return rc;
12570}
12571
12572
12573/**
12574 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12575 */
12576HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12577{
12578 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12579
12580 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12581 return VINF_EM_RAW_INTERRUPT;
12582}
12583
12584
12585/**
12586 * VM-exit handler for when the TPR value is lowered below the specified
12587 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12588 */
12589HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12590{
12591 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12592 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12593
12594 /*
12595 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12596 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12597 */
12598 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12599 return VINF_SUCCESS;
12600}
12601
12602
12603/**
12604 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12605 * VM-exit.
12606 *
12607 * @retval VINF_SUCCESS when guest execution can continue.
12608 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12609 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12610 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12611 * interpreter.
12612 */
12613HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12614{
12615 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12616 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12617 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12618 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12619 AssertRCReturn(rc, rc);
12620
12621 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12622 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12623 PVM pVM = pVCpu->CTX_SUFF(pVM);
12624 VBOXSTRICTRC rcStrict;
12625 rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, true /*fNeedRsp*/);
12626 switch (uAccessType)
12627 {
12628 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12629 {
12630 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12631 AssertRCReturn(rc, rc);
12632
12633 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12634 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12635 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12636 AssertMsg( rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE
12637 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12638 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12639 {
12640 case 0: /* CR0 */
12641 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12642 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12643 break;
12644 case 2: /* CR2 */
12645 /* Nothing to do here, CR2 it's not part of the VMCS. */
12646 break;
12647 case 3: /* CR3 */
12648 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
12649 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12650 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12651 break;
12652 case 4: /* CR4 */
12653 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12654 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12655 VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12656 break;
12657 case 8: /* CR8 */
12658 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12659 /* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12660 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_APIC_STATE);
12661 break;
12662 default:
12663 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12664 break;
12665 }
12666
12667 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12668 break;
12669 }
12670
12671 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12672 {
12673 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12674 AssertRCReturn(rc, rc);
12675
12676 Assert( !pVM->hm.s.fNestedPaging
12677 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12678 || pVCpu->hm.s.fUsingDebugLoop
12679 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12680
12681 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12682 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12683 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12684
12685 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12686 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12687 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12688 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12689 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12690 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12691 VBOXSTRICTRC_VAL(rcStrict)));
12692 if (VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification) == X86_GREG_xSP)
12693 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RSP);
12694 break;
12695 }
12696
12697 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12698 {
12699 AssertRCReturn(rc, rc);
12700 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12701 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12702 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12703 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12704 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12705 break;
12706 }
12707
12708 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12709 {
12710 AssertRCReturn(rc, rc);
12711 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12712 VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12713 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE,
12714 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12715 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12716 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12717 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12718 break;
12719 }
12720
12721 default:
12722 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12723 VERR_VMX_UNEXPECTED_EXCEPTION);
12724 }
12725
12726 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12727 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12728 NOREF(pVM);
12729 return rcStrict;
12730}
12731
12732
12733/**
12734 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12735 * VM-exit.
12736 */
12737HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12738{
12739 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12740 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12741
12742 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12743 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12744 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12745 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12746 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12747 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12748 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12749 AssertRCReturn(rc, rc);
12750
12751 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12752 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12753 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12754 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12755 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12756 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12757 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12758 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12759 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12760
12761 /* I/O operation lookup arrays. */
12762 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12763 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12764
12765 VBOXSTRICTRC rcStrict;
12766 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12767 uint32_t const cbInstr = pVmxTransient->cbInstr;
12768 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12769 PVM pVM = pVCpu->CTX_SUFF(pVM);
12770 if (fIOString)
12771 {
12772#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12773 See @bugref{5752#c158}. Should work now. */
12774 /*
12775 * INS/OUTS - I/O String instruction.
12776 *
12777 * Use instruction-information if available, otherwise fall back on
12778 * interpreting the instruction.
12779 */
12780 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12781 fIOWrite ? 'w' : 'r'));
12782 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12783 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12784 {
12785 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12786 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12787 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12788 AssertRCReturn(rc2, rc2);
12789 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12790 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12791 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12792 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12793 if (fIOWrite)
12794 {
12795 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12796 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12797 }
12798 else
12799 {
12800 /*
12801 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12802 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12803 * See Intel Instruction spec. for "INS".
12804 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12805 */
12806 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12807 }
12808 }
12809 else
12810 {
12811 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12812 int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12813 AssertRCReturn(rc2, rc2);
12814 rcStrict = IEMExecOne(pVCpu);
12815 }
12816 /** @todo IEM needs to be setting these flags somehow. */
12817 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12818 fUpdateRipAlready = true;
12819#else
12820 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12821 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12822 if (RT_SUCCESS(rcStrict))
12823 {
12824 if (fIOWrite)
12825 {
12826 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12827 (DISCPUMODE)pDis->uAddrMode, cbValue);
12828 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12829 }
12830 else
12831 {
12832 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12833 (DISCPUMODE)pDis->uAddrMode, cbValue);
12834 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12835 }
12836 }
12837 else
12838 {
12839 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12840 pMixedCtx->rip));
12841 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12842 }
12843#endif
12844 }
12845 else
12846 {
12847 /*
12848 * IN/OUT - I/O instruction.
12849 */
12850 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12851 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12852 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12853 if (fIOWrite)
12854 {
12855 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12856 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12857 }
12858 else
12859 {
12860 uint32_t u32Result = 0;
12861 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12862 if (IOM_SUCCESS(rcStrict))
12863 {
12864 /* Save result of I/O IN instr. in AL/AX/EAX. */
12865 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12866 }
12867 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12868 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12869 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12870 }
12871 }
12872
12873 if (IOM_SUCCESS(rcStrict))
12874 {
12875 if (!fUpdateRipAlready)
12876 {
12877 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12878 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12879 }
12880
12881 /*
12882 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12883 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12884 */
12885 if (fIOString)
12886 {
12887 /** @todo Single-step for INS/OUTS with REP prefix? */
12888 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12889 }
12890 else if ( !fDbgStepping
12891 && fGstStepping)
12892 {
12893 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12894 }
12895
12896 /*
12897 * If any I/O breakpoints are armed, we need to check if one triggered
12898 * and take appropriate action.
12899 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12900 */
12901 int rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12902 AssertRCReturn(rc2, rc2);
12903
12904 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12905 * execution engines about whether hyper BPs and such are pending. */
12906 uint32_t const uDr7 = pMixedCtx->dr[7];
12907 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12908 && X86_DR7_ANY_RW_IO(uDr7)
12909 && (pMixedCtx->cr4 & X86_CR4_DE))
12910 || DBGFBpIsHwIoArmed(pVM)))
12911 {
12912 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12913
12914 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12915 VMMRZCallRing3Disable(pVCpu);
12916 HM_DISABLE_PREEMPT();
12917
12918 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12919
12920 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12921 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12922 {
12923 /* Raise #DB. */
12924 if (fIsGuestDbgActive)
12925 ASMSetDR6(pMixedCtx->dr[6]);
12926 if (pMixedCtx->dr[7] != uDr7)
12927 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12928
12929 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12930 }
12931 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12932 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12933 else if ( rcStrict2 != VINF_SUCCESS
12934 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12935 rcStrict = rcStrict2;
12936 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12937
12938 HM_RESTORE_PREEMPT();
12939 VMMRZCallRing3Enable(pVCpu);
12940 }
12941 }
12942
12943#ifdef VBOX_STRICT
12944 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12945 Assert(!fIOWrite);
12946 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12947 Assert(fIOWrite);
12948 else
12949 {
12950#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12951 * statuses, that the VMM device and some others may return. See
12952 * IOM_SUCCESS() for guidance. */
12953 AssertMsg( RT_FAILURE(rcStrict)
12954 || rcStrict == VINF_SUCCESS
12955 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12956 || rcStrict == VINF_EM_DBG_BREAKPOINT
12957 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12958 || rcStrict == VINF_EM_RAW_TO_R3
12959 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12960#endif
12961 }
12962#endif
12963
12964 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12965 return rcStrict;
12966}
12967
12968
12969/**
12970 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12971 * VM-exit.
12972 */
12973HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12974{
12975 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12976
12977 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12978 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12979 AssertRCReturn(rc, rc);
12980 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12981 {
12982 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12983 AssertRCReturn(rc, rc);
12984 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12985 {
12986 uint32_t uErrCode;
12987 RTGCUINTPTR GCPtrFaultAddress;
12988 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12989 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12990 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12991 if (fErrorCodeValid)
12992 {
12993 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12994 AssertRCReturn(rc, rc);
12995 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12996 }
12997 else
12998 uErrCode = 0;
12999
13000 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
13001 && uVector == X86_XCPT_PF)
13002 GCPtrFaultAddress = pMixedCtx->cr2;
13003 else
13004 GCPtrFaultAddress = 0;
13005
13006 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
13007 0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
13008
13009 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
13010 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
13011 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13012 }
13013 }
13014
13015 /* Fall back to the interpreter to emulate the task-switch. */
13016 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
13017 return VERR_EM_INTERPRETER;
13018}
13019
13020
13021/**
13022 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
13023 */
13024HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13025{
13026 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13027 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
13028 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
13029 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
13030 AssertRCReturn(rc, rc);
13031 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
13032 return VINF_EM_DBG_STEPPED;
13033}
13034
13035
13036/**
13037 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
13038 */
13039HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13040{
13041 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13042
13043 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
13044
13045 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13046 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13047 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13048 {
13049 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
13050 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13051 {
13052 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
13053 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13054 }
13055 }
13056 else
13057 {
13058 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13059 rcStrict1 = VINF_SUCCESS;
13060 return rcStrict1;
13061 }
13062
13063#if 0
13064 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
13065 * just sync the whole thing. */
13066 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13067#else
13068 /* Aggressive state sync. for now. */
13069 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13070 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13071 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13072#endif
13073 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13074 AssertRCReturn(rc, rc);
13075
13076 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
13077 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
13078 VBOXSTRICTRC rcStrict2;
13079 switch (uAccessType)
13080 {
13081 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
13082 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
13083 {
13084 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
13085 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
13086 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
13087
13088 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
13089 GCPhys &= PAGE_BASE_GC_MASK;
13090 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
13091 PVM pVM = pVCpu->CTX_SUFF(pVM);
13092 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
13093 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
13094
13095 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
13096 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
13097 CPUMCTX2CORE(pMixedCtx), GCPhys);
13098 Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
13099 if ( rcStrict2 == VINF_SUCCESS
13100 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13101 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
13102 {
13103 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13104 | HM_CHANGED_GUEST_RSP
13105 | HM_CHANGED_GUEST_RFLAGS
13106 | HM_CHANGED_GUEST_APIC_STATE);
13107 rcStrict2 = VINF_SUCCESS;
13108 }
13109 break;
13110 }
13111
13112 default:
13113 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
13114 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
13115 break;
13116 }
13117
13118 if (rcStrict2 != VINF_SUCCESS)
13119 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
13120 return rcStrict2;
13121}
13122
13123
13124/**
13125 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
13126 * VM-exit.
13127 */
13128HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13129{
13130 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13131
13132 /* We should -not- get this VM-exit if the guest's debug registers were active. */
13133 if (pVmxTransient->fWasGuestDebugStateActive)
13134 {
13135 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
13136 HMVMX_RETURN_UNEXPECTED_EXIT();
13137 }
13138
13139 if ( !pVCpu->hm.s.fSingleInstruction
13140 && !pVmxTransient->fWasHyperDebugStateActive)
13141 {
13142 Assert(!DBGFIsStepping(pVCpu));
13143 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
13144
13145 /* Don't intercept MOV DRx any more. */
13146 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
13147 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
13148 AssertRCReturn(rc, rc);
13149
13150 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
13151 VMMRZCallRing3Disable(pVCpu);
13152 HM_DISABLE_PREEMPT();
13153
13154 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
13155 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
13156 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
13157
13158 HM_RESTORE_PREEMPT();
13159 VMMRZCallRing3Enable(pVCpu);
13160
13161#ifdef VBOX_WITH_STATISTICS
13162 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13163 AssertRCReturn(rc, rc);
13164 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
13165 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
13166 else
13167 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
13168#endif
13169 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
13170 return VINF_SUCCESS;
13171 }
13172
13173 /*
13174 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
13175 * Update the segment registers and DR7 from the CPU.
13176 */
13177 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13178 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13179 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13180 AssertRCReturn(rc, rc);
13181 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13182
13183 PVM pVM = pVCpu->CTX_SUFF(pVM);
13184 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
13185 {
13186 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
13187 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
13188 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
13189 if (RT_SUCCESS(rc))
13190 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
13191 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
13192 }
13193 else
13194 {
13195 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
13196 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
13197 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
13198 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
13199 }
13200
13201 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
13202 if (RT_SUCCESS(rc))
13203 {
13204 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13205 AssertRCReturn(rc2, rc2);
13206 return VINF_SUCCESS;
13207 }
13208 return rc;
13209}
13210
13211
13212/**
13213 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
13214 * Conditional VM-exit.
13215 */
13216HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13217{
13218 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13219 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
13220
13221 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13222 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13223 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13224 {
13225 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
13226 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
13227 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13228 {
13229 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
13230 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13231 }
13232 }
13233 else
13234 {
13235 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13236 rcStrict1 = VINF_SUCCESS;
13237 return rcStrict1;
13238 }
13239
13240 RTGCPHYS GCPhys = 0;
13241 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
13242
13243#if 0
13244 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
13245#else
13246 /* Aggressive state sync. for now. */
13247 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13248 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13249 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13250#endif
13251 AssertRCReturn(rc, rc);
13252
13253 /*
13254 * If we succeed, resume guest execution.
13255 * If we fail in interpreting the instruction because we couldn't get the guest physical address
13256 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
13257 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
13258 * weird case. See @bugref{6043}.
13259 */
13260 PVM pVM = pVCpu->CTX_SUFF(pVM);
13261 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
13262 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
13263 if ( rcStrict2 == VINF_SUCCESS
13264 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13265 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
13266 {
13267 /* Successfully handled MMIO operation. */
13268 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13269 | HM_CHANGED_GUEST_RSP
13270 | HM_CHANGED_GUEST_RFLAGS
13271 | HM_CHANGED_GUEST_APIC_STATE);
13272 return VINF_SUCCESS;
13273 }
13274 return rcStrict2;
13275}
13276
13277
13278/**
13279 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
13280 * VM-exit.
13281 */
13282HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13283{
13284 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13285 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
13286
13287 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13288 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13289 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13290 {
13291 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
13292 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13293 Log4(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
13294 }
13295 else
13296 {
13297 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13298 rcStrict1 = VINF_SUCCESS;
13299 return rcStrict1;
13300 }
13301
13302 RTGCPHYS GCPhys = 0;
13303 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
13304 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13305#if 0
13306 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
13307#else
13308 /* Aggressive state sync. for now. */
13309 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13310 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13311 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13312#endif
13313 AssertRCReturn(rc, rc);
13314
13315 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
13316 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
13317
13318 RTGCUINT uErrorCode = 0;
13319 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
13320 uErrorCode |= X86_TRAP_PF_ID;
13321 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
13322 uErrorCode |= X86_TRAP_PF_RW;
13323 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
13324 uErrorCode |= X86_TRAP_PF_P;
13325
13326 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
13327
13328 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
13329 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13330
13331 /* Handle the pagefault trap for the nested shadow table. */
13332 PVM pVM = pVCpu->CTX_SUFF(pVM);
13333 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
13334 TRPMResetTrap(pVCpu);
13335
13336 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
13337 if ( rcStrict2 == VINF_SUCCESS
13338 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13339 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
13340 {
13341 /* Successfully synced our nested page tables. */
13342 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
13343 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13344 | HM_CHANGED_GUEST_RSP
13345 | HM_CHANGED_GUEST_RFLAGS);
13346 return VINF_SUCCESS;
13347 }
13348
13349 Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
13350 return rcStrict2;
13351}
13352
13353/** @} */
13354
13355/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13356/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
13357/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13358
13359/** @name VM-exit exception handlers.
13360 * @{
13361 */
13362
13363/**
13364 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
13365 */
13366static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13367{
13368 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13369 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13370
13371 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13372 AssertRCReturn(rc, rc);
13373
13374 if (!(pMixedCtx->cr0 & X86_CR0_NE))
13375 {
13376 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13377 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13378
13379 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13380 * provides VM-exit instruction length. If this causes problem later,
13381 * disassemble the instruction like it's done on AMD-V. */
13382 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13383 AssertRCReturn(rc2, rc2);
13384 return rc;
13385 }
13386
13387 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13388 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13389 return rc;
13390}
13391
13392
13393/**
13394 * VM-exit exception handler for \#BP (Breakpoint exception).
13395 */
13396static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13397{
13398 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13399 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13400
13401 /** @todo Try optimize this by not saving the entire guest state unless
13402 * really needed. */
13403 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13404 AssertRCReturn(rc, rc);
13405
13406 PVM pVM = pVCpu->CTX_SUFF(pVM);
13407 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
13408 if (rc == VINF_EM_RAW_GUEST_TRAP)
13409 {
13410 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13411 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13412 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13413 AssertRCReturn(rc, rc);
13414
13415 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13416 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13417 }
13418
13419 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
13420 return rc;
13421}
13422
13423
13424/**
13425 * VM-exit exception handler for \#AC (alignment check exception).
13426 */
13427static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13428{
13429 RT_NOREF_PV(pMixedCtx);
13430 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13431
13432 /*
13433 * Re-inject it. We'll detect any nesting before getting here.
13434 */
13435 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13436 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13437 AssertRCReturn(rc, rc);
13438 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13439
13440 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13441 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13442 return VINF_SUCCESS;
13443}
13444
13445
13446/**
13447 * VM-exit exception handler for \#DB (Debug exception).
13448 */
13449static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13450{
13451 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13452 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13453 Log6(("XcptDB\n"));
13454
13455 /*
13456 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13457 * for processing.
13458 */
13459 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13460 AssertRCReturn(rc, rc);
13461
13462 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13463 uint64_t uDR6 = X86_DR6_INIT_VAL;
13464 uDR6 |= ( pVmxTransient->uExitQualification
13465 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
13466
13467 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13468 if (rc == VINF_EM_RAW_GUEST_TRAP)
13469 {
13470 /*
13471 * The exception was for the guest. Update DR6, DR7.GD and
13472 * IA32_DEBUGCTL.LBR before forwarding it.
13473 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13474 */
13475 VMMRZCallRing3Disable(pVCpu);
13476 HM_DISABLE_PREEMPT();
13477
13478 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
13479 pMixedCtx->dr[6] |= uDR6;
13480 if (CPUMIsGuestDebugStateActive(pVCpu))
13481 ASMSetDR6(pMixedCtx->dr[6]);
13482
13483 HM_RESTORE_PREEMPT();
13484 VMMRZCallRing3Enable(pVCpu);
13485
13486 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13487 AssertRCReturn(rc, rc);
13488
13489 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13490 pMixedCtx->dr[7] &= ~X86_DR7_GD;
13491
13492 /* Paranoia. */
13493 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13494 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
13495
13496 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
13497 AssertRCReturn(rc, rc);
13498
13499 /*
13500 * Raise #DB in the guest.
13501 *
13502 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
13503 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
13504 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
13505 *
13506 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
13507 */
13508 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13509 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13510 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13511 AssertRCReturn(rc, rc);
13512 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13513 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13514 return VINF_SUCCESS;
13515 }
13516
13517 /*
13518 * Not a guest trap, must be a hypervisor related debug event then.
13519 * Update DR6 in case someone is interested in it.
13520 */
13521 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13522 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13523 CPUMSetHyperDR6(pVCpu, uDR6);
13524
13525 return rc;
13526}
13527
13528
13529/**
13530 * VM-exit exception handler for \#NM (Device-not-available exception: floating
13531 * point exception).
13532 */
13533static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13534{
13535 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13536
13537 /* We require CR0 and EFER. EFER is always up-to-date. */
13538 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13539 AssertRCReturn(rc, rc);
13540
13541 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
13542 VMMRZCallRing3Disable(pVCpu);
13543 HM_DISABLE_PREEMPT();
13544
13545 /* If the guest FPU was active at the time of the #NM VM-exit, then it's a guest fault. */
13546 if (pVmxTransient->fWasGuestFPUStateActive)
13547 {
13548 rc = VINF_EM_RAW_GUEST_TRAP;
13549 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
13550 }
13551 else
13552 {
13553#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13554 Assert(!pVmxTransient->fWasGuestFPUStateActive || pVCpu->hm.s.fUsingDebugLoop);
13555#endif
13556 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu);
13557 Assert( rc == VINF_EM_RAW_GUEST_TRAP
13558 || ((rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
13559 if (rc == VINF_CPUM_HOST_CR0_MODIFIED)
13560 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
13561 }
13562
13563 HM_RESTORE_PREEMPT();
13564 VMMRZCallRing3Enable(pVCpu);
13565
13566 if (rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED)
13567 {
13568 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
13569 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13570 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
13571 pVCpu->hm.s.fPreloadGuestFpu = true;
13572 }
13573 else
13574 {
13575 /* Forward #NM to the guest. */
13576 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
13577 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13578 AssertRCReturn(rc, rc);
13579 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13580 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
13581 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13582 }
13583
13584 return VINF_SUCCESS;
13585}
13586
13587
13588/**
13589 * VM-exit exception handler for \#GP (General-protection exception).
13590 *
13591 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13592 */
13593static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13594{
13595 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13596 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13597
13598 int rc;
13599 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13600 { /* likely */ }
13601 else
13602 {
13603#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13604 Assert(pVCpu->hm.s.fUsingDebugLoop);
13605#endif
13606 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13607 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13608 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13609 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13610 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13611 AssertRCReturn(rc, rc);
13612 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13613 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13614 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13615 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13616 return rc;
13617 }
13618
13619 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13620 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13621
13622 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13623 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13624 AssertRCReturn(rc, rc);
13625
13626 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13627 uint32_t cbOp = 0;
13628 PVM pVM = pVCpu->CTX_SUFF(pVM);
13629 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13630 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13631 if (RT_SUCCESS(rc))
13632 {
13633 rc = VINF_SUCCESS;
13634 Assert(cbOp == pDis->cbInstr);
13635 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13636 switch (pDis->pCurInstr->uOpcode)
13637 {
13638 case OP_CLI:
13639 {
13640 pMixedCtx->eflags.Bits.u1IF = 0;
13641 pMixedCtx->eflags.Bits.u1RF = 0;
13642 pMixedCtx->rip += pDis->cbInstr;
13643 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13644 if ( !fDbgStepping
13645 && pMixedCtx->eflags.Bits.u1TF)
13646 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13647 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13648 break;
13649 }
13650
13651 case OP_STI:
13652 {
13653 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13654 pMixedCtx->eflags.Bits.u1IF = 1;
13655 pMixedCtx->eflags.Bits.u1RF = 0;
13656 pMixedCtx->rip += pDis->cbInstr;
13657 if (!fOldIF)
13658 {
13659 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13660 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13661 }
13662 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13663 if ( !fDbgStepping
13664 && pMixedCtx->eflags.Bits.u1TF)
13665 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13666 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13667 break;
13668 }
13669
13670 case OP_HLT:
13671 {
13672 rc = VINF_EM_HALT;
13673 pMixedCtx->rip += pDis->cbInstr;
13674 pMixedCtx->eflags.Bits.u1RF = 0;
13675 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13676 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13677 break;
13678 }
13679
13680 case OP_POPF:
13681 {
13682 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13683 uint32_t cbParm;
13684 uint32_t uMask;
13685 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13686 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13687 {
13688 cbParm = 4;
13689 uMask = 0xffffffff;
13690 }
13691 else
13692 {
13693 cbParm = 2;
13694 uMask = 0xffff;
13695 }
13696
13697 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13698 RTGCPTR GCPtrStack = 0;
13699 X86EFLAGS Eflags;
13700 Eflags.u32 = 0;
13701 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13702 &GCPtrStack);
13703 if (RT_SUCCESS(rc))
13704 {
13705 Assert(sizeof(Eflags.u32) >= cbParm);
13706 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13707 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13708 }
13709 if (RT_FAILURE(rc))
13710 {
13711 rc = VERR_EM_INTERPRETER;
13712 break;
13713 }
13714 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13715 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13716 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13717 pMixedCtx->esp += cbParm;
13718 pMixedCtx->esp &= uMask;
13719 pMixedCtx->rip += pDis->cbInstr;
13720 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13721 | HM_CHANGED_GUEST_RSP
13722 | HM_CHANGED_GUEST_RFLAGS);
13723 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13724 POPF restores EFLAGS.TF. */
13725 if ( !fDbgStepping
13726 && fGstStepping)
13727 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13728 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13729 break;
13730 }
13731
13732 case OP_PUSHF:
13733 {
13734 uint32_t cbParm;
13735 uint32_t uMask;
13736 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13737 {
13738 cbParm = 4;
13739 uMask = 0xffffffff;
13740 }
13741 else
13742 {
13743 cbParm = 2;
13744 uMask = 0xffff;
13745 }
13746
13747 /* Get the stack pointer & push the contents of eflags onto the stack. */
13748 RTGCPTR GCPtrStack = 0;
13749 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13750 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13751 if (RT_FAILURE(rc))
13752 {
13753 rc = VERR_EM_INTERPRETER;
13754 break;
13755 }
13756 X86EFLAGS Eflags = pMixedCtx->eflags;
13757 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13758 Eflags.Bits.u1RF = 0;
13759 Eflags.Bits.u1VM = 0;
13760
13761 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13762 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13763 {
13764 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13765 rc = VERR_EM_INTERPRETER;
13766 break;
13767 }
13768 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13769 pMixedCtx->esp -= cbParm;
13770 pMixedCtx->esp &= uMask;
13771 pMixedCtx->rip += pDis->cbInstr;
13772 pMixedCtx->eflags.Bits.u1RF = 0;
13773 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13774 | HM_CHANGED_GUEST_RSP
13775 | HM_CHANGED_GUEST_RFLAGS);
13776 if ( !fDbgStepping
13777 && pMixedCtx->eflags.Bits.u1TF)
13778 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13779 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13780 break;
13781 }
13782
13783 case OP_IRET:
13784 {
13785 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13786 * instruction reference. */
13787 RTGCPTR GCPtrStack = 0;
13788 uint32_t uMask = 0xffff;
13789 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13790 uint16_t aIretFrame[3];
13791 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13792 {
13793 rc = VERR_EM_INTERPRETER;
13794 break;
13795 }
13796 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13797 &GCPtrStack);
13798 if (RT_SUCCESS(rc))
13799 {
13800 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13801 PGMACCESSORIGIN_HM));
13802 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13803 }
13804 if (RT_FAILURE(rc))
13805 {
13806 rc = VERR_EM_INTERPRETER;
13807 break;
13808 }
13809 pMixedCtx->eip = 0;
13810 pMixedCtx->ip = aIretFrame[0];
13811 pMixedCtx->cs.Sel = aIretFrame[1];
13812 pMixedCtx->cs.ValidSel = aIretFrame[1];
13813 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13814 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13815 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13816 pMixedCtx->sp += sizeof(aIretFrame);
13817 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13818 | HM_CHANGED_GUEST_SEGMENT_REGS
13819 | HM_CHANGED_GUEST_RSP
13820 | HM_CHANGED_GUEST_RFLAGS);
13821 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13822 if ( !fDbgStepping
13823 && fGstStepping)
13824 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13825 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13826 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13827 break;
13828 }
13829
13830 case OP_INT:
13831 {
13832 uint16_t uVector = pDis->Param1.uValue & 0xff;
13833 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13834 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13835 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13836 break;
13837 }
13838
13839 case OP_INTO:
13840 {
13841 if (pMixedCtx->eflags.Bits.u1OF)
13842 {
13843 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13844 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13845 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13846 }
13847 else
13848 {
13849 pMixedCtx->eflags.Bits.u1RF = 0;
13850 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13851 }
13852 break;
13853 }
13854
13855 default:
13856 {
13857 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13858 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13859 EMCODETYPE_SUPERVISOR);
13860 rc = VBOXSTRICTRC_VAL(rc2);
13861 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13862 /** @todo We have to set pending-debug exceptions here when the guest is
13863 * single-stepping depending on the instruction that was interpreted. */
13864 Log4(("#GP rc=%Rrc\n", rc));
13865 break;
13866 }
13867 }
13868 }
13869 else
13870 rc = VERR_EM_INTERPRETER;
13871
13872 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13873 ("#GP Unexpected rc=%Rrc\n", rc));
13874 return rc;
13875}
13876
13877
13878/**
13879 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13880 * the exception reported in the VMX transient structure back into the VM.
13881 *
13882 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13883 * up-to-date.
13884 */
13885static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13886{
13887 RT_NOREF_PV(pMixedCtx);
13888 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13889#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13890 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13891 ("uVector=%#04x u32XcptBitmap=%#010RX32\n",
13892 VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13893#endif
13894
13895 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13896 hmR0VmxCheckExitDueToEventDelivery(). */
13897 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13898 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13899 AssertRCReturn(rc, rc);
13900 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13901
13902#ifdef DEBUG_ramshankar
13903 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13904 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13905 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13906#endif
13907
13908 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13909 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13910 return VINF_SUCCESS;
13911}
13912
13913
13914/**
13915 * VM-exit exception handler for \#PF (Page-fault exception).
13916 */
13917static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13918{
13919 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13920 PVM pVM = pVCpu->CTX_SUFF(pVM);
13921 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13922 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13923 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13924 AssertRCReturn(rc, rc);
13925
13926 if (!pVM->hm.s.fNestedPaging)
13927 { /* likely */ }
13928 else
13929 {
13930#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13931 Assert(pVCpu->hm.s.fUsingDebugLoop);
13932#endif
13933 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13934 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13935 {
13936 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13937 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13938 }
13939 else
13940 {
13941 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13942 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13943 Log4(("Pending #DF due to vectoring #PF. NP\n"));
13944 }
13945 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13946 return rc;
13947 }
13948
13949 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13950 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13951 if (pVmxTransient->fVectoringPF)
13952 {
13953 Assert(pVCpu->hm.s.Event.fPending);
13954 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13955 }
13956
13957 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13958 AssertRCReturn(rc, rc);
13959
13960 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13961 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13962
13963 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13964 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13965 (RTGCPTR)pVmxTransient->uExitQualification);
13966
13967 Log4(("#PF: rc=%Rrc\n", rc));
13968 if (rc == VINF_SUCCESS)
13969 {
13970#if 0
13971 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13972 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
13973 * memory? We don't update the whole state here... */
13974 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13975 | HM_CHANGED_GUEST_RSP
13976 | HM_CHANGED_GUEST_RFLAGS
13977 | HM_CHANGED_GUEST_APIC_STATE);
13978#else
13979 /*
13980 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13981 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13982 */
13983 /** @todo take advantage of CPUM changed flags instead of brute forcing. */
13984 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13985#endif
13986 TRPMResetTrap(pVCpu);
13987 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13988 return rc;
13989 }
13990
13991 if (rc == VINF_EM_RAW_GUEST_TRAP)
13992 {
13993 if (!pVmxTransient->fVectoringDoublePF)
13994 {
13995 /* It's a guest page fault and needs to be reflected to the guest. */
13996 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13997 TRPMResetTrap(pVCpu);
13998 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13999 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14000 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
14001 }
14002 else
14003 {
14004 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
14005 TRPMResetTrap(pVCpu);
14006 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
14007 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
14008 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
14009 }
14010
14011 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
14012 return VINF_SUCCESS;
14013 }
14014
14015 TRPMResetTrap(pVCpu);
14016 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
14017 return rc;
14018}
14019
14020/** @} */
14021
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette