VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 72620

Last change on this file since 72620 was 72620, checked in by vboxsync, 7 years ago

HMVMXR0.cpp: Retry new RDTSC and RDTSCP code with HM_CHANGED_XXXX flags set (duh). bugref:6973
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 605.0 KB
Line 
1/* $Id: HMVMXR0.cpp 72620 2018-06-20 09:53:34Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/x86.h>
25#include <iprt/asm-amd64-x86.h>
26#include <iprt/thread.h>
27
28#include <VBox/vmm/pdmapi.h>
29#include <VBox/vmm/dbgf.h>
30#include <VBox/vmm/iem.h>
31#include <VBox/vmm/iom.h>
32#include <VBox/vmm/selm.h>
33#include <VBox/vmm/tm.h>
34#include <VBox/vmm/gim.h>
35#include <VBox/vmm/apic.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#include "HMInternal.h"
40#include <VBox/vmm/vm.h>
41#include "HMVMXR0.h"
42#include "dtrace/VBoxVMM.h"
43
44#define HMVMX_USE_IEM_EVENT_REFLECTION
45#ifdef DEBUG_ramshankar
46# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
47# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
49# define HMVMX_ALWAYS_CHECK_GUEST_STATE
50# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
51# define HMVMX_ALWAYS_TRAP_PF
52# define HMVMX_ALWAYS_FLUSH_TLB
53# define HMVMX_ALWAYS_SWAP_EFER
54#endif
55
56
57/*********************************************************************************************************************************
58* Defined Constants And Macros *
59*********************************************************************************************************************************/
60/** Use the function table. */
61#define HMVMX_USE_FUNCTION_TABLE
62
63/** Determine which tagged-TLB flush handler to use. */
64#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
65#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
66#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
67#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
68
69/** @name Updated-guest-state flags.
70 * @{ */
71#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
72#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
73#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
74#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
75#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
76#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
77#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
78#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
79#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
80#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
81#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
82#define HMVMX_UPDATED_GUEST_DR7 RT_BIT(11)
83#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
84#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
85#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
86#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
87#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
88#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
89#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
90#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
91#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
92 | HMVMX_UPDATED_GUEST_RSP \
93 | HMVMX_UPDATED_GUEST_RFLAGS \
94 | HMVMX_UPDATED_GUEST_CR0 \
95 | HMVMX_UPDATED_GUEST_CR3 \
96 | HMVMX_UPDATED_GUEST_CR4 \
97 | HMVMX_UPDATED_GUEST_GDTR \
98 | HMVMX_UPDATED_GUEST_IDTR \
99 | HMVMX_UPDATED_GUEST_LDTR \
100 | HMVMX_UPDATED_GUEST_TR \
101 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
102 | HMVMX_UPDATED_GUEST_DR7 \
103 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
104 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
105 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
106 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
107 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
108 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
109 | HMVMX_UPDATED_GUEST_INTR_STATE \
110 | HMVMX_UPDATED_GUEST_APIC_STATE)
111/** @} */
112
113/** @name
114 * Flags to skip redundant reads of some common VMCS fields that are not part of
115 * the guest-CPU state but are in the transient structure.
116 */
117#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
118#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
119#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
120#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
123#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
124/** @} */
125
126/** @name
127 * States of the VMCS.
128 *
129 * This does not reflect all possible VMCS states but currently only those
130 * needed for maintaining the VMCS consistently even when thread-context hooks
131 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
132 */
133#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
134#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
135#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
136/** @} */
137
138/**
139 * Exception bitmap mask for real-mode guests (real-on-v86).
140 *
141 * We need to intercept all exceptions manually except:
142 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
143 * due to bugs in Intel CPUs.
144 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
145 * support.
146 */
147#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
148 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
149 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
150 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
151 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
152 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
153 | RT_BIT(X86_XCPT_XF))
154
155/**
156 * Exception bitmap mask for all contributory exceptions.
157 *
158 * Page fault is deliberately excluded here as it's conditional as to whether
159 * it's contributory or benign. Page faults are handled separately.
160 */
161#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
162 | RT_BIT(X86_XCPT_DE))
163
164/** Maximum VM-instruction error number. */
165#define HMVMX_INSTR_ERROR_MAX 28
166
167/** Profiling macro. */
168#ifdef HM_PROFILE_EXIT_DISPATCH
169# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
170# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
171#else
172# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
173# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
174#endif
175
176/** Assert that preemption is disabled or covered by thread-context hooks. */
177#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
178 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
179
180/** Assert that we haven't migrated CPUs when thread-context hooks are not
181 * used. */
182#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
183 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
184 ("Illegal migration! Entered on CPU %u Current %u\n", \
185 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
186
187/** Helper macro for VM-exit handlers called unexpectedly. */
188#define HMVMX_RETURN_UNEXPECTED_EXIT() \
189 do { \
190 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
191 return VERR_VMX_UNEXPECTED_EXIT; \
192 } while (0)
193
194
195/*********************************************************************************************************************************
196* Structures and Typedefs *
197*********************************************************************************************************************************/
198/**
199 * VMX transient state.
200 *
201 * A state structure for holding miscellaneous information across
202 * VMX non-root operation and restored after the transition.
203 */
204typedef struct VMXTRANSIENT
205{
206 /** The host's rflags/eflags. */
207 RTCCUINTREG fEFlags;
208#if HC_ARCH_BITS == 32
209 uint32_t u32Alignment0;
210#endif
211 /** The guest's TPR value used for TPR shadowing. */
212 uint8_t u8GuestTpr;
213 /** Alignment. */
214 uint8_t abAlignment0[7];
215
216 /** The basic VM-exit reason. */
217 uint16_t uExitReason;
218 /** Alignment. */
219 uint16_t u16Alignment0;
220 /** The VM-exit interruption error code. */
221 uint32_t uExitIntErrorCode;
222 /** The VM-exit exit code qualification. */
223 uint64_t uExitQualification;
224
225 /** The VM-exit interruption-information field. */
226 uint32_t uExitIntInfo;
227 /** The VM-exit instruction-length field. */
228 uint32_t cbInstr;
229 /** The VM-exit instruction-information field. */
230 union
231 {
232 /** Plain unsigned int representation. */
233 uint32_t u;
234 /** INS and OUTS information. */
235 struct
236 {
237 uint32_t u7Reserved0 : 7;
238 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
239 uint32_t u3AddrSize : 3;
240 uint32_t u5Reserved1 : 5;
241 /** The segment register (X86_SREG_XXX). */
242 uint32_t iSegReg : 3;
243 uint32_t uReserved2 : 14;
244 } StrIo;
245 /** INVEPT, INVVPID, INVPCID information. */
246 struct
247 {
248 /** Scaling; 0=no scaling, 1=scale-by-2, 2=scale-by-4, 3=scale-by-8. */
249 uint32_t u2Scaling : 2;
250 uint32_t u5Reserved0 : 5;
251 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
252 uint32_t u3AddrSize : 3;
253 uint32_t u1Reserved0 : 1;
254 uint32_t u4Reserved0 : 4;
255 /** The segment register (X86_SREG_XXX). */
256 uint32_t iSegReg : 3;
257 /** The index register (X86_GREG_XXX). */
258 uint32_t iIdxReg : 4;
259 /** Set if index register is invalid. */
260 uint32_t fIdxRegValid : 1;
261 /** The base register (X86_GREG_XXX). */
262 uint32_t iBaseReg : 4;
263 /** Set if base register is invalid. */
264 uint32_t fBaseRegValid : 1;
265 /** Register 2 (X86_GREG_XXX). */
266 uint32_t iReg2 : 4;
267 } Inv;
268 } ExitInstrInfo;
269 /** Whether the VM-entry failed or not. */
270 bool fVMEntryFailed;
271 /** Alignment. */
272 uint8_t abAlignment1[3];
273
274 /** The VM-entry interruption-information field. */
275 uint32_t uEntryIntInfo;
276 /** The VM-entry exception error code field. */
277 uint32_t uEntryXcptErrorCode;
278 /** The VM-entry instruction length field. */
279 uint32_t cbEntryInstr;
280
281 /** IDT-vectoring information field. */
282 uint32_t uIdtVectoringInfo;
283 /** IDT-vectoring error code. */
284 uint32_t uIdtVectoringErrorCode;
285
286 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
287 uint32_t fVmcsFieldsRead;
288
289 /** Whether the guest debug state was active at the time of VM-exit. */
290 bool fWasGuestDebugStateActive;
291 /** Whether the hyper debug state was active at the time of VM-exit. */
292 bool fWasHyperDebugStateActive;
293 /** Whether TSC-offsetting should be setup before VM-entry. */
294 bool fUpdateTscOffsettingAndPreemptTimer;
295 /** Whether the VM-exit was caused by a page-fault during delivery of a
296 * contributory exception or a page-fault. */
297 bool fVectoringDoublePF;
298 /** Whether the VM-exit was caused by a page-fault during delivery of an
299 * external interrupt or NMI. */
300 bool fVectoringPF;
301} VMXTRANSIENT;
302AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
303AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
304AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
305AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
306AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
307/** Pointer to VMX transient state. */
308typedef VMXTRANSIENT *PVMXTRANSIENT;
309
310
311/**
312 * MSR-bitmap read permissions.
313 */
314typedef enum VMXMSREXITREAD
315{
316 /** Reading this MSR causes a VM-exit. */
317 VMXMSREXIT_INTERCEPT_READ = 0xb,
318 /** Reading this MSR does not cause a VM-exit. */
319 VMXMSREXIT_PASSTHRU_READ
320} VMXMSREXITREAD;
321/** Pointer to MSR-bitmap read permissions. */
322typedef VMXMSREXITREAD* PVMXMSREXITREAD;
323
324/**
325 * MSR-bitmap write permissions.
326 */
327typedef enum VMXMSREXITWRITE
328{
329 /** Writing to this MSR causes a VM-exit. */
330 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
331 /** Writing to this MSR does not cause a VM-exit. */
332 VMXMSREXIT_PASSTHRU_WRITE
333} VMXMSREXITWRITE;
334/** Pointer to MSR-bitmap write permissions. */
335typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
336
337
338/**
339 * VMX VM-exit handler.
340 *
341 * @returns Strict VBox status code (i.e. informational status codes too).
342 * @param pVCpu The cross context virtual CPU structure.
343 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
344 * out-of-sync. Make sure to update the required
345 * fields before using them.
346 * @param pVmxTransient Pointer to the VMX-transient structure.
347 */
348#ifndef HMVMX_USE_FUNCTION_TABLE
349typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
350#else
351typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
352/** Pointer to VM-exit handler. */
353typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
354#endif
355
356/**
357 * VMX VM-exit handler, non-strict status code.
358 *
359 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
360 *
361 * @returns VBox status code, no informational status code returned.
362 * @param pVCpu The cross context virtual CPU structure.
363 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
364 * out-of-sync. Make sure to update the required
365 * fields before using them.
366 * @param pVmxTransient Pointer to the VMX-transient structure.
367 *
368 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
369 * use of that status code will be replaced with VINF_EM_SOMETHING
370 * later when switching over to IEM.
371 */
372#ifndef HMVMX_USE_FUNCTION_TABLE
373typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
374#else
375typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
376#endif
377
378
379/*********************************************************************************************************************************
380* Internal Functions *
381*********************************************************************************************************************************/
382static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
383static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
384static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
385static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
386 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
387 bool fStepping, uint32_t *puIntState);
388#if HC_ARCH_BITS == 32
389static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
390#endif
391#ifndef HMVMX_USE_FUNCTION_TABLE
392DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
393# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
394# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
395#else
396# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
397# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
398#endif
399
400
401/** @name VM-exit handlers.
402 * @{
403 */
404static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
405static FNVMXEXITHANDLER hmR0VmxExitExtInt;
406static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
407static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
408static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
409static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
410static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
411static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
412static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
413static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
414static FNVMXEXITHANDLER hmR0VmxExitCpuid;
415static FNVMXEXITHANDLER hmR0VmxExitGetsec;
416static FNVMXEXITHANDLER hmR0VmxExitHlt;
417static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
418static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
419static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
420static FNVMXEXITHANDLER hmR0VmxExitVmcall;
421static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
422static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
423static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
424static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
425static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
426static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
427static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
428static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
429static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
430static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
431static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
432static FNVMXEXITHANDLER hmR0VmxExitMwait;
433static FNVMXEXITHANDLER hmR0VmxExitMtf;
434static FNVMXEXITHANDLER hmR0VmxExitMonitor;
435static FNVMXEXITHANDLER hmR0VmxExitPause;
436static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
437static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
438static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
439static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
440static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
441static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
442static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
443static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
444static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
445static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
446static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
447static FNVMXEXITHANDLER hmR0VmxExitRdrand;
448static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
449/** @} */
450
451static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
452static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
453static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
454static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
455static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
456static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
457static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
458static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
459
460
461/*********************************************************************************************************************************
462* Global Variables *
463*********************************************************************************************************************************/
464#ifdef HMVMX_USE_FUNCTION_TABLE
465
466/**
467 * VMX_EXIT dispatch table.
468 */
469static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
470{
471 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
472 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
473 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
474 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
475 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
476 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
477 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
478 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
479 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
480 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
481 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
482 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
483 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
484 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
485 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
486 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
487 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
488 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
489 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
490 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
491 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
492 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
493 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
494 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
495 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
496 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
497 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
498 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
499 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
500 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
501 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
502 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
503 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
504 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
505 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
506 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
507 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
508 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
509 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
510 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
511 /* 40 UNDEFINED */ hmR0VmxExitPause,
512 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
513 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
514 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
515 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
516 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
517 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
518 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
519 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
520 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
521 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
522 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
523 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
524 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
525 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
526 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
527 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
528 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
529 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
530 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
531 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
532 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
533 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
534 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
535 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
536};
537#endif /* HMVMX_USE_FUNCTION_TABLE */
538
539#ifdef VBOX_STRICT
540static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
541{
542 /* 0 */ "(Not Used)",
543 /* 1 */ "VMCALL executed in VMX root operation.",
544 /* 2 */ "VMCLEAR with invalid physical address.",
545 /* 3 */ "VMCLEAR with VMXON pointer.",
546 /* 4 */ "VMLAUNCH with non-clear VMCS.",
547 /* 5 */ "VMRESUME with non-launched VMCS.",
548 /* 6 */ "VMRESUME after VMXOFF",
549 /* 7 */ "VM-entry with invalid control fields.",
550 /* 8 */ "VM-entry with invalid host state fields.",
551 /* 9 */ "VMPTRLD with invalid physical address.",
552 /* 10 */ "VMPTRLD with VMXON pointer.",
553 /* 11 */ "VMPTRLD with incorrect revision identifier.",
554 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
555 /* 13 */ "VMWRITE to read-only VMCS component.",
556 /* 14 */ "(Not Used)",
557 /* 15 */ "VMXON executed in VMX root operation.",
558 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
559 /* 17 */ "VM-entry with non-launched executing VMCS.",
560 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
561 /* 19 */ "VMCALL with non-clear VMCS.",
562 /* 20 */ "VMCALL with invalid VM-exit control fields.",
563 /* 21 */ "(Not Used)",
564 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
565 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
566 /* 24 */ "VMCALL with invalid SMM-monitor features.",
567 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
568 /* 26 */ "VM-entry with events blocked by MOV SS.",
569 /* 27 */ "(Not Used)",
570 /* 28 */ "Invalid operand to INVEPT/INVVPID."
571};
572#endif /* VBOX_STRICT */
573
574
575
576/**
577 * Updates the VM's last error record.
578 *
579 * If there was a VMX instruction error, reads the error data from the VMCS and
580 * updates VCPU's last error record as well.
581 *
582 * @param pVM The cross context VM structure.
583 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
584 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
585 * VERR_VMX_INVALID_VMCS_FIELD.
586 * @param rc The error code.
587 */
588static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
589{
590 AssertPtr(pVM);
591 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
592 || rc == VERR_VMX_UNABLE_TO_START_VM)
593 {
594 AssertPtrReturnVoid(pVCpu);
595 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
596 }
597 pVM->hm.s.lLastError = rc;
598}
599
600
601/**
602 * Reads the VM-entry interruption-information field from the VMCS into the VMX
603 * transient structure.
604 *
605 * @returns VBox status code.
606 * @param pVmxTransient Pointer to the VMX transient structure.
607 *
608 * @remarks No-long-jump zone!!!
609 */
610DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
611{
612 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
613 AssertRCReturn(rc, rc);
614 return VINF_SUCCESS;
615}
616
617
618#ifdef VBOX_STRICT
619/**
620 * Reads the VM-entry exception error code field from the VMCS into
621 * the VMX transient structure.
622 *
623 * @returns VBox status code.
624 * @param pVmxTransient Pointer to the VMX transient structure.
625 *
626 * @remarks No-long-jump zone!!!
627 */
628DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
629{
630 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
631 AssertRCReturn(rc, rc);
632 return VINF_SUCCESS;
633}
634#endif /* VBOX_STRICT */
635
636
637#ifdef VBOX_STRICT
638/**
639 * Reads the VM-entry exception error code field from the VMCS into
640 * the VMX transient structure.
641 *
642 * @returns VBox status code.
643 * @param pVmxTransient Pointer to the VMX transient structure.
644 *
645 * @remarks No-long-jump zone!!!
646 */
647DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
648{
649 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
650 AssertRCReturn(rc, rc);
651 return VINF_SUCCESS;
652}
653#endif /* VBOX_STRICT */
654
655
656/**
657 * Reads the VM-exit interruption-information field from the VMCS into the VMX
658 * transient structure.
659 *
660 * @returns VBox status code.
661 * @param pVmxTransient Pointer to the VMX transient structure.
662 */
663DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
664{
665 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
666 {
667 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
668 AssertRCReturn(rc, rc);
669 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
670 }
671 return VINF_SUCCESS;
672}
673
674
675/**
676 * Reads the VM-exit interruption error code from the VMCS into the VMX
677 * transient structure.
678 *
679 * @returns VBox status code.
680 * @param pVmxTransient Pointer to the VMX transient structure.
681 */
682DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
683{
684 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
685 {
686 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
687 AssertRCReturn(rc, rc);
688 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
689 }
690 return VINF_SUCCESS;
691}
692
693
694/**
695 * Reads the VM-exit instruction length field from the VMCS into the VMX
696 * transient structure.
697 *
698 * @returns VBox status code.
699 * @param pVmxTransient Pointer to the VMX transient structure.
700 */
701DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
702{
703 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
704 {
705 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
706 AssertRCReturn(rc, rc);
707 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
708 }
709 return VINF_SUCCESS;
710}
711
712
713/**
714 * Reads the VM-exit instruction-information field from the VMCS into
715 * the VMX transient structure.
716 *
717 * @returns VBox status code.
718 * @param pVmxTransient Pointer to the VMX transient structure.
719 */
720DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
721{
722 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
723 {
724 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
725 AssertRCReturn(rc, rc);
726 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
727 }
728 return VINF_SUCCESS;
729}
730
731
732/**
733 * Reads the exit code qualification from the VMCS into the VMX transient
734 * structure.
735 *
736 * @returns VBox status code.
737 * @param pVCpu The cross context virtual CPU structure of the
738 * calling EMT. (Required for the VMCS cache case.)
739 * @param pVmxTransient Pointer to the VMX transient structure.
740 */
741DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
742{
743 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
744 {
745 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
746 AssertRCReturn(rc, rc);
747 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
748 }
749 return VINF_SUCCESS;
750}
751
752
753/**
754 * Reads the IDT-vectoring information field from the VMCS into the VMX
755 * transient structure.
756 *
757 * @returns VBox status code.
758 * @param pVmxTransient Pointer to the VMX transient structure.
759 *
760 * @remarks No-long-jump zone!!!
761 */
762DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
763{
764 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
765 {
766 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
767 AssertRCReturn(rc, rc);
768 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
769 }
770 return VINF_SUCCESS;
771}
772
773
774/**
775 * Reads the IDT-vectoring error code from the VMCS into the VMX
776 * transient structure.
777 *
778 * @returns VBox status code.
779 * @param pVmxTransient Pointer to the VMX transient structure.
780 */
781DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
782{
783 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
784 {
785 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
786 AssertRCReturn(rc, rc);
787 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
788 }
789 return VINF_SUCCESS;
790}
791
792
793/**
794 * Enters VMX root mode operation on the current CPU.
795 *
796 * @returns VBox status code.
797 * @param pVM The cross context VM structure. Can be
798 * NULL, after a resume.
799 * @param HCPhysCpuPage Physical address of the VMXON region.
800 * @param pvCpuPage Pointer to the VMXON region.
801 */
802static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
803{
804 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
805 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
806 Assert(pvCpuPage);
807 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
808
809 if (pVM)
810 {
811 /* Write the VMCS revision dword to the VMXON region. */
812 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
813 }
814
815 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
816 RTCCUINTREG fEFlags = ASMIntDisableFlags();
817
818 /* Enable the VMX bit in CR4 if necessary. */
819 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
820
821 /* Enter VMX root mode. */
822 int rc = VMXEnable(HCPhysCpuPage);
823 if (RT_FAILURE(rc))
824 {
825 if (!(uOldCr4 & X86_CR4_VMXE))
826 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
827
828 if (pVM)
829 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
830 }
831
832 /* Restore interrupts. */
833 ASMSetFlags(fEFlags);
834 return rc;
835}
836
837
838/**
839 * Exits VMX root mode operation on the current CPU.
840 *
841 * @returns VBox status code.
842 */
843static int hmR0VmxLeaveRootMode(void)
844{
845 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
846
847 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
848 RTCCUINTREG fEFlags = ASMIntDisableFlags();
849
850 /* If we're for some reason not in VMX root mode, then don't leave it. */
851 RTCCUINTREG uHostCR4 = ASMGetCR4();
852
853 int rc;
854 if (uHostCR4 & X86_CR4_VMXE)
855 {
856 /* Exit VMX root mode and clear the VMX bit in CR4. */
857 VMXDisable();
858 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
859 rc = VINF_SUCCESS;
860 }
861 else
862 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
863
864 /* Restore interrupts. */
865 ASMSetFlags(fEFlags);
866 return rc;
867}
868
869
870/**
871 * Allocates and maps one physically contiguous page. The allocated page is
872 * zero'd out. (Used by various VT-x structures).
873 *
874 * @returns IPRT status code.
875 * @param pMemObj Pointer to the ring-0 memory object.
876 * @param ppVirt Where to store the virtual address of the
877 * allocation.
878 * @param pHCPhys Where to store the physical address of the
879 * allocation.
880 */
881DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
882{
883 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
884 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
885 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
886
887 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
888 if (RT_FAILURE(rc))
889 return rc;
890 *ppVirt = RTR0MemObjAddress(*pMemObj);
891 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
892 ASMMemZero32(*ppVirt, PAGE_SIZE);
893 return VINF_SUCCESS;
894}
895
896
897/**
898 * Frees and unmaps an allocated physical page.
899 *
900 * @param pMemObj Pointer to the ring-0 memory object.
901 * @param ppVirt Where to re-initialize the virtual address of
902 * allocation as 0.
903 * @param pHCPhys Where to re-initialize the physical address of the
904 * allocation as 0.
905 */
906DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
907{
908 AssertPtr(pMemObj);
909 AssertPtr(ppVirt);
910 AssertPtr(pHCPhys);
911 if (*pMemObj != NIL_RTR0MEMOBJ)
912 {
913 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
914 AssertRC(rc);
915 *pMemObj = NIL_RTR0MEMOBJ;
916 *ppVirt = 0;
917 *pHCPhys = 0;
918 }
919}
920
921
922/**
923 * Worker function to free VT-x related structures.
924 *
925 * @returns IPRT status code.
926 * @param pVM The cross context VM structure.
927 */
928static void hmR0VmxStructsFree(PVM pVM)
929{
930 for (VMCPUID i = 0; i < pVM->cCpus; i++)
931 {
932 PVMCPU pVCpu = &pVM->aCpus[i];
933 AssertPtr(pVCpu);
934
935 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
936 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
937
938 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
939 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
940
941 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
942 }
943
944 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
945#ifdef VBOX_WITH_CRASHDUMP_MAGIC
946 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
947#endif
948}
949
950
951/**
952 * Worker function to allocate VT-x related VM structures.
953 *
954 * @returns IPRT status code.
955 * @param pVM The cross context VM structure.
956 */
957static int hmR0VmxStructsAlloc(PVM pVM)
958{
959 /*
960 * Initialize members up-front so we can cleanup properly on allocation failure.
961 */
962#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
963 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
964 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
965 pVM->hm.s.vmx.HCPhys##a_Name = 0;
966
967#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
968 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
969 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
970 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
971
972#ifdef VBOX_WITH_CRASHDUMP_MAGIC
973 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
974#endif
975 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
976
977 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
978 for (VMCPUID i = 0; i < pVM->cCpus; i++)
979 {
980 PVMCPU pVCpu = &pVM->aCpus[i];
981 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
982 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
983 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
984 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
985 }
986#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
987#undef VMXLOCAL_INIT_VM_MEMOBJ
988
989 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
990 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
991 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
992 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
993
994 /*
995 * Allocate all the VT-x structures.
996 */
997 int rc = VINF_SUCCESS;
998#ifdef VBOX_WITH_CRASHDUMP_MAGIC
999 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1000 if (RT_FAILURE(rc))
1001 goto cleanup;
1002 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1003 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1004#endif
1005
1006 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1007 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1008 {
1009 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1010 &pVM->hm.s.vmx.HCPhysApicAccess);
1011 if (RT_FAILURE(rc))
1012 goto cleanup;
1013 }
1014
1015 /*
1016 * Initialize per-VCPU VT-x structures.
1017 */
1018 for (VMCPUID i = 0; i < pVM->cCpus; i++)
1019 {
1020 PVMCPU pVCpu = &pVM->aCpus[i];
1021 AssertPtr(pVCpu);
1022
1023 /* Allocate the VM control structure (VMCS). */
1024 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1025 if (RT_FAILURE(rc))
1026 goto cleanup;
1027
1028 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1029 if ( PDMHasApic(pVM)
1030 && (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW))
1031 {
1032 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1033 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1034 if (RT_FAILURE(rc))
1035 goto cleanup;
1036 }
1037
1038 /*
1039 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1040 * transparent accesses of specific MSRs.
1041 *
1042 * If the condition for enabling MSR bitmaps changes here, don't forget to
1043 * update HMAreMsrBitmapsAvailable().
1044 */
1045 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1046 {
1047 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1048 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1049 if (RT_FAILURE(rc))
1050 goto cleanup;
1051 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1052 }
1053
1054 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1055 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1056 if (RT_FAILURE(rc))
1057 goto cleanup;
1058
1059 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1060 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1061 if (RT_FAILURE(rc))
1062 goto cleanup;
1063 }
1064
1065 return VINF_SUCCESS;
1066
1067cleanup:
1068 hmR0VmxStructsFree(pVM);
1069 return rc;
1070}
1071
1072
1073/**
1074 * Does global VT-x initialization (called during module initialization).
1075 *
1076 * @returns VBox status code.
1077 */
1078VMMR0DECL(int) VMXR0GlobalInit(void)
1079{
1080#ifdef HMVMX_USE_FUNCTION_TABLE
1081 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1082# ifdef VBOX_STRICT
1083 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1084 Assert(g_apfnVMExitHandlers[i]);
1085# endif
1086#endif
1087 return VINF_SUCCESS;
1088}
1089
1090
1091/**
1092 * Does global VT-x termination (called during module termination).
1093 */
1094VMMR0DECL(void) VMXR0GlobalTerm()
1095{
1096 /* Nothing to do currently. */
1097}
1098
1099
1100/**
1101 * Sets up and activates VT-x on the current CPU.
1102 *
1103 * @returns VBox status code.
1104 * @param pCpu Pointer to the global CPU info struct.
1105 * @param pVM The cross context VM structure. Can be
1106 * NULL after a host resume operation.
1107 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1108 * fEnabledByHost is @c true).
1109 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1110 * @a fEnabledByHost is @c true).
1111 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1112 * enable VT-x on the host.
1113 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1114 */
1115VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1116 void *pvMsrs)
1117{
1118 Assert(pCpu);
1119 Assert(pvMsrs);
1120 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1121
1122 /* Enable VT-x if it's not already enabled by the host. */
1123 if (!fEnabledByHost)
1124 {
1125 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1126 if (RT_FAILURE(rc))
1127 return rc;
1128 }
1129
1130 /*
1131 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1132 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1133 */
1134 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1135 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1136 {
1137 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1138 pCpu->fFlushAsidBeforeUse = false;
1139 }
1140 else
1141 pCpu->fFlushAsidBeforeUse = true;
1142
1143 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1144 ++pCpu->cTlbFlushes;
1145
1146 return VINF_SUCCESS;
1147}
1148
1149
1150/**
1151 * Deactivates VT-x on the current CPU.
1152 *
1153 * @returns VBox status code.
1154 * @param pCpu Pointer to the global CPU info struct.
1155 * @param pvCpuPage Pointer to the VMXON region.
1156 * @param HCPhysCpuPage Physical address of the VMXON region.
1157 *
1158 * @remarks This function should never be called when SUPR0EnableVTx() or
1159 * similar was used to enable VT-x on the host.
1160 */
1161VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1162{
1163 NOREF(pCpu);
1164 NOREF(pvCpuPage);
1165 NOREF(HCPhysCpuPage);
1166
1167 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1168 return hmR0VmxLeaveRootMode();
1169}
1170
1171
1172/**
1173 * Sets the permission bits for the specified MSR in the MSR bitmap.
1174 *
1175 * @param pVCpu The cross context virtual CPU structure.
1176 * @param uMsr The MSR value.
1177 * @param enmRead Whether reading this MSR causes a VM-exit.
1178 * @param enmWrite Whether writing this MSR causes a VM-exit.
1179 */
1180static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1181{
1182 int32_t iBit;
1183 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1184
1185 /*
1186 * Layout:
1187 * 0x000 - 0x3ff - Low MSR read bits
1188 * 0x400 - 0x7ff - High MSR read bits
1189 * 0x800 - 0xbff - Low MSR write bits
1190 * 0xc00 - 0xfff - High MSR write bits
1191 */
1192 if (uMsr <= 0x00001FFF)
1193 iBit = uMsr;
1194 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1195 {
1196 iBit = uMsr - UINT32_C(0xC0000000);
1197 pbMsrBitmap += 0x400;
1198 }
1199 else
1200 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1201
1202 Assert(iBit <= 0x1fff);
1203 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1204 ASMBitSet(pbMsrBitmap, iBit);
1205 else
1206 ASMBitClear(pbMsrBitmap, iBit);
1207
1208 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1209 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1210 else
1211 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1212}
1213
1214
1215#ifdef VBOX_STRICT
1216/**
1217 * Gets the permission bits for the specified MSR in the MSR bitmap.
1218 *
1219 * @returns VBox status code.
1220 * @retval VINF_SUCCESS if the specified MSR is found.
1221 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1222 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1223 *
1224 * @param pVCpu The cross context virtual CPU structure.
1225 * @param uMsr The MSR.
1226 * @param penmRead Where to store the read permissions.
1227 * @param penmWrite Where to store the write permissions.
1228 */
1229static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1230{
1231 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1232 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1233 int32_t iBit;
1234 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1235
1236 /* See hmR0VmxSetMsrPermission() for the layout. */
1237 if (uMsr <= 0x00001FFF)
1238 iBit = uMsr;
1239 else if ( uMsr >= 0xC0000000
1240 && uMsr <= 0xC0001FFF)
1241 {
1242 iBit = (uMsr - 0xC0000000);
1243 pbMsrBitmap += 0x400;
1244 }
1245 else
1246 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1247
1248 Assert(iBit <= 0x1fff);
1249 if (ASMBitTest(pbMsrBitmap, iBit))
1250 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1251 else
1252 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1253
1254 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1255 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1256 else
1257 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1258 return VINF_SUCCESS;
1259}
1260#endif /* VBOX_STRICT */
1261
1262
1263/**
1264 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1265 * area.
1266 *
1267 * @returns VBox status code.
1268 * @param pVCpu The cross context virtual CPU structure.
1269 * @param cMsrs The number of MSRs.
1270 */
1271DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1272{
1273 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1274 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1275 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1276 {
1277 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1278 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1279 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1280 }
1281
1282 /* Update number of guest MSRs to load/store across the world-switch. */
1283 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1284 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1285
1286 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1287 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1288 AssertRCReturn(rc, rc);
1289
1290 /* Update the VCPU's copy of the MSR count. */
1291 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1292
1293 return VINF_SUCCESS;
1294}
1295
1296
1297/**
1298 * Adds a new (or updates the value of an existing) guest/host MSR
1299 * pair to be swapped during the world-switch as part of the
1300 * auto-load/store MSR area in the VMCS.
1301 *
1302 * @returns VBox status code.
1303 * @param pVCpu The cross context virtual CPU structure.
1304 * @param uMsr The MSR.
1305 * @param uGuestMsrValue Value of the guest MSR.
1306 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1307 * necessary.
1308 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1309 * its value was updated. Optional, can be NULL.
1310 */
1311static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1312 bool *pfAddedAndUpdated)
1313{
1314 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1315 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1316 uint32_t i;
1317 for (i = 0; i < cMsrs; i++)
1318 {
1319 if (pGuestMsr->u32Msr == uMsr)
1320 break;
1321 pGuestMsr++;
1322 }
1323
1324 bool fAdded = false;
1325 if (i == cMsrs)
1326 {
1327 ++cMsrs;
1328 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1329 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1330
1331 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1332 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1333 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1334
1335 fAdded = true;
1336 }
1337
1338 /* Update the MSR values in the auto-load/store MSR area. */
1339 pGuestMsr->u32Msr = uMsr;
1340 pGuestMsr->u64Value = uGuestMsrValue;
1341
1342 /* Create/update the MSR slot in the host MSR area. */
1343 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1344 pHostMsr += i;
1345 pHostMsr->u32Msr = uMsr;
1346
1347 /*
1348 * Update the host MSR only when requested by the caller AND when we're
1349 * adding it to the auto-load/store area. Otherwise, it would have been
1350 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1351 */
1352 bool fUpdatedMsrValue = false;
1353 if ( fAdded
1354 && fUpdateHostMsr)
1355 {
1356 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1357 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1358 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1359 fUpdatedMsrValue = true;
1360 }
1361
1362 if (pfAddedAndUpdated)
1363 *pfAddedAndUpdated = fUpdatedMsrValue;
1364 return VINF_SUCCESS;
1365}
1366
1367
1368/**
1369 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1370 * auto-load/store MSR area in the VMCS.
1371 *
1372 * @returns VBox status code.
1373 * @param pVCpu The cross context virtual CPU structure.
1374 * @param uMsr The MSR.
1375 */
1376static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1377{
1378 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1379 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1380 for (uint32_t i = 0; i < cMsrs; i++)
1381 {
1382 /* Find the MSR. */
1383 if (pGuestMsr->u32Msr == uMsr)
1384 {
1385 /* If it's the last MSR, simply reduce the count. */
1386 if (i == cMsrs - 1)
1387 {
1388 --cMsrs;
1389 break;
1390 }
1391
1392 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1393 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1394 pLastGuestMsr += cMsrs - 1;
1395 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1396 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1397
1398 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1399 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1400 pLastHostMsr += cMsrs - 1;
1401 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1402 pHostMsr->u64Value = pLastHostMsr->u64Value;
1403 --cMsrs;
1404 break;
1405 }
1406 pGuestMsr++;
1407 }
1408
1409 /* Update the VMCS if the count changed (meaning the MSR was found). */
1410 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1411 {
1412 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1413 AssertRCReturn(rc, rc);
1414
1415 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1416 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1417 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1418
1419 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1420 return VINF_SUCCESS;
1421 }
1422
1423 return VERR_NOT_FOUND;
1424}
1425
1426
1427/**
1428 * Checks if the specified guest MSR is part of the auto-load/store area in
1429 * the VMCS.
1430 *
1431 * @returns true if found, false otherwise.
1432 * @param pVCpu The cross context virtual CPU structure.
1433 * @param uMsr The MSR to find.
1434 */
1435static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1436{
1437 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1438 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1439
1440 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1441 {
1442 if (pGuestMsr->u32Msr == uMsr)
1443 return true;
1444 }
1445 return false;
1446}
1447
1448
1449/**
1450 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1451 *
1452 * @param pVCpu The cross context virtual CPU structure.
1453 *
1454 * @remarks No-long-jump zone!!!
1455 */
1456static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1457{
1458 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1459 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1460 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1461 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1462
1463 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1464 {
1465 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1466
1467 /*
1468 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1469 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1470 */
1471 if (pHostMsr->u32Msr == MSR_K6_EFER)
1472 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1473 else
1474 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1475 }
1476
1477 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1478}
1479
1480
1481/**
1482 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1483 * perform lazy restoration of the host MSRs while leaving VT-x.
1484 *
1485 * @param pVCpu The cross context virtual CPU structure.
1486 *
1487 * @remarks No-long-jump zone!!!
1488 */
1489static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1490{
1491 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1492
1493 /*
1494 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1495 */
1496 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1497 {
1498 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1499#if HC_ARCH_BITS == 64
1500 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1501 {
1502 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1503 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1504 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1505 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1506 }
1507#endif
1508 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1509 }
1510}
1511
1512
1513/**
1514 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1515 * lazily while leaving VT-x.
1516 *
1517 * @returns true if it does, false otherwise.
1518 * @param pVCpu The cross context virtual CPU structure.
1519 * @param uMsr The MSR to check.
1520 */
1521static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1522{
1523 NOREF(pVCpu);
1524#if HC_ARCH_BITS == 64
1525 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1526 {
1527 switch (uMsr)
1528 {
1529 case MSR_K8_LSTAR:
1530 case MSR_K6_STAR:
1531 case MSR_K8_SF_MASK:
1532 case MSR_K8_KERNEL_GS_BASE:
1533 return true;
1534 }
1535 }
1536#else
1537 RT_NOREF(pVCpu, uMsr);
1538#endif
1539 return false;
1540}
1541
1542
1543/**
1544 * Saves a set of guest MSRs back into the guest-CPU context.
1545 *
1546 * @param pVCpu The cross context virtual CPU structure.
1547 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1548 * out-of-sync. Make sure to update the required fields
1549 * before using them.
1550 *
1551 * @remarks No-long-jump zone!!!
1552 */
1553static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1554{
1555 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1556 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1557
1558 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1559 {
1560 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1561#if HC_ARCH_BITS == 64
1562 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1563 {
1564 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1565 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1566 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1567 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1568 }
1569#else
1570 NOREF(pMixedCtx);
1571#endif
1572 }
1573}
1574
1575
1576/**
1577 * Loads a set of guests MSRs to allow read/passthru to the guest.
1578 *
1579 * The name of this function is slightly confusing. This function does NOT
1580 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1581 * common prefix for functions dealing with "lazy restoration" of the shared
1582 * MSRs.
1583 *
1584 * @param pVCpu The cross context virtual CPU structure.
1585 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1586 * out-of-sync. Make sure to update the required fields
1587 * before using them.
1588 *
1589 * @remarks No-long-jump zone!!!
1590 */
1591static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1592{
1593 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1594 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1595
1596 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1597#if HC_ARCH_BITS == 64
1598 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1599 {
1600 /*
1601 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1602 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1603 * we can skip a few MSR writes.
1604 *
1605 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1606 * guest MSR values in the guest-CPU context might be different to what's currently
1607 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1608 * CPU, see @bugref{8728}.
1609 */
1610 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1611 && pMixedCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1612 && pMixedCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1613 && pMixedCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1614 && pMixedCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1615 {
1616#ifdef VBOX_STRICT
1617 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pMixedCtx->msrKERNELGSBASE);
1618 Assert(ASMRdMsr(MSR_K8_LSTAR) == pMixedCtx->msrLSTAR);
1619 Assert(ASMRdMsr(MSR_K6_STAR) == pMixedCtx->msrSTAR);
1620 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pMixedCtx->msrSFMASK);
1621#endif
1622 }
1623 else
1624 {
1625 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1626 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1627 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1628 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1629 }
1630 }
1631#else
1632 RT_NOREF(pMixedCtx);
1633#endif
1634 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1635}
1636
1637
1638/**
1639 * Performs lazy restoration of the set of host MSRs if they were previously
1640 * loaded with guest MSR values.
1641 *
1642 * @param pVCpu The cross context virtual CPU structure.
1643 *
1644 * @remarks No-long-jump zone!!!
1645 * @remarks The guest MSRs should have been saved back into the guest-CPU
1646 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1647 */
1648static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1649{
1650 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1651 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1652
1653 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1654 {
1655 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1656#if HC_ARCH_BITS == 64
1657 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1658 {
1659 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1660 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1661 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1662 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1663 }
1664#endif
1665 }
1666 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1667}
1668
1669
1670/**
1671 * Verifies that our cached values of the VMCS controls are all
1672 * consistent with what's actually present in the VMCS.
1673 *
1674 * @returns VBox status code.
1675 * @param pVCpu The cross context virtual CPU structure.
1676 */
1677static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1678{
1679 uint32_t u32Val;
1680 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1681 AssertRCReturn(rc, rc);
1682 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1683 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1684
1685 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1686 AssertRCReturn(rc, rc);
1687 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1688 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1689
1690 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1691 AssertRCReturn(rc, rc);
1692 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1693 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1694
1695 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1696 AssertRCReturn(rc, rc);
1697 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1698 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1699
1700 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1701 {
1702 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1703 AssertRCReturn(rc, rc);
1704 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1705 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1706 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1707 }
1708
1709 return VINF_SUCCESS;
1710}
1711
1712
1713#ifdef VBOX_STRICT
1714/**
1715 * Verifies that our cached host EFER value has not changed
1716 * since we cached it.
1717 *
1718 * @param pVCpu The cross context virtual CPU structure.
1719 */
1720static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1721{
1722 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1723
1724 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1725 {
1726 uint64_t u64Val;
1727 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1728 AssertRC(rc);
1729
1730 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1731 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1732 }
1733}
1734
1735
1736/**
1737 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1738 * VMCS are correct.
1739 *
1740 * @param pVCpu The cross context virtual CPU structure.
1741 */
1742static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1743{
1744 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1745
1746 /* Verify MSR counts in the VMCS are what we think it should be. */
1747 uint32_t cMsrs;
1748 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1749 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1750
1751 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1752 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1753
1754 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1755 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1756
1757 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1758 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1759 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1760 {
1761 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1762 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1763 pGuestMsr->u32Msr, cMsrs));
1764
1765 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1766 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1767 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1768
1769 /* Verify that the permissions are as expected in the MSR bitmap. */
1770 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1771 {
1772 VMXMSREXITREAD enmRead;
1773 VMXMSREXITWRITE enmWrite;
1774 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1775 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1776 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1777 {
1778 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1779 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1780 }
1781 else
1782 {
1783 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1784 pGuestMsr->u32Msr, cMsrs));
1785 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1786 pGuestMsr->u32Msr, cMsrs));
1787 }
1788 }
1789 }
1790}
1791#endif /* VBOX_STRICT */
1792
1793
1794/**
1795 * Flushes the TLB using EPT.
1796 *
1797 * @returns VBox status code.
1798 * @param pVCpu The cross context virtual CPU structure of the calling
1799 * EMT. Can be NULL depending on @a enmFlush.
1800 * @param enmFlush Type of flush.
1801 *
1802 * @remarks Caller is responsible for making sure this function is called only
1803 * when NestedPaging is supported and providing @a enmFlush that is
1804 * supported by the CPU.
1805 * @remarks Can be called with interrupts disabled.
1806 */
1807static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1808{
1809 uint64_t au64Descriptor[2];
1810 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1811 au64Descriptor[0] = 0;
1812 else
1813 {
1814 Assert(pVCpu);
1815 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1816 }
1817 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1818
1819 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1820 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1821 rc));
1822 if ( RT_SUCCESS(rc)
1823 && pVCpu)
1824 {
1825 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1826 }
1827}
1828
1829
1830/**
1831 * Flushes the TLB using VPID.
1832 *
1833 * @returns VBox status code.
1834 * @param pVM The cross context VM structure.
1835 * @param pVCpu The cross context virtual CPU structure of the calling
1836 * EMT. Can be NULL depending on @a enmFlush.
1837 * @param enmFlush Type of flush.
1838 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1839 * on @a enmFlush).
1840 *
1841 * @remarks Can be called with interrupts disabled.
1842 */
1843static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1844{
1845 NOREF(pVM);
1846 AssertPtr(pVM);
1847 Assert(pVM->hm.s.vmx.fVpid);
1848
1849 uint64_t au64Descriptor[2];
1850 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1851 {
1852 au64Descriptor[0] = 0;
1853 au64Descriptor[1] = 0;
1854 }
1855 else
1856 {
1857 AssertPtr(pVCpu);
1858 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1859 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1860 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1861 au64Descriptor[1] = GCPtr;
1862 }
1863
1864 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1865 AssertMsg(rc == VINF_SUCCESS,
1866 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1867 if ( RT_SUCCESS(rc)
1868 && pVCpu)
1869 {
1870 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1871 }
1872}
1873
1874
1875/**
1876 * Invalidates a guest page by guest virtual address. Only relevant for
1877 * EPT/VPID, otherwise there is nothing really to invalidate.
1878 *
1879 * @returns VBox status code.
1880 * @param pVM The cross context VM structure.
1881 * @param pVCpu The cross context virtual CPU structure.
1882 * @param GCVirt Guest virtual address of the page to invalidate.
1883 */
1884VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1885{
1886 AssertPtr(pVM);
1887 AssertPtr(pVCpu);
1888 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1889
1890 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1891 if (!fFlushPending)
1892 {
1893 /*
1894 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1895 * See @bugref{6043} and @bugref{6177}.
1896 *
1897 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1898 * function maybe called in a loop with individual addresses.
1899 */
1900 if (pVM->hm.s.vmx.fVpid)
1901 {
1902 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1903
1904#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1905 /*
1906 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1907 * where executing INVVPID outside 64-bit mode does not flush translations of
1908 * 64-bit linear addresses, see @bugref{6208#c72}.
1909 */
1910 if (RT_HI_U32(GCVirt))
1911 fVpidFlush = false;
1912#endif
1913
1914 if (fVpidFlush)
1915 {
1916 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1917 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1918 }
1919 else
1920 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1921 }
1922 else if (pVM->hm.s.fNestedPaging)
1923 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1924 }
1925
1926 return VINF_SUCCESS;
1927}
1928
1929
1930/**
1931 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1932 * otherwise there is nothing really to invalidate.
1933 *
1934 * @returns VBox status code.
1935 * @param pVM The cross context VM structure.
1936 * @param pVCpu The cross context virtual CPU structure.
1937 * @param GCPhys Guest physical address of the page to invalidate.
1938 */
1939VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1940{
1941 NOREF(pVM); NOREF(GCPhys);
1942 LogFlowFunc(("%RGp\n", GCPhys));
1943
1944 /*
1945 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1946 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1947 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1948 */
1949 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1950 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1951 return VINF_SUCCESS;
1952}
1953
1954
1955/**
1956 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1957 * case where neither EPT nor VPID is supported by the CPU.
1958 *
1959 * @param pVM The cross context VM structure.
1960 * @param pVCpu The cross context virtual CPU structure.
1961 * @param pCpu Pointer to the global HM struct.
1962 *
1963 * @remarks Called with interrupts disabled.
1964 */
1965static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1966{
1967 AssertPtr(pVCpu);
1968 AssertPtr(pCpu);
1969 NOREF(pVM);
1970
1971 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1972
1973 Assert(pCpu->idCpu != NIL_RTCPUID);
1974 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1975 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1976 pVCpu->hm.s.fForceTLBFlush = false;
1977 return;
1978}
1979
1980
1981/**
1982 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1983 *
1984 * @param pVM The cross context VM structure.
1985 * @param pVCpu The cross context virtual CPU structure.
1986 * @param pCpu Pointer to the global HM CPU struct.
1987 * @remarks All references to "ASID" in this function pertains to "VPID" in
1988 * Intel's nomenclature. The reason is, to avoid confusion in compare
1989 * statements since the host-CPU copies are named "ASID".
1990 *
1991 * @remarks Called with interrupts disabled.
1992 */
1993static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1994{
1995#ifdef VBOX_WITH_STATISTICS
1996 bool fTlbFlushed = false;
1997# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1998# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1999 if (!fTlbFlushed) \
2000 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2001 } while (0)
2002#else
2003# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2004# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2005#endif
2006
2007 AssertPtr(pVM);
2008 AssertPtr(pCpu);
2009 AssertPtr(pVCpu);
2010 Assert(pCpu->idCpu != NIL_RTCPUID);
2011
2012 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2013 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2014 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2015
2016 /*
2017 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2018 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2019 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2020 */
2021 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2022 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2023 {
2024 ++pCpu->uCurrentAsid;
2025 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2026 {
2027 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2028 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2029 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2030 }
2031
2032 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2033 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2034 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2035
2036 /*
2037 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2038 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2039 */
2040 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2041 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2042 HMVMX_SET_TAGGED_TLB_FLUSHED();
2043 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
2044 }
2045
2046 /* Check for explicit TLB flushes. */
2047 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2048 {
2049 /*
2050 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
2051 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2052 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2053 * but not guest-physical mappings.
2054 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2055 */
2056 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2057 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2058 HMVMX_SET_TAGGED_TLB_FLUSHED();
2059 }
2060
2061 pVCpu->hm.s.fForceTLBFlush = false;
2062 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2063
2064 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2065 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2066 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2067 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2068 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2069 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2070 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2071 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2072 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2073
2074 /* Update VMCS with the VPID. */
2075 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2076 AssertRC(rc);
2077
2078#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2079}
2080
2081
2082/**
2083 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2084 *
2085 * @returns VBox status code.
2086 * @param pVM The cross context VM structure.
2087 * @param pVCpu The cross context virtual CPU structure.
2088 * @param pCpu Pointer to the global HM CPU struct.
2089 *
2090 * @remarks Called with interrupts disabled.
2091 */
2092static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2093{
2094 AssertPtr(pVM);
2095 AssertPtr(pVCpu);
2096 AssertPtr(pCpu);
2097 Assert(pCpu->idCpu != NIL_RTCPUID);
2098 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2099 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2100
2101 /*
2102 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2103 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2104 */
2105 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2106 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2107 {
2108 pVCpu->hm.s.fForceTLBFlush = true;
2109 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2110 }
2111
2112 /* Check for explicit TLB flushes. */
2113 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2114 {
2115 pVCpu->hm.s.fForceTLBFlush = true;
2116 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2117 }
2118
2119 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2120 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2121
2122 if (pVCpu->hm.s.fForceTLBFlush)
2123 {
2124 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2125 pVCpu->hm.s.fForceTLBFlush = false;
2126 }
2127}
2128
2129
2130/**
2131 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2132 *
2133 * @returns VBox status code.
2134 * @param pVM The cross context VM structure.
2135 * @param pVCpu The cross context virtual CPU structure.
2136 * @param pCpu Pointer to the global HM CPU struct.
2137 *
2138 * @remarks Called with interrupts disabled.
2139 */
2140static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2141{
2142 AssertPtr(pVM);
2143 AssertPtr(pVCpu);
2144 AssertPtr(pCpu);
2145 Assert(pCpu->idCpu != NIL_RTCPUID);
2146 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2147 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2148
2149 /*
2150 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2151 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2152 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2153 */
2154 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2155 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2156 {
2157 pVCpu->hm.s.fForceTLBFlush = true;
2158 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2159 }
2160
2161 /* Check for explicit TLB flushes. */
2162 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2163 {
2164 /*
2165 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2166 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2167 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2168 */
2169 pVCpu->hm.s.fForceTLBFlush = true;
2170 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2171 }
2172
2173 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2174 if (pVCpu->hm.s.fForceTLBFlush)
2175 {
2176 ++pCpu->uCurrentAsid;
2177 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2178 {
2179 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2180 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2181 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2182 }
2183
2184 pVCpu->hm.s.fForceTLBFlush = false;
2185 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2186 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2187 if (pCpu->fFlushAsidBeforeUse)
2188 {
2189 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2190 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2191 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2192 {
2193 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2194 pCpu->fFlushAsidBeforeUse = false;
2195 }
2196 else
2197 {
2198 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2199 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2200 }
2201 }
2202 }
2203
2204 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2205 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2206 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2207 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2208 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2209 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2210 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2211
2212 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2213 AssertRC(rc);
2214}
2215
2216
2217/**
2218 * Flushes the guest TLB entry based on CPU capabilities.
2219 *
2220 * @param pVCpu The cross context virtual CPU structure.
2221 * @param pCpu Pointer to the global HM CPU struct.
2222 */
2223DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2224{
2225#ifdef HMVMX_ALWAYS_FLUSH_TLB
2226 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2227#endif
2228 PVM pVM = pVCpu->CTX_SUFF(pVM);
2229 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2230 {
2231 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2232 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2233 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2234 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2235 default:
2236 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2237 break;
2238 }
2239
2240 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2241}
2242
2243
2244/**
2245 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2246 * TLB entries from the host TLB before VM-entry.
2247 *
2248 * @returns VBox status code.
2249 * @param pVM The cross context VM structure.
2250 */
2251static int hmR0VmxSetupTaggedTlb(PVM pVM)
2252{
2253 /*
2254 * Determine optimal flush type for Nested Paging.
2255 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2256 * guest execution (see hmR3InitFinalizeR0()).
2257 */
2258 if (pVM->hm.s.fNestedPaging)
2259 {
2260 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2261 {
2262 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2263 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2264 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2265 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2266 else
2267 {
2268 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2269 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2270 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2271 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2272 }
2273
2274 /* Make sure the write-back cacheable memory type for EPT is supported. */
2275 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2276 {
2277 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2278 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2279 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2280 }
2281
2282 /* EPT requires a page-walk length of 4. */
2283 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2284 {
2285 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2286 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2287 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2288 }
2289 }
2290 else
2291 {
2292 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2293 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2294 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2295 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2296 }
2297 }
2298
2299 /*
2300 * Determine optimal flush type for VPID.
2301 */
2302 if (pVM->hm.s.vmx.fVpid)
2303 {
2304 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2305 {
2306 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2307 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2308 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2309 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2310 else
2311 {
2312 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2313 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2314 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2315 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2316 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2317 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2318 pVM->hm.s.vmx.fVpid = false;
2319 }
2320 }
2321 else
2322 {
2323 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2324 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2325 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2326 pVM->hm.s.vmx.fVpid = false;
2327 }
2328 }
2329
2330 /*
2331 * Setup the handler for flushing tagged-TLBs.
2332 */
2333 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2334 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2335 else if (pVM->hm.s.fNestedPaging)
2336 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2337 else if (pVM->hm.s.vmx.fVpid)
2338 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2339 else
2340 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2341 return VINF_SUCCESS;
2342}
2343
2344
2345/**
2346 * Sets up pin-based VM-execution controls in the VMCS.
2347 *
2348 * @returns VBox status code.
2349 * @param pVM The cross context VM structure.
2350 * @param pVCpu The cross context virtual CPU structure.
2351 */
2352static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2353{
2354 AssertPtr(pVM);
2355 AssertPtr(pVCpu);
2356
2357 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2358 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2359
2360 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2361 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2362
2363 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2364 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2365
2366 /* Enable the VMX preemption timer. */
2367 if (pVM->hm.s.vmx.fUsePreemptTimer)
2368 {
2369 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2370 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2371 }
2372
2373#if 0
2374 /* Enable posted-interrupt processing. */
2375 if (pVM->hm.s.fPostedIntrs)
2376 {
2377 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2378 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2379 val |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2380 }
2381#endif
2382
2383 if ((val & zap) != val)
2384 {
2385 LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2386 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2387 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2388 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2389 }
2390
2391 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2392 AssertRCReturn(rc, rc);
2393
2394 pVCpu->hm.s.vmx.u32PinCtls = val;
2395 return rc;
2396}
2397
2398
2399/**
2400 * Sets up processor-based VM-execution controls in the VMCS.
2401 *
2402 * @returns VBox status code.
2403 * @param pVM The cross context VM structure.
2404 * @param pVCpu The cross context virtual CPU structure.
2405 */
2406static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2407{
2408 AssertPtr(pVM);
2409 AssertPtr(pVCpu);
2410
2411 int rc = VERR_INTERNAL_ERROR_5;
2412 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2413 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2414
2415 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2416 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2417 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2418 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2419 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2420 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2421 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2422
2423 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2424 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2425 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2426 {
2427 LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2428 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2429 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2430 }
2431
2432 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2433 if (!pVM->hm.s.fNestedPaging)
2434 {
2435 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2436 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2437 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2438 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2439 }
2440
2441 /* Use TPR shadowing if supported by the CPU. */
2442 if ( PDMHasApic(pVM)
2443 && pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2444 {
2445 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2446 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2447 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2448 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2449 AssertRCReturn(rc, rc);
2450
2451 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2452 /* CR8 writes cause a VM-exit based on TPR threshold. */
2453 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2454 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2455 }
2456 else
2457 {
2458 /*
2459 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2460 * Set this control only for 64-bit guests.
2461 */
2462 if (pVM->hm.s.fAllow64BitGuests)
2463 {
2464 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2465 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2466 }
2467 }
2468
2469 /* Use MSR-bitmaps if supported by the CPU. */
2470 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2471 {
2472 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2473
2474 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2475 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2476 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2477 AssertRCReturn(rc, rc);
2478
2479 /*
2480 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2481 * automatically using dedicated fields in the VMCS.
2482 */
2483 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2484 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2485 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2486 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2487 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2488
2489#if HC_ARCH_BITS == 64
2490 /*
2491 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2492 */
2493 if (pVM->hm.s.fAllow64BitGuests)
2494 {
2495 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2496 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2497 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2498 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2499 }
2500#endif
2501 /*
2502 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2503 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2504 */
2505 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2506 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2507
2508 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2509 }
2510
2511 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2512 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2513 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2514
2515 if ((val & zap) != val)
2516 {
2517 LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2518 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2519 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2520 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2521 }
2522
2523 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2524 AssertRCReturn(rc, rc);
2525
2526 pVCpu->hm.s.vmx.u32ProcCtls = val;
2527
2528 /*
2529 * Secondary processor-based VM-execution controls.
2530 */
2531 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2532 {
2533 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2534 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2535
2536 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2537 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2538
2539 if (pVM->hm.s.fNestedPaging)
2540 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2541
2542 /*
2543 * Enable the INVPCID instruction if supported by the hardware and we expose
2544 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2545 */
2546 if ( (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2547 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2548 {
2549 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2550 }
2551
2552 if (pVM->hm.s.vmx.fVpid)
2553 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2554
2555 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2556 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2557
2558#if 0
2559 if (pVM->hm.s.fVirtApicRegs)
2560 {
2561 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2562 val |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2563
2564 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2565 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2566 }
2567#endif
2568
2569 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2570 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2571 * done dynamically. */
2572 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2573 {
2574 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2575 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2576 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2577 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2578 AssertRCReturn(rc, rc);
2579 }
2580
2581 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2582 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2583
2584 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2585 && pVM->hm.s.vmx.cPleGapTicks
2586 && pVM->hm.s.vmx.cPleWindowTicks)
2587 {
2588 val |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2589
2590 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2591 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2592 AssertRCReturn(rc, rc);
2593 }
2594
2595 if ((val & zap) != val)
2596 {
2597 LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2598 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2599 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2600 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2601 }
2602
2603 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2604 AssertRCReturn(rc, rc);
2605
2606 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2607 }
2608 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2609 {
2610 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2611 "available\n"));
2612 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2613 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2614 }
2615
2616 return VINF_SUCCESS;
2617}
2618
2619
2620/**
2621 * Sets up miscellaneous (everything other than Pin & Processor-based
2622 * VM-execution) control fields in the VMCS.
2623 *
2624 * @returns VBox status code.
2625 * @param pVM The cross context VM structure.
2626 * @param pVCpu The cross context virtual CPU structure.
2627 */
2628static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2629{
2630 NOREF(pVM);
2631 AssertPtr(pVM);
2632 AssertPtr(pVCpu);
2633
2634 int rc = VERR_GENERAL_FAILURE;
2635
2636 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2637#if 0
2638 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2639 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2640 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2641
2642 /*
2643 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2644 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2645 * We thus use the exception bitmap to control it rather than use both.
2646 */
2647 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2648 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2649
2650 /** @todo Explore possibility of using IO-bitmaps. */
2651 /* All IO & IOIO instructions cause VM-exits. */
2652 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2653 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2654
2655 /* Initialize the MSR-bitmap area. */
2656 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2657 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2658 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2659 AssertRCReturn(rc, rc);
2660#endif
2661
2662 /* Setup MSR auto-load/store area. */
2663 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2664 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2665 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2666 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2667 AssertRCReturn(rc, rc);
2668
2669 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2670 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2671 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2672 AssertRCReturn(rc, rc);
2673
2674 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2675 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2676 AssertRCReturn(rc, rc);
2677
2678 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2679#if 0
2680 /* Setup debug controls */
2681 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2682 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2683 AssertRCReturn(rc, rc);
2684#endif
2685
2686 return rc;
2687}
2688
2689
2690/**
2691 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2692 *
2693 * We shall setup those exception intercepts that don't change during the
2694 * lifetime of the VM here. The rest are done dynamically while loading the
2695 * guest state.
2696 *
2697 * @returns VBox status code.
2698 * @param pVM The cross context VM structure.
2699 * @param pVCpu The cross context virtual CPU structure.
2700 */
2701static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2702{
2703 AssertPtr(pVM);
2704 AssertPtr(pVCpu);
2705
2706 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2707
2708 uint32_t u32XcptBitmap = 0;
2709
2710 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2711 u32XcptBitmap |= RT_BIT_32(X86_XCPT_AC);
2712
2713 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2714 and writes, and because recursive #DBs can cause the CPU hang, we must always
2715 intercept #DB. */
2716 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2717
2718 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2719 if (!pVM->hm.s.fNestedPaging)
2720 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2721
2722 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2723 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2724 AssertRCReturn(rc, rc);
2725 return rc;
2726}
2727
2728
2729/**
2730 * Sets up the initial guest-state mask. The guest-state mask is consulted
2731 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2732 * for the nested virtualization case (as it would cause a VM-exit).
2733 *
2734 * @param pVCpu The cross context virtual CPU structure.
2735 */
2736static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2737{
2738 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2739 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2740 return VINF_SUCCESS;
2741}
2742
2743
2744/**
2745 * Does per-VM VT-x initialization.
2746 *
2747 * @returns VBox status code.
2748 * @param pVM The cross context VM structure.
2749 */
2750VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2751{
2752 LogFlowFunc(("pVM=%p\n", pVM));
2753
2754 int rc = hmR0VmxStructsAlloc(pVM);
2755 if (RT_FAILURE(rc))
2756 {
2757 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2758 return rc;
2759 }
2760
2761 return VINF_SUCCESS;
2762}
2763
2764
2765/**
2766 * Does per-VM VT-x termination.
2767 *
2768 * @returns VBox status code.
2769 * @param pVM The cross context VM structure.
2770 */
2771VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2772{
2773 LogFlowFunc(("pVM=%p\n", pVM));
2774
2775#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2776 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2777 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2778#endif
2779 hmR0VmxStructsFree(pVM);
2780 return VINF_SUCCESS;
2781}
2782
2783
2784/**
2785 * Sets up the VM for execution under VT-x.
2786 * This function is only called once per-VM during initialization.
2787 *
2788 * @returns VBox status code.
2789 * @param pVM The cross context VM structure.
2790 */
2791VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2792{
2793 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2794 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2795
2796 LogFlowFunc(("pVM=%p\n", pVM));
2797
2798 /*
2799 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2800 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2801 */
2802 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2803 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2804 || !pVM->hm.s.vmx.pRealModeTSS))
2805 {
2806 LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2807 return VERR_INTERNAL_ERROR;
2808 }
2809
2810 /* Initialize these always, see hmR3InitFinalizeR0().*/
2811 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2812 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2813
2814 /* Setup the tagged-TLB flush handlers. */
2815 int rc = hmR0VmxSetupTaggedTlb(pVM);
2816 if (RT_FAILURE(rc))
2817 {
2818 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2819 return rc;
2820 }
2821
2822 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2823 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2824#if HC_ARCH_BITS == 64
2825 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2826 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2827 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2828 {
2829 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2830 }
2831#endif
2832
2833 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2834 RTCCUINTREG uHostCR4 = ASMGetCR4();
2835 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2836 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2837
2838 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2839 {
2840 PVMCPU pVCpu = &pVM->aCpus[i];
2841 AssertPtr(pVCpu);
2842 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2843
2844 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2845 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2846
2847 /* Set revision dword at the beginning of the VMCS structure. */
2848 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2849
2850 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2851 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2852 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2853 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2854
2855 /* Load this VMCS as the current VMCS. */
2856 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2857 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2858 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2859
2860 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2861 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2862 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2863
2864 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2865 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2866 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2867
2868 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2869 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2870 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2871
2872 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2873 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2874 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2875
2876 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2877 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2878 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2879
2880#if HC_ARCH_BITS == 32
2881 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2882 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2883 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2884#endif
2885
2886 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2887 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2888 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2889 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2890
2891 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2892
2893 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2894 }
2895
2896 return VINF_SUCCESS;
2897}
2898
2899
2900/**
2901 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2902 * the VMCS.
2903 *
2904 * @returns VBox status code.
2905 * @param pVM The cross context VM structure.
2906 * @param pVCpu The cross context virtual CPU structure.
2907 */
2908DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2909{
2910 NOREF(pVM); NOREF(pVCpu);
2911
2912 RTCCUINTREG uReg = ASMGetCR0();
2913 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2914 AssertRCReturn(rc, rc);
2915
2916 uReg = ASMGetCR3();
2917 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2918 AssertRCReturn(rc, rc);
2919
2920 uReg = ASMGetCR4();
2921 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2922 AssertRCReturn(rc, rc);
2923 return rc;
2924}
2925
2926
2927#if HC_ARCH_BITS == 64
2928/**
2929 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2930 * requirements. See hmR0VmxSaveHostSegmentRegs().
2931 */
2932# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2933 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2934 { \
2935 bool fValidSelector = true; \
2936 if ((selValue) & X86_SEL_LDT) \
2937 { \
2938 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2939 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2940 } \
2941 if (fValidSelector) \
2942 { \
2943 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2944 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2945 } \
2946 (selValue) = 0; \
2947 }
2948#endif
2949
2950
2951/**
2952 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2953 * the host-state area in the VMCS.
2954 *
2955 * @returns VBox status code.
2956 * @param pVM The cross context VM structure.
2957 * @param pVCpu The cross context virtual CPU structure.
2958 */
2959DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2960{
2961 int rc = VERR_INTERNAL_ERROR_5;
2962
2963#if HC_ARCH_BITS == 64
2964 /*
2965 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2966 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2967 *
2968 * This apparently can happen (most likely the FPU changes), deal with it rather than asserting.
2969 * Was observed booting Solaris10u10 32-bit guest.
2970 */
2971 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2972 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2973 {
2974 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2975 pVCpu->idCpu));
2976 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2977 }
2978 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2979#else
2980 RT_NOREF(pVCpu);
2981#endif
2982
2983 /*
2984 * Host DS, ES, FS and GS segment registers.
2985 */
2986#if HC_ARCH_BITS == 64
2987 RTSEL uSelDS = ASMGetDS();
2988 RTSEL uSelES = ASMGetES();
2989 RTSEL uSelFS = ASMGetFS();
2990 RTSEL uSelGS = ASMGetGS();
2991#else
2992 RTSEL uSelDS = 0;
2993 RTSEL uSelES = 0;
2994 RTSEL uSelFS = 0;
2995 RTSEL uSelGS = 0;
2996#endif
2997
2998 /*
2999 * Host CS and SS segment registers.
3000 */
3001 RTSEL uSelCS = ASMGetCS();
3002 RTSEL uSelSS = ASMGetSS();
3003
3004 /*
3005 * Host TR segment register.
3006 */
3007 RTSEL uSelTR = ASMGetTR();
3008
3009#if HC_ARCH_BITS == 64
3010 /*
3011 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
3012 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
3013 */
3014 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
3015 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
3016 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
3017 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
3018# undef VMXLOCAL_ADJUST_HOST_SEG
3019#endif
3020
3021 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
3022 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
3023 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
3024 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
3025 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
3026 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
3027 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
3028 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
3029 Assert(uSelCS);
3030 Assert(uSelTR);
3031
3032 /* Assertion is right but we would not have updated u32ExitCtls yet. */
3033#if 0
3034 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
3035 Assert(uSelSS != 0);
3036#endif
3037
3038 /* Write these host selector fields into the host-state area in the VMCS. */
3039 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3040 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3041#if HC_ARCH_BITS == 64
3042 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3043 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3044 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3045 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3046#else
3047 NOREF(uSelDS);
3048 NOREF(uSelES);
3049 NOREF(uSelFS);
3050 NOREF(uSelGS);
3051#endif
3052 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3053 AssertRCReturn(rc, rc);
3054
3055 /*
3056 * Host GDTR and IDTR.
3057 */
3058 RTGDTR Gdtr;
3059 RTIDTR Idtr;
3060 RT_ZERO(Gdtr);
3061 RT_ZERO(Idtr);
3062 ASMGetGDTR(&Gdtr);
3063 ASMGetIDTR(&Idtr);
3064 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3065 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3066 AssertRCReturn(rc, rc);
3067
3068#if HC_ARCH_BITS == 64
3069 /*
3070 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3071 * maximum limit (0xffff) on every VM-exit.
3072 */
3073 if (Gdtr.cbGdt != 0xffff)
3074 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3075
3076 /*
3077 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3078 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3079 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3080 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3081 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3082 * hosts where we are pretty sure it won't cause trouble.
3083 */
3084# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3085 if (Idtr.cbIdt < 0x0fff)
3086# else
3087 if (Idtr.cbIdt != 0xffff)
3088# endif
3089 {
3090 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3091 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3092 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3093 }
3094#endif
3095
3096 /*
3097 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3098 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3099 */
3100 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3101 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3102 VERR_VMX_INVALID_HOST_STATE);
3103
3104 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3105#if HC_ARCH_BITS == 64
3106 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3107
3108 /*
3109 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3110 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3111 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3112 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3113 *
3114 * [1] See Intel spec. 3.5 "System Descriptor Types".
3115 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3116 */
3117 Assert(pDesc->System.u4Type == 11);
3118 if ( pDesc->System.u16LimitLow != 0x67
3119 || pDesc->System.u4LimitHigh)
3120 {
3121 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3122 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3123 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3124 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3125 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3126 }
3127
3128 /*
3129 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3130 */
3131 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3132 {
3133 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3134 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3135 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3136 {
3137 /* The GDT is read-only but the writable GDT is available. */
3138 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3139 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3140 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3141 AssertRCReturn(rc, rc);
3142 }
3143 }
3144#else
3145 NOREF(pVM);
3146 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3147#endif
3148 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3149 AssertRCReturn(rc, rc);
3150
3151 /*
3152 * Host FS base and GS base.
3153 */
3154#if HC_ARCH_BITS == 64
3155 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3156 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3157 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3158 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3159 AssertRCReturn(rc, rc);
3160
3161 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3162 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3163 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3164 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3165 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3166#endif
3167 return rc;
3168}
3169
3170
3171/**
3172 * Saves certain host MSRs in the VM-exit MSR-load area and some in the
3173 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3174 * the host after every successful VM-exit.
3175 *
3176 * @returns VBox status code.
3177 * @param pVM The cross context VM structure.
3178 * @param pVCpu The cross context virtual CPU structure.
3179 *
3180 * @remarks No-long-jump zone!!!
3181 */
3182DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3183{
3184 NOREF(pVM);
3185
3186 AssertPtr(pVCpu);
3187 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3188
3189 /*
3190 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3191 * rather than swapping them on every VM-entry.
3192 */
3193 hmR0VmxLazySaveHostMsrs(pVCpu);
3194
3195 /*
3196 * Host Sysenter MSRs.
3197 */
3198 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3199#if HC_ARCH_BITS == 32
3200 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3201 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3202#else
3203 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3204 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3205#endif
3206 AssertRCReturn(rc, rc);
3207
3208 /*
3209 * Host EFER MSR.
3210 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3211 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3212 */
3213 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3214 {
3215 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3216 AssertRCReturn(rc, rc);
3217 }
3218
3219 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3220 * hmR0VmxLoadGuestExitCtls() !! */
3221
3222 return rc;
3223}
3224
3225
3226/**
3227 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3228 *
3229 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3230 * these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3231 * hmR0VMxLoadGuestEntryCtls().
3232 *
3233 * @returns true if we need to load guest EFER, false otherwise.
3234 * @param pVCpu The cross context virtual CPU structure.
3235 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3236 * out-of-sync. Make sure to update the required fields
3237 * before using them.
3238 *
3239 * @remarks Requires EFER, CR4.
3240 * @remarks No-long-jump zone!!!
3241 */
3242static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3243{
3244#ifdef HMVMX_ALWAYS_SWAP_EFER
3245 return true;
3246#endif
3247
3248#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3249 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3250 if (CPUMIsGuestInLongMode(pVCpu))
3251 return false;
3252#endif
3253
3254 PVM pVM = pVCpu->CTX_SUFF(pVM);
3255 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3256 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3257
3258 /*
3259 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3260 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3261 */
3262 if ( CPUMIsGuestInLongMode(pVCpu)
3263 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3264 {
3265 return true;
3266 }
3267
3268 /*
3269 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3270 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3271 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3272 */
3273 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3274 && (pMixedCtx->cr0 & X86_CR0_PG)
3275 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3276 {
3277 /* Assert that host is PAE capable. */
3278 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3279 return true;
3280 }
3281
3282 /** @todo Check the latest Intel spec. for any other bits,
3283 * like SMEP/SMAP? */
3284 return false;
3285}
3286
3287
3288/**
3289 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3290 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3291 * controls".
3292 *
3293 * @returns VBox status code.
3294 * @param pVCpu The cross context virtual CPU structure.
3295 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3296 * out-of-sync. Make sure to update the required fields
3297 * before using them.
3298 *
3299 * @remarks Requires EFER.
3300 * @remarks No-long-jump zone!!!
3301 */
3302DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3303{
3304 int rc = VINF_SUCCESS;
3305 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3306 {
3307 PVM pVM = pVCpu->CTX_SUFF(pVM);
3308 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3309 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3310
3311 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3312 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3313
3314 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3315 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3316 {
3317 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3318 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3319 }
3320 else
3321 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3322
3323 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3324 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3325 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3326 {
3327 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3328 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3329 }
3330
3331 /*
3332 * The following should -not- be set (since we're not in SMM mode):
3333 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3334 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3335 */
3336
3337 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3338 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3339
3340 if ((val & zap) != val)
3341 {
3342 LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3343 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3344 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3345 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3346 }
3347
3348 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3349 AssertRCReturn(rc, rc);
3350
3351 pVCpu->hm.s.vmx.u32EntryCtls = val;
3352 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3353 }
3354 return rc;
3355}
3356
3357
3358/**
3359 * Sets up the VM-exit controls in the VMCS.
3360 *
3361 * @returns VBox status code.
3362 * @param pVCpu The cross context virtual CPU structure.
3363 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3364 * out-of-sync. Make sure to update the required fields
3365 * before using them.
3366 *
3367 * @remarks Requires EFER.
3368 */
3369DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3370{
3371 NOREF(pMixedCtx);
3372
3373 int rc = VINF_SUCCESS;
3374 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3375 {
3376 PVM pVM = pVCpu->CTX_SUFF(pVM);
3377 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3378 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3379
3380 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3381 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3382
3383 /*
3384 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3385 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3386 */
3387#if HC_ARCH_BITS == 64
3388 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3389 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3390#else
3391 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3392 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3393 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3394 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3395 {
3396 /* The switcher returns to long mode, EFER is managed by the switcher. */
3397 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3398 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3399 }
3400 else
3401 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3402#endif
3403
3404 /* If the newer VMCS fields for managing EFER exists, use it. */
3405 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3406 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3407 {
3408 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3409 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3410 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3411 }
3412
3413 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3414 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3415
3416 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3417 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3418 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3419
3420 if ( pVM->hm.s.vmx.fUsePreemptTimer
3421 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3422 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3423
3424 if ((val & zap) != val)
3425 {
3426 LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3427 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3428 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3429 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3430 }
3431
3432 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3433 AssertRCReturn(rc, rc);
3434
3435 pVCpu->hm.s.vmx.u32ExitCtls = val;
3436 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3437 }
3438 return rc;
3439}
3440
3441
3442/**
3443 * Sets the TPR threshold in the VMCS.
3444 *
3445 * @returns VBox status code.
3446 * @param pVCpu The cross context virtual CPU structure.
3447 * @param u32TprThreshold The TPR threshold (task-priority class only).
3448 */
3449DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3450{
3451 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3452 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3453 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3454}
3455
3456
3457/**
3458 * Loads the guest APIC and related state.
3459 *
3460 * @returns VBox status code.
3461 * @param pVCpu The cross context virtual CPU structure.
3462 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3463 * out-of-sync. Make sure to update the required fields
3464 * before using them.
3465 *
3466 * @remarks No-long-jump zone!!!
3467 */
3468DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3469{
3470 NOREF(pMixedCtx);
3471
3472 int rc = VINF_SUCCESS;
3473 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_APIC_STATE))
3474 {
3475 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3476 && APICIsEnabled(pVCpu))
3477 {
3478 /*
3479 * Setup TPR shadowing.
3480 */
3481 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3482 {
3483 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3484
3485 bool fPendingIntr = false;
3486 uint8_t u8Tpr = 0;
3487 uint8_t u8PendingIntr = 0;
3488 rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3489 AssertRCReturn(rc, rc);
3490
3491 /*
3492 * If there are interrupts pending but masked by the TPR, instruct VT-x to cause a TPR-below-threshold VM-exit
3493 * when the guest lowers its TPR below the priority of the pending interrupt so we can deliver the interrupt.
3494 * If there are no interrupts pending, set threshold to 0 to not cause any TPR-below-threshold VM-exits.
3495 */
3496 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3497 uint32_t u32TprThreshold = 0;
3498 if (fPendingIntr)
3499 {
3500 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3501 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3502 const uint8_t u8TprPriority = u8Tpr >> 4;
3503 if (u8PendingPriority <= u8TprPriority)
3504 u32TprThreshold = u8PendingPriority;
3505 }
3506
3507 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3508 AssertRCReturn(rc, rc);
3509 }
3510 }
3511 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_APIC_STATE);
3512 }
3513
3514 return rc;
3515}
3516
3517
3518/**
3519 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3520 *
3521 * @returns Guest's interruptibility-state.
3522 * @param pVCpu The cross context virtual CPU structure.
3523 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3524 * out-of-sync. Make sure to update the required fields
3525 * before using them.
3526 *
3527 * @remarks No-long-jump zone!!!
3528 */
3529DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3530{
3531 /*
3532 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3533 */
3534 uint32_t uIntrState = 0;
3535 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3536 {
3537 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3538 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3539 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3540 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3541 {
3542 if (pMixedCtx->eflags.Bits.u1IF)
3543 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3544 else
3545 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3546 }
3547 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3548 {
3549 /*
3550 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3551 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3552 */
3553 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3554 }
3555 }
3556
3557 /*
3558 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3559 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3560 * setting this would block host-NMIs and IRET will not clear the blocking.
3561 *
3562 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3563 */
3564 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3565 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3566 {
3567 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3568 }
3569
3570 return uIntrState;
3571}
3572
3573
3574/**
3575 * Loads the guest's interruptibility-state into the guest-state area in the
3576 * VMCS.
3577 *
3578 * @returns VBox status code.
3579 * @param pVCpu The cross context virtual CPU structure.
3580 * @param uIntrState The interruptibility-state to set.
3581 */
3582static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3583{
3584 NOREF(pVCpu);
3585 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3586 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3587 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3588 AssertRC(rc);
3589 return rc;
3590}
3591
3592
3593/**
3594 * Loads the exception intercepts required for guest execution in the VMCS.
3595 *
3596 * @returns VBox status code.
3597 * @param pVCpu The cross context virtual CPU structure.
3598 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3599 * out-of-sync. Make sure to update the required fields
3600 * before using them.
3601 */
3602static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3603{
3604 NOREF(pMixedCtx);
3605 int rc = VINF_SUCCESS;
3606 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMM_GUEST_XCPT_INTERCEPTS))
3607 {
3608 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3609 if (pVCpu->hm.s.fGIMTrapXcptUD)
3610 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3611#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3612 else
3613 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3614#endif
3615
3616 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3617 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3618
3619 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3620 AssertRCReturn(rc, rc);
3621
3622 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMM_GUEST_XCPT_INTERCEPTS);
3623 Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3624 pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3625 }
3626 return rc;
3627}
3628
3629
3630/**
3631 * Loads the guest's RIP into the guest-state area in the VMCS.
3632 *
3633 * @returns VBox status code.
3634 * @param pVCpu The cross context virtual CPU structure.
3635 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3636 * out-of-sync. Make sure to update the required fields
3637 * before using them.
3638 *
3639 * @remarks No-long-jump zone!!!
3640 */
3641static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3642{
3643 int rc = VINF_SUCCESS;
3644 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3645 {
3646 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3647 AssertRCReturn(rc, rc);
3648
3649 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3650 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3651 HMCPU_CF_VALUE(pVCpu)));
3652
3653 /* Update the exit history entry with the correct CS.BASE + RIP or just RIP. */
3654 if (HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
3655 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->cs.u64Base + pMixedCtx->rip, true);
3656 else
3657 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->rip, false);
3658 }
3659 return rc;
3660}
3661
3662
3663/**
3664 * Loads the guest's RSP into the guest-state area in the VMCS.
3665 *
3666 * @returns VBox status code.
3667 * @param pVCpu The cross context virtual CPU structure.
3668 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3669 * out-of-sync. Make sure to update the required fields
3670 * before using them.
3671 *
3672 * @remarks No-long-jump zone!!!
3673 */
3674static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3675{
3676 int rc = VINF_SUCCESS;
3677 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3678 {
3679 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3680 AssertRCReturn(rc, rc);
3681
3682 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3683 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3684 }
3685 return rc;
3686}
3687
3688
3689/**
3690 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3691 *
3692 * @returns VBox status code.
3693 * @param pVCpu The cross context virtual CPU structure.
3694 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3695 * out-of-sync. Make sure to update the required fields
3696 * before using them.
3697 *
3698 * @remarks No-long-jump zone!!!
3699 */
3700static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3701{
3702 int rc = VINF_SUCCESS;
3703 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3704 {
3705 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3706 Let us assert it as such and use 32-bit VMWRITE. */
3707 Assert(!(pMixedCtx->rflags.u64 >> 32));
3708 X86EFLAGS Eflags = pMixedCtx->eflags;
3709 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3710 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3711 * These will never be cleared/set, unless some other part of the VMM
3712 * code is buggy - in which case we're better of finding and fixing
3713 * those bugs than hiding them. */
3714 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3715 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3716 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3717 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3718
3719 /*
3720 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3721 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3722 */
3723 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3724 {
3725 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3726 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3727 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3728 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3729 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3730 }
3731
3732 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3733 AssertRCReturn(rc, rc);
3734
3735 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3736 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3737 }
3738 return rc;
3739}
3740
3741
3742/**
3743 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3744 *
3745 * @returns VBox status code.
3746 * @param pVCpu The cross context virtual CPU structure.
3747 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3748 * out-of-sync. Make sure to update the required fields
3749 * before using them.
3750 *
3751 * @remarks No-long-jump zone!!!
3752 */
3753DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3754{
3755 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3756 rc |= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3757 rc |= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3758 AssertRCReturn(rc, rc);
3759 return rc;
3760}
3761
3762
3763/**
3764 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3765 * CR0 is partially shared with the host and we have to consider the FPU bits.
3766 *
3767 * @returns VBox status code.
3768 * @param pVCpu The cross context virtual CPU structure.
3769 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3770 * out-of-sync. Make sure to update the required fields
3771 * before using them.
3772 *
3773 * @remarks No-long-jump zone!!!
3774 */
3775static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3776{
3777 Assert(CPUMIsGuestFPUStateActive(pVCpu));
3778
3779 /*
3780 * Guest CR0.
3781 * Guest FPU.
3782 */
3783 int rc = VINF_SUCCESS;
3784 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3785 {
3786 Assert(!(pMixedCtx->cr0 >> 32));
3787 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3788 PVM pVM = pVCpu->CTX_SUFF(pVM);
3789
3790 /* The guest's view (read access) of its CR0 is unblemished. */
3791 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3792 AssertRCReturn(rc, rc);
3793 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3794
3795 /* Setup VT-x's view of the guest CR0. */
3796 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3797 if (pVM->hm.s.fNestedPaging)
3798 {
3799 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3800 {
3801 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3802 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3803 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3804 }
3805 else
3806 {
3807 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3808 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3809 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3810 }
3811
3812 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3813 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3814 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3815
3816 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3817 AssertRCReturn(rc, rc);
3818 }
3819 else
3820 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3821
3822 /*
3823 * Guest FPU bits.
3824 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3825 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3826 */
3827 u32GuestCR0 |= X86_CR0_NE;
3828
3829 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3830 bool fInterceptMF = false;
3831 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3832 fInterceptMF = true;
3833
3834 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3835 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3836 {
3837 Assert(PDMVmmDevHeapIsEnabled(pVM));
3838 Assert(pVM->hm.s.vmx.pRealModeTSS);
3839 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3840 }
3841 else
3842 {
3843 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3844 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3845 if (fInterceptMF)
3846 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3847 }
3848 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMM_GUEST_XCPT_INTERCEPTS);
3849
3850 /* Additional intercepts for debugging, define these yourself explicitly. */
3851#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3852 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3853 | RT_BIT(X86_XCPT_BP)
3854 | RT_BIT(X86_XCPT_DE)
3855 | RT_BIT(X86_XCPT_NM)
3856 | RT_BIT(X86_XCPT_TS)
3857 | RT_BIT(X86_XCPT_UD)
3858 | RT_BIT(X86_XCPT_NP)
3859 | RT_BIT(X86_XCPT_SS)
3860 | RT_BIT(X86_XCPT_GP)
3861 | RT_BIT(X86_XCPT_PF)
3862 | RT_BIT(X86_XCPT_MF)
3863 ;
3864#elif defined(HMVMX_ALWAYS_TRAP_PF)
3865 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3866#endif
3867
3868 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3869
3870 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3871 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3872 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3873 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3874 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3875 else
3876 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3877
3878 u32GuestCR0 |= uSetCR0;
3879 u32GuestCR0 &= uZapCR0;
3880 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3881
3882 /* Write VT-x's view of the guest CR0 into the VMCS. */
3883 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3884 AssertRCReturn(rc, rc);
3885 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3886 uZapCR0));
3887
3888 /*
3889 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3890 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3891 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3892 */
3893 uint32_t u32CR0Mask = 0;
3894 u32CR0Mask = X86_CR0_PE
3895 | X86_CR0_NE
3896 | X86_CR0_WP
3897 | X86_CR0_PG
3898 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3899 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3900 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3901
3902 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3903 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3904 * and @bugref{6944}. */
3905#if 0
3906 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3907 u32CR0Mask &= ~X86_CR0_PE;
3908#endif
3909 if (pVM->hm.s.fNestedPaging)
3910 u32CR0Mask &= ~X86_CR0_WP;
3911
3912 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3913 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3914 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3915 AssertRCReturn(rc, rc);
3916 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3917
3918 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3919 }
3920 return rc;
3921}
3922
3923
3924/**
3925 * Loads the guest control registers (CR3, CR4) into the guest-state area
3926 * in the VMCS.
3927 *
3928 * @returns VBox strict status code.
3929 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3930 * without unrestricted guest access and the VMMDev is not presently
3931 * mapped (e.g. EFI32).
3932 *
3933 * @param pVCpu The cross context virtual CPU structure.
3934 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3935 * out-of-sync. Make sure to update the required fields
3936 * before using them.
3937 *
3938 * @remarks No-long-jump zone!!!
3939 */
3940static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3941{
3942 int rc = VINF_SUCCESS;
3943 PVM pVM = pVCpu->CTX_SUFF(pVM);
3944
3945 /*
3946 * Guest CR2.
3947 * It's always loaded in the assembler code. Nothing to do here.
3948 */
3949
3950 /*
3951 * Guest CR3.
3952 */
3953 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3954 {
3955 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3956 if (pVM->hm.s.fNestedPaging)
3957 {
3958 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3959
3960 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3961 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3962 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3963 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3964
3965 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3966 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3967 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3968
3969 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3970 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3971 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3972 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3973 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3974 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3975 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3976
3977 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3978 AssertRCReturn(rc, rc);
3979 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3980
3981 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3982 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3983 {
3984 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3985 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3986 {
3987 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3988 AssertRCReturn(rc, rc);
3989 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3990 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3991 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3992 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3993 AssertRCReturn(rc, rc);
3994 }
3995
3996 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3997 have Unrestricted Execution to handle the guest when it's not using paging. */
3998 GCPhysGuestCR3 = pMixedCtx->cr3;
3999 }
4000 else
4001 {
4002 /*
4003 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
4004 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
4005 * EPT takes care of translating it to host-physical addresses.
4006 */
4007 RTGCPHYS GCPhys;
4008 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
4009
4010 /* We obtain it here every time as the guest could have relocated this PCI region. */
4011 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
4012 if (RT_SUCCESS(rc))
4013 { /* likely */ }
4014 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4015 {
4016 Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
4017 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4018 }
4019 else
4020 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4021
4022 GCPhysGuestCR3 = GCPhys;
4023 }
4024
4025 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
4026 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4027 }
4028 else
4029 {
4030 /* Non-nested paging case, just use the hypervisor's CR3. */
4031 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4032
4033 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
4034 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4035 }
4036 AssertRCReturn(rc, rc);
4037
4038 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
4039 }
4040
4041 /*
4042 * Guest CR4.
4043 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
4044 */
4045 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
4046 {
4047 Assert(!(pMixedCtx->cr4 >> 32));
4048 uint32_t u32GuestCR4 = pMixedCtx->cr4;
4049
4050 /* The guest's view of its CR4 is unblemished. */
4051 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4052 AssertRCReturn(rc, rc);
4053 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4054
4055 /* Setup VT-x's view of the guest CR4. */
4056 /*
4057 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4058 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4059 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4060 */
4061 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4062 {
4063 Assert(pVM->hm.s.vmx.pRealModeTSS);
4064 Assert(PDMVmmDevHeapIsEnabled(pVM));
4065 u32GuestCR4 &= ~X86_CR4_VME;
4066 }
4067
4068 if (pVM->hm.s.fNestedPaging)
4069 {
4070 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4071 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4072 {
4073 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4074 u32GuestCR4 |= X86_CR4_PSE;
4075 /* Our identity mapping is a 32-bit page directory. */
4076 u32GuestCR4 &= ~X86_CR4_PAE;
4077 }
4078 /* else use guest CR4.*/
4079 }
4080 else
4081 {
4082 /*
4083 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4084 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4085 */
4086 switch (pVCpu->hm.s.enmShadowMode)
4087 {
4088 case PGMMODE_REAL: /* Real-mode. */
4089 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4090 case PGMMODE_32_BIT: /* 32-bit paging. */
4091 {
4092 u32GuestCR4 &= ~X86_CR4_PAE;
4093 break;
4094 }
4095
4096 case PGMMODE_PAE: /* PAE paging. */
4097 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4098 {
4099 u32GuestCR4 |= X86_CR4_PAE;
4100 break;
4101 }
4102
4103 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4104 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4105#ifdef VBOX_ENABLE_64_BITS_GUESTS
4106 break;
4107#endif
4108 default:
4109 AssertFailed();
4110 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4111 }
4112 }
4113
4114 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4115 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4116 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4117 u32GuestCR4 |= uSetCR4;
4118 u32GuestCR4 &= uZapCR4;
4119
4120 /* Write VT-x's view of the guest CR4 into the VMCS. */
4121 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4122 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4123 AssertRCReturn(rc, rc);
4124
4125 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4126 uint32_t u32CR4Mask = X86_CR4_VME
4127 | X86_CR4_PAE
4128 | X86_CR4_PGE
4129 | X86_CR4_PSE
4130 | X86_CR4_VMXE;
4131 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4132 u32CR4Mask |= X86_CR4_OSXSAVE;
4133 if (pVM->cpum.ro.GuestFeatures.fPcid)
4134 u32CR4Mask |= X86_CR4_PCIDE;
4135 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4136 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4137 AssertRCReturn(rc, rc);
4138
4139 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4140 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4141
4142 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4143 }
4144 return rc;
4145}
4146
4147
4148/**
4149 * Loads the guest debug registers into the guest-state area in the VMCS.
4150 *
4151 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4152 *
4153 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4154 *
4155 * @returns VBox status code.
4156 * @param pVCpu The cross context virtual CPU structure.
4157 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4158 * out-of-sync. Make sure to update the required fields
4159 * before using them.
4160 *
4161 * @remarks No-long-jump zone!!!
4162 */
4163static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4164{
4165 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4166 return VINF_SUCCESS;
4167
4168#ifdef VBOX_STRICT
4169 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4170 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4171 {
4172 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4173 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4174 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4175 }
4176#endif
4177
4178 int rc;
4179 PVM pVM = pVCpu->CTX_SUFF(pVM);
4180 bool fSteppingDB = false;
4181 bool fInterceptMovDRx = false;
4182 if (pVCpu->hm.s.fSingleInstruction)
4183 {
4184 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4185 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4186 {
4187 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4188 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4189 AssertRCReturn(rc, rc);
4190 Assert(fSteppingDB == false);
4191 }
4192 else
4193 {
4194 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4195 pVCpu->hm.s.fClearTrapFlag = true;
4196 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4197 fSteppingDB = true;
4198 }
4199 }
4200
4201 if ( fSteppingDB
4202 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4203 {
4204 /*
4205 * Use the combined guest and host DRx values found in the hypervisor
4206 * register set because the debugger has breakpoints active or someone
4207 * is single stepping on the host side without a monitor trap flag.
4208 *
4209 * Note! DBGF expects a clean DR6 state before executing guest code.
4210 */
4211#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4212 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4213 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4214 {
4215 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4216 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4217 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4218 }
4219 else
4220#endif
4221 if (!CPUMIsHyperDebugStateActive(pVCpu))
4222 {
4223 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4224 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4225 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4226 }
4227
4228 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4229 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4230 AssertRCReturn(rc, rc);
4231
4232 pVCpu->hm.s.fUsingHyperDR7 = true;
4233 fInterceptMovDRx = true;
4234 }
4235 else
4236 {
4237 /*
4238 * If the guest has enabled debug registers, we need to load them prior to
4239 * executing guest code so they'll trigger at the right time.
4240 */
4241 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4242 {
4243#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4244 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4245 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4246 {
4247 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4248 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4249 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4250 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4251 }
4252 else
4253#endif
4254 if (!CPUMIsGuestDebugStateActive(pVCpu))
4255 {
4256 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4257 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4258 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4259 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4260 }
4261 Assert(!fInterceptMovDRx);
4262 }
4263 /*
4264 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4265 * must intercept #DB in order to maintain a correct DR6 guest value, and
4266 * because we need to intercept it to prevent nested #DBs from hanging the
4267 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4268 */
4269#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4270 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4271 && !CPUMIsGuestDebugStateActive(pVCpu))
4272#else
4273 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4274#endif
4275 {
4276 fInterceptMovDRx = true;
4277 }
4278
4279 /* Update guest DR7. */
4280 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4281 AssertRCReturn(rc, rc);
4282
4283 pVCpu->hm.s.fUsingHyperDR7 = false;
4284 }
4285
4286 /*
4287 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4288 */
4289 if (fInterceptMovDRx)
4290 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4291 else
4292 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4293 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4294 AssertRCReturn(rc, rc);
4295
4296 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4297 return VINF_SUCCESS;
4298}
4299
4300
4301#ifdef VBOX_STRICT
4302/**
4303 * Strict function to validate segment registers.
4304 *
4305 * @remarks ASSUMES CR0 is up to date.
4306 */
4307static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4308{
4309 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4310 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4311 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4312 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4313 && ( !CPUMIsGuestInRealModeEx(pCtx)
4314 && !CPUMIsGuestInV86ModeEx(pCtx)))
4315 {
4316 /* Protected mode checks */
4317 /* CS */
4318 Assert(pCtx->cs.Attr.n.u1Present);
4319 Assert(!(pCtx->cs.Attr.u & 0xf00));
4320 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4321 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4322 || !(pCtx->cs.Attr.n.u1Granularity));
4323 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4324 || (pCtx->cs.Attr.n.u1Granularity));
4325 /* CS cannot be loaded with NULL in protected mode. */
4326 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4327 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4328 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4329 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4330 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4331 else
4332 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4333 /* SS */
4334 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4335 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4336 if ( !(pCtx->cr0 & X86_CR0_PE)
4337 || pCtx->cs.Attr.n.u4Type == 3)
4338 {
4339 Assert(!pCtx->ss.Attr.n.u2Dpl);
4340 }
4341 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4342 {
4343 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4344 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4345 Assert(pCtx->ss.Attr.n.u1Present);
4346 Assert(!(pCtx->ss.Attr.u & 0xf00));
4347 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4348 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4349 || !(pCtx->ss.Attr.n.u1Granularity));
4350 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4351 || (pCtx->ss.Attr.n.u1Granularity));
4352 }
4353 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4354 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4355 {
4356 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4357 Assert(pCtx->ds.Attr.n.u1Present);
4358 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4359 Assert(!(pCtx->ds.Attr.u & 0xf00));
4360 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4361 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4362 || !(pCtx->ds.Attr.n.u1Granularity));
4363 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4364 || (pCtx->ds.Attr.n.u1Granularity));
4365 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4366 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4367 }
4368 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4369 {
4370 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4371 Assert(pCtx->es.Attr.n.u1Present);
4372 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4373 Assert(!(pCtx->es.Attr.u & 0xf00));
4374 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4375 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4376 || !(pCtx->es.Attr.n.u1Granularity));
4377 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4378 || (pCtx->es.Attr.n.u1Granularity));
4379 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4380 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4381 }
4382 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4383 {
4384 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4385 Assert(pCtx->fs.Attr.n.u1Present);
4386 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4387 Assert(!(pCtx->fs.Attr.u & 0xf00));
4388 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4389 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4390 || !(pCtx->fs.Attr.n.u1Granularity));
4391 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4392 || (pCtx->fs.Attr.n.u1Granularity));
4393 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4394 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4395 }
4396 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4397 {
4398 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4399 Assert(pCtx->gs.Attr.n.u1Present);
4400 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4401 Assert(!(pCtx->gs.Attr.u & 0xf00));
4402 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4403 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4404 || !(pCtx->gs.Attr.n.u1Granularity));
4405 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4406 || (pCtx->gs.Attr.n.u1Granularity));
4407 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4408 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4409 }
4410 /* 64-bit capable CPUs. */
4411# if HC_ARCH_BITS == 64
4412 Assert(!(pCtx->cs.u64Base >> 32));
4413 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4414 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4415 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4416# endif
4417 }
4418 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4419 || ( CPUMIsGuestInRealModeEx(pCtx)
4420 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4421 {
4422 /* Real and v86 mode checks. */
4423 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4424 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4425 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4426 {
4427 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4428 }
4429 else
4430 {
4431 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4432 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4433 }
4434
4435 /* CS */
4436 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4437 Assert(pCtx->cs.u32Limit == 0xffff);
4438 Assert(u32CSAttr == 0xf3);
4439 /* SS */
4440 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4441 Assert(pCtx->ss.u32Limit == 0xffff);
4442 Assert(u32SSAttr == 0xf3);
4443 /* DS */
4444 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4445 Assert(pCtx->ds.u32Limit == 0xffff);
4446 Assert(u32DSAttr == 0xf3);
4447 /* ES */
4448 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4449 Assert(pCtx->es.u32Limit == 0xffff);
4450 Assert(u32ESAttr == 0xf3);
4451 /* FS */
4452 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4453 Assert(pCtx->fs.u32Limit == 0xffff);
4454 Assert(u32FSAttr == 0xf3);
4455 /* GS */
4456 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4457 Assert(pCtx->gs.u32Limit == 0xffff);
4458 Assert(u32GSAttr == 0xf3);
4459 /* 64-bit capable CPUs. */
4460# if HC_ARCH_BITS == 64
4461 Assert(!(pCtx->cs.u64Base >> 32));
4462 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4463 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4464 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4465# endif
4466 }
4467}
4468#endif /* VBOX_STRICT */
4469
4470
4471/**
4472 * Writes a guest segment register into the guest-state area in the VMCS.
4473 *
4474 * @returns VBox status code.
4475 * @param pVCpu The cross context virtual CPU structure.
4476 * @param idxSel Index of the selector in the VMCS.
4477 * @param idxLimit Index of the segment limit in the VMCS.
4478 * @param idxBase Index of the segment base in the VMCS.
4479 * @param idxAccess Index of the access rights of the segment in the VMCS.
4480 * @param pSelReg Pointer to the segment selector.
4481 *
4482 * @remarks No-long-jump zone!!!
4483 */
4484static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4485 uint32_t idxAccess, PCPUMSELREG pSelReg)
4486{
4487 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4488 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4489 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4490 AssertRCReturn(rc, rc);
4491
4492 uint32_t u32Access = pSelReg->Attr.u;
4493 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4494 {
4495 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4496 u32Access = 0xf3;
4497 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4498 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4499 }
4500 else
4501 {
4502 /*
4503 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4504 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4505 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4506 * loaded in protected-mode have their attribute as 0.
4507 */
4508 if (!u32Access)
4509 u32Access = X86DESCATTR_UNUSABLE;
4510 }
4511
4512 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4513 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4514 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4515
4516 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4517 AssertRCReturn(rc, rc);
4518 return rc;
4519}
4520
4521
4522/**
4523 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4524 * into the guest-state area in the VMCS.
4525 *
4526 * @returns VBox status code.
4527 * @param pVCpu The cross context virtual CPU structure.
4528 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4529 * out-of-sync. Make sure to update the required fields
4530 * before using them.
4531 *
4532 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4533 * @remarks No-long-jump zone!!!
4534 */
4535static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4536{
4537 int rc = VERR_INTERNAL_ERROR_5;
4538 PVM pVM = pVCpu->CTX_SUFF(pVM);
4539
4540 /*
4541 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4542 */
4543 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4544 {
4545 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4546 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4547 {
4548 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4549 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4550 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4551 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4552 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4553 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4554 }
4555
4556#ifdef VBOX_WITH_REM
4557 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4558 {
4559 Assert(pVM->hm.s.vmx.pRealModeTSS);
4560 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4561 if ( pVCpu->hm.s.vmx.fWasInRealMode
4562 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4563 {
4564 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4565 in real-mode (e.g. OpenBSD 4.0) */
4566 REMFlushTBs(pVM);
4567 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4568 pVCpu->hm.s.vmx.fWasInRealMode = false;
4569 }
4570 }
4571#endif
4572 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4573 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4574 AssertRCReturn(rc, rc);
4575 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4576 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4577 AssertRCReturn(rc, rc);
4578 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4579 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4580 AssertRCReturn(rc, rc);
4581 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4582 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4583 AssertRCReturn(rc, rc);
4584 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4585 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4586 AssertRCReturn(rc, rc);
4587 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4588 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4589 AssertRCReturn(rc, rc);
4590
4591#ifdef VBOX_STRICT
4592 /* Validate. */
4593 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4594#endif
4595
4596 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4597 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4598 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4599
4600 /* Update the exit history entry with the correct CS.BASE + RIP. */
4601 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
4602 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->cs.u64Base + pMixedCtx->rip, true);
4603 }
4604
4605 /*
4606 * Guest TR.
4607 */
4608 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4609 {
4610 /*
4611 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4612 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4613 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4614 */
4615 uint16_t u16Sel = 0;
4616 uint32_t u32Limit = 0;
4617 uint64_t u64Base = 0;
4618 uint32_t u32AccessRights = 0;
4619
4620 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4621 {
4622 u16Sel = pMixedCtx->tr.Sel;
4623 u32Limit = pMixedCtx->tr.u32Limit;
4624 u64Base = pMixedCtx->tr.u64Base;
4625 u32AccessRights = pMixedCtx->tr.Attr.u;
4626 }
4627 else
4628 {
4629 Assert(pVM->hm.s.vmx.pRealModeTSS);
4630 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4631
4632 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4633 RTGCPHYS GCPhys;
4634 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4635 AssertRCReturn(rc, rc);
4636
4637 X86DESCATTR DescAttr;
4638 DescAttr.u = 0;
4639 DescAttr.n.u1Present = 1;
4640 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4641
4642 u16Sel = 0;
4643 u32Limit = HM_VTX_TSS_SIZE;
4644 u64Base = GCPhys; /* in real-mode phys = virt. */
4645 u32AccessRights = DescAttr.u;
4646 }
4647
4648 /* Validate. */
4649 Assert(!(u16Sel & RT_BIT(2)));
4650 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4651 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4652 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4653 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4654 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4655 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4656 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4657 Assert( (u32Limit & 0xfff) == 0xfff
4658 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4659 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4660 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4661
4662 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4663 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4664 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4665 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4666 AssertRCReturn(rc, rc);
4667
4668 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4669 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4670 }
4671
4672 /*
4673 * Guest GDTR.
4674 */
4675 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4676 {
4677 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4678 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4679 AssertRCReturn(rc, rc);
4680
4681 /* Validate. */
4682 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4683
4684 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4685 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4686 }
4687
4688 /*
4689 * Guest LDTR.
4690 */
4691 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4692 {
4693 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4694 uint32_t u32Access = 0;
4695 if (!pMixedCtx->ldtr.Attr.u)
4696 u32Access = X86DESCATTR_UNUSABLE;
4697 else
4698 u32Access = pMixedCtx->ldtr.Attr.u;
4699
4700 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4701 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4702 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4703 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4704 AssertRCReturn(rc, rc);
4705
4706 /* Validate. */
4707 if (!(u32Access & X86DESCATTR_UNUSABLE))
4708 {
4709 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4710 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4711 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4712 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4713 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4714 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4715 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4716 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4717 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4718 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4719 }
4720
4721 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4722 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4723 }
4724
4725 /*
4726 * Guest IDTR.
4727 */
4728 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4729 {
4730 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4731 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4732 AssertRCReturn(rc, rc);
4733
4734 /* Validate. */
4735 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4736
4737 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4738 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4739 }
4740
4741 return VINF_SUCCESS;
4742}
4743
4744
4745/**
4746 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4747 * areas.
4748 *
4749 * These MSRs will automatically be loaded to the host CPU on every successful
4750 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4751 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4752 * -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4753 *
4754 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4755 *
4756 * @returns VBox status code.
4757 * @param pVCpu The cross context virtual CPU structure.
4758 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4759 * out-of-sync. Make sure to update the required fields
4760 * before using them.
4761 *
4762 * @remarks No-long-jump zone!!!
4763 */
4764static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4765{
4766 AssertPtr(pVCpu);
4767 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4768
4769 /*
4770 * MSRs that we use the auto-load/store MSR area in the VMCS.
4771 */
4772 PVM pVM = pVCpu->CTX_SUFF(pVM);
4773 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4774 {
4775 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4776#if HC_ARCH_BITS == 32
4777 if (pVM->hm.s.fAllow64BitGuests)
4778 {
4779 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4780 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4781 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4782 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4783 AssertRCReturn(rc, rc);
4784# ifdef LOG_ENABLED
4785 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4786 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4787 {
4788 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4789 pMsr->u64Value));
4790 }
4791# endif
4792 }
4793#endif
4794 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4795 }
4796
4797 /*
4798 * Guest Sysenter MSRs.
4799 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4800 * VM-exits on WRMSRs for these MSRs.
4801 */
4802 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4803 {
4804 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4805 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4806 }
4807
4808 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4809 {
4810 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4811 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4812 }
4813
4814 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4815 {
4816 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4817 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4818 }
4819
4820 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4821 {
4822 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4823 {
4824 /*
4825 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4826 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4827 */
4828 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4829 {
4830 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4831 AssertRCReturn(rc,rc);
4832 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4833 }
4834 else
4835 {
4836 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4837 NULL /* pfAddedAndUpdated */);
4838 AssertRCReturn(rc, rc);
4839
4840 /* We need to intercept reads too, see @bugref{7386#c16}. */
4841 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4842 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4843 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4844 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4845 }
4846 }
4847 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4848 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4849 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4850 }
4851
4852 return VINF_SUCCESS;
4853}
4854
4855
4856/**
4857 * Loads the guest activity state into the guest-state area in the VMCS.
4858 *
4859 * @returns VBox status code.
4860 * @param pVCpu The cross context virtual CPU structure.
4861 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4862 * out-of-sync. Make sure to update the required fields
4863 * before using them.
4864 *
4865 * @remarks No-long-jump zone!!!
4866 */
4867static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4868{
4869 NOREF(pMixedCtx);
4870 /** @todo See if we can make use of other states, e.g.
4871 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4872 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4873 {
4874 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4875 AssertRCReturn(rc, rc);
4876
4877 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4878 }
4879 return VINF_SUCCESS;
4880}
4881
4882
4883#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4884/**
4885 * Check if guest state allows safe use of 32-bit switcher again.
4886 *
4887 * Segment bases and protected mode structures must be 32-bit addressable
4888 * because the 32-bit switcher will ignore high dword when writing these VMCS
4889 * fields. See @bugref{8432} for details.
4890 *
4891 * @returns true if safe, false if must continue to use the 64-bit switcher.
4892 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4893 * out-of-sync. Make sure to update the required fields
4894 * before using them.
4895 *
4896 * @remarks No-long-jump zone!!!
4897 */
4898static bool hmR0VmxIs32BitSwitcherSafe(PCPUMCTX pMixedCtx)
4899{
4900 if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000))
4901 return false;
4902 if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000))
4903 return false;
4904 if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000))
4905 return false;
4906 if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000))
4907 return false;
4908 if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000))
4909 return false;
4910 if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000))
4911 return false;
4912 if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000))
4913 return false;
4914 if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000))
4915 return false;
4916 if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000))
4917 return false;
4918 if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000))
4919 return false;
4920 /* All good, bases are 32-bit. */
4921 return true;
4922}
4923#endif
4924
4925
4926/**
4927 * Sets up the appropriate function to run guest code.
4928 *
4929 * @returns VBox status code.
4930 * @param pVCpu The cross context virtual CPU structure.
4931 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4932 * out-of-sync. Make sure to update the required fields
4933 * before using them.
4934 *
4935 * @remarks No-long-jump zone!!!
4936 */
4937static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4938{
4939 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4940 {
4941#ifndef VBOX_ENABLE_64_BITS_GUESTS
4942 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4943#endif
4944 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4945#if HC_ARCH_BITS == 32
4946 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4947 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4948 {
4949 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4950 {
4951 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4952 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4953 | HM_CHANGED_VMX_ENTRY_CTLS
4954 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4955 }
4956 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4957
4958 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4959 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4960 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4961 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 64-bit switcher\n", pVCpu->idCpu));
4962 }
4963#else
4964 /* 64-bit host. */
4965 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4966#endif
4967 }
4968 else
4969 {
4970 /* Guest is not in long mode, use the 32-bit handler. */
4971#if HC_ARCH_BITS == 32
4972 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4973 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4974 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4975 {
4976 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4977 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4978 | HM_CHANGED_VMX_ENTRY_CTLS
4979 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4980 }
4981# ifdef VBOX_ENABLE_64_BITS_GUESTS
4982 /*
4983 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel design, see @bugref{8432#c7}.
4984 * If real-on-v86 mode is active, clear the 64-bit switcher flag because now we know the guest is in a sane
4985 * state where it's safe to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4986 * the much faster 32-bit switcher again.
4987 */
4988 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4989 {
4990 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4991 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher\n", pVCpu->idCpu));
4992 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4993 }
4994 else
4995 {
4996 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4997 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4998 || hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
4999 {
5000 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
5001 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5002 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR
5003 | HM_CHANGED_VMX_ENTRY_CTLS
5004 | HM_CHANGED_VMX_EXIT_CTLS
5005 | HM_CHANGED_HOST_CONTEXT);
5006 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher (safe)\n", pVCpu->idCpu));
5007 }
5008 }
5009# else
5010 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5011# endif
5012#else
5013 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5014#endif
5015 }
5016 Assert(pVCpu->hm.s.vmx.pfnStartVM);
5017 return VINF_SUCCESS;
5018}
5019
5020
5021/**
5022 * Wrapper for running the guest code in VT-x.
5023 *
5024 * @returns VBox status code, no informational status codes.
5025 * @param pVM The cross context VM structure.
5026 * @param pVCpu The cross context virtual CPU structure.
5027 * @param pCtx Pointer to the guest-CPU context.
5028 *
5029 * @remarks No-long-jump zone!!!
5030 */
5031DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5032{
5033 /*
5034 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
5035 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
5036 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
5037 */
5038 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
5039 /** @todo Add stats for resume vs launch. */
5040#ifdef VBOX_WITH_KERNEL_USING_XMM
5041 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
5042#else
5043 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
5044#endif
5045 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
5046 return rc;
5047}
5048
5049
5050/**
5051 * Reports world-switch error and dumps some useful debug info.
5052 *
5053 * @param pVM The cross context VM structure.
5054 * @param pVCpu The cross context virtual CPU structure.
5055 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5056 * @param pCtx Pointer to the guest-CPU context.
5057 * @param pVmxTransient Pointer to the VMX transient structure (only
5058 * exitReason updated).
5059 */
5060static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
5061{
5062 Assert(pVM);
5063 Assert(pVCpu);
5064 Assert(pCtx);
5065 Assert(pVmxTransient);
5066 HMVMX_ASSERT_PREEMPT_SAFE();
5067
5068 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
5069 switch (rcVMRun)
5070 {
5071 case VERR_VMX_INVALID_VMXON_PTR:
5072 AssertFailed();
5073 break;
5074 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5075 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5076 {
5077 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5078 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5079 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5080 AssertRC(rc);
5081
5082 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5083 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5084 Cannot do it here as we may have been long preempted. */
5085
5086#ifdef VBOX_STRICT
5087 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5088 pVmxTransient->uExitReason));
5089 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5090 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5091 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5092 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5093 else
5094 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5095 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5096 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5097
5098 /* VMX control bits. */
5099 uint32_t u32Val;
5100 uint64_t u64Val;
5101 RTHCUINTREG uHCReg;
5102 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5103 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5104 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5105 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5106 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5107 {
5108 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5109 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5110 }
5111 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5112 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5113 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5114 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5115 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5116 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5117 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5118 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5119 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5120 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5121 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5122 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5123 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5124 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5125 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5126 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5127 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5128 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5129 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5130 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5131 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5132 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5133 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5134 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5135 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5136 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5137 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5138 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5139 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5140 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5141 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5142 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5143 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5144 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5145 if (pVM->hm.s.fNestedPaging)
5146 {
5147 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5148 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5149 }
5150
5151 /* Guest bits. */
5152 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5153 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5154 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5155 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5156 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5157 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5158 if (pVM->hm.s.vmx.fVpid)
5159 {
5160 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5161 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5162 }
5163
5164 /* Host bits. */
5165 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5166 Log4(("Host CR0 %#RHr\n", uHCReg));
5167 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5168 Log4(("Host CR3 %#RHr\n", uHCReg));
5169 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5170 Log4(("Host CR4 %#RHr\n", uHCReg));
5171
5172 RTGDTR HostGdtr;
5173 PCX86DESCHC pDesc;
5174 ASMGetGDTR(&HostGdtr);
5175 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5176 Log4(("Host CS %#08x\n", u32Val));
5177 if (u32Val < HostGdtr.cbGdt)
5178 {
5179 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5180 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5181 }
5182
5183 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5184 Log4(("Host DS %#08x\n", u32Val));
5185 if (u32Val < HostGdtr.cbGdt)
5186 {
5187 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5188 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5189 }
5190
5191 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5192 Log4(("Host ES %#08x\n", u32Val));
5193 if (u32Val < HostGdtr.cbGdt)
5194 {
5195 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5196 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5197 }
5198
5199 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5200 Log4(("Host FS %#08x\n", u32Val));
5201 if (u32Val < HostGdtr.cbGdt)
5202 {
5203 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5204 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5205 }
5206
5207 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5208 Log4(("Host GS %#08x\n", u32Val));
5209 if (u32Val < HostGdtr.cbGdt)
5210 {
5211 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5212 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5213 }
5214
5215 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5216 Log4(("Host SS %#08x\n", u32Val));
5217 if (u32Val < HostGdtr.cbGdt)
5218 {
5219 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5220 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5221 }
5222
5223 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5224 Log4(("Host TR %#08x\n", u32Val));
5225 if (u32Val < HostGdtr.cbGdt)
5226 {
5227 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5228 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5229 }
5230
5231 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5232 Log4(("Host TR Base %#RHv\n", uHCReg));
5233 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5234 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5235 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5236 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5237 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5238 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5239 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5240 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5241 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5242 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5243 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5244 Log4(("Host RSP %#RHv\n", uHCReg));
5245 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5246 Log4(("Host RIP %#RHv\n", uHCReg));
5247# if HC_ARCH_BITS == 64
5248 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5249 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5250 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5251 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5252 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5253 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5254# endif
5255#endif /* VBOX_STRICT */
5256 break;
5257 }
5258
5259 default:
5260 /* Impossible */
5261 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5262 break;
5263 }
5264 NOREF(pVM); NOREF(pCtx);
5265}
5266
5267
5268#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5269#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5270# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5271#endif
5272#ifdef VBOX_STRICT
5273static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5274{
5275 switch (idxField)
5276 {
5277 case VMX_VMCS_GUEST_RIP:
5278 case VMX_VMCS_GUEST_RSP:
5279 case VMX_VMCS_GUEST_SYSENTER_EIP:
5280 case VMX_VMCS_GUEST_SYSENTER_ESP:
5281 case VMX_VMCS_GUEST_GDTR_BASE:
5282 case VMX_VMCS_GUEST_IDTR_BASE:
5283 case VMX_VMCS_GUEST_CS_BASE:
5284 case VMX_VMCS_GUEST_DS_BASE:
5285 case VMX_VMCS_GUEST_ES_BASE:
5286 case VMX_VMCS_GUEST_FS_BASE:
5287 case VMX_VMCS_GUEST_GS_BASE:
5288 case VMX_VMCS_GUEST_SS_BASE:
5289 case VMX_VMCS_GUEST_LDTR_BASE:
5290 case VMX_VMCS_GUEST_TR_BASE:
5291 case VMX_VMCS_GUEST_CR3:
5292 return true;
5293 }
5294 return false;
5295}
5296
5297static bool hmR0VmxIsValidReadField(uint32_t idxField)
5298{
5299 switch (idxField)
5300 {
5301 /* Read-only fields. */
5302 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5303 return true;
5304 }
5305 /* Remaining readable fields should also be writable. */
5306 return hmR0VmxIsValidWriteField(idxField);
5307}
5308#endif /* VBOX_STRICT */
5309
5310
5311/**
5312 * Executes the specified handler in 64-bit mode.
5313 *
5314 * @returns VBox status code (no informational status codes).
5315 * @param pVM The cross context VM structure.
5316 * @param pVCpu The cross context virtual CPU structure.
5317 * @param pCtx Pointer to the guest CPU context.
5318 * @param enmOp The operation to perform.
5319 * @param cParams Number of parameters.
5320 * @param paParam Array of 32-bit parameters.
5321 */
5322VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5323 uint32_t cParams, uint32_t *paParam)
5324{
5325 NOREF(pCtx);
5326
5327 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5328 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5329 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5330 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5331
5332#ifdef VBOX_STRICT
5333 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5334 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5335
5336 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5337 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5338#endif
5339
5340 /* Disable interrupts. */
5341 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5342
5343#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5344 RTCPUID idHostCpu = RTMpCpuId();
5345 CPUMR0SetLApic(pVCpu, idHostCpu);
5346#endif
5347
5348 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5349 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5350
5351 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5352 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5353 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5354
5355 /* Leave VMX Root Mode. */
5356 VMXDisable();
5357
5358 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5359
5360 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5361 CPUMSetHyperEIP(pVCpu, enmOp);
5362 for (int i = (int)cParams - 1; i >= 0; i--)
5363 CPUMPushHyper(pVCpu, paParam[i]);
5364
5365 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5366
5367 /* Call the switcher. */
5368 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5369 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5370
5371 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5372 /* Make sure the VMX instructions don't cause #UD faults. */
5373 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5374
5375 /* Re-enter VMX Root Mode */
5376 int rc2 = VMXEnable(HCPhysCpuPage);
5377 if (RT_FAILURE(rc2))
5378 {
5379 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5380 ASMSetFlags(fOldEFlags);
5381 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5382 return rc2;
5383 }
5384
5385 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5386 AssertRC(rc2);
5387 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5388 Assert(!(ASMGetFlags() & X86_EFL_IF));
5389 ASMSetFlags(fOldEFlags);
5390 return rc;
5391}
5392
5393
5394/**
5395 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5396 * supporting 64-bit guests.
5397 *
5398 * @returns VBox status code.
5399 * @param fResume Whether to VMLAUNCH or VMRESUME.
5400 * @param pCtx Pointer to the guest-CPU context.
5401 * @param pCache Pointer to the VMCS cache.
5402 * @param pVM The cross context VM structure.
5403 * @param pVCpu The cross context virtual CPU structure.
5404 */
5405DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5406{
5407 NOREF(fResume);
5408
5409 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5410 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5411
5412#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5413 pCache->uPos = 1;
5414 pCache->interPD = PGMGetInterPaeCR3(pVM);
5415 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5416#endif
5417
5418#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5419 pCache->TestIn.HCPhysCpuPage = 0;
5420 pCache->TestIn.HCPhysVmcs = 0;
5421 pCache->TestIn.pCache = 0;
5422 pCache->TestOut.HCPhysVmcs = 0;
5423 pCache->TestOut.pCache = 0;
5424 pCache->TestOut.pCtx = 0;
5425 pCache->TestOut.eflags = 0;
5426#else
5427 NOREF(pCache);
5428#endif
5429
5430 uint32_t aParam[10];
5431 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5432 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5433 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5434 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5435 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5436 aParam[5] = 0;
5437 aParam[6] = VM_RC_ADDR(pVM, pVM);
5438 aParam[7] = 0;
5439 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5440 aParam[9] = 0;
5441
5442#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5443 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5444 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5445#endif
5446 int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5447
5448#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5449 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5450 Assert(pCtx->dr[4] == 10);
5451 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5452#endif
5453
5454#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5455 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5456 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5457 pVCpu->hm.s.vmx.HCPhysVmcs));
5458 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5459 pCache->TestOut.HCPhysVmcs));
5460 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5461 pCache->TestOut.pCache));
5462 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5463 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5464 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5465 pCache->TestOut.pCtx));
5466 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5467#endif
5468 return rc;
5469}
5470
5471
5472/**
5473 * Initialize the VMCS-Read cache.
5474 *
5475 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5476 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5477 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5478 * (those that have a 32-bit FULL & HIGH part).
5479 *
5480 * @returns VBox status code.
5481 * @param pVM The cross context VM structure.
5482 * @param pVCpu The cross context virtual CPU structure.
5483 */
5484static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5485{
5486#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5487{ \
5488 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5489 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5490 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5491 ++cReadFields; \
5492}
5493
5494 AssertPtr(pVM);
5495 AssertPtr(pVCpu);
5496 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5497 uint32_t cReadFields = 0;
5498
5499 /*
5500 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5501 * and serve to indicate exceptions to the rules.
5502 */
5503
5504 /* Guest-natural selector base fields. */
5505#if 0
5506 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5507 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5508 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5509#endif
5510 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5511 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5512 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5513 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5514 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5515 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5516 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5517 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5518 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5519 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5520 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5521 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5522#if 0
5523 /* Unused natural width guest-state fields. */
5524 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5525 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5526#endif
5527 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5528 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5529
5530 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5531#if 0
5532 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5533 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5534 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5535 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5536 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5537 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5538 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5539 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5540 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5541#endif
5542
5543 /* Natural width guest-state fields. */
5544 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5545#if 0
5546 /* Currently unused field. */
5547 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5548#endif
5549
5550 if (pVM->hm.s.fNestedPaging)
5551 {
5552 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5553 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5554 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5555 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5556 }
5557 else
5558 {
5559 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5560 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5561 }
5562
5563#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5564 return VINF_SUCCESS;
5565}
5566
5567
5568/**
5569 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5570 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5571 * darwin, running 64-bit guests).
5572 *
5573 * @returns VBox status code.
5574 * @param pVCpu The cross context virtual CPU structure.
5575 * @param idxField The VMCS field encoding.
5576 * @param u64Val 16, 32 or 64-bit value.
5577 */
5578VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5579{
5580 int rc;
5581 switch (idxField)
5582 {
5583 /*
5584 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5585 */
5586 /* 64-bit Control fields. */
5587 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5588 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5589 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5590 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5591 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5592 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5593 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5594 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5595 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5596 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5597 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5598 case VMX_VMCS64_CTRL_EPTP_FULL:
5599 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5600 /* 64-bit Guest-state fields. */
5601 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5602 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5603 case VMX_VMCS64_GUEST_PAT_FULL:
5604 case VMX_VMCS64_GUEST_EFER_FULL:
5605 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5606 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5607 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5608 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5609 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5610 /* 64-bit Host-state fields. */
5611 case VMX_VMCS64_HOST_PAT_FULL:
5612 case VMX_VMCS64_HOST_EFER_FULL:
5613 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5614 {
5615 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5616 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5617 break;
5618 }
5619
5620 /*
5621 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5622 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5623 */
5624 /* Natural-width Guest-state fields. */
5625 case VMX_VMCS_GUEST_CR3:
5626 case VMX_VMCS_GUEST_ES_BASE:
5627 case VMX_VMCS_GUEST_CS_BASE:
5628 case VMX_VMCS_GUEST_SS_BASE:
5629 case VMX_VMCS_GUEST_DS_BASE:
5630 case VMX_VMCS_GUEST_FS_BASE:
5631 case VMX_VMCS_GUEST_GS_BASE:
5632 case VMX_VMCS_GUEST_LDTR_BASE:
5633 case VMX_VMCS_GUEST_TR_BASE:
5634 case VMX_VMCS_GUEST_GDTR_BASE:
5635 case VMX_VMCS_GUEST_IDTR_BASE:
5636 case VMX_VMCS_GUEST_RSP:
5637 case VMX_VMCS_GUEST_RIP:
5638 case VMX_VMCS_GUEST_SYSENTER_ESP:
5639 case VMX_VMCS_GUEST_SYSENTER_EIP:
5640 {
5641 if (!(RT_HI_U32(u64Val)))
5642 {
5643 /* If this field is 64-bit, VT-x will zero out the top bits. */
5644 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5645 }
5646 else
5647 {
5648 /* Assert that only the 32->64 switcher case should ever come here. */
5649 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5650 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5651 }
5652 break;
5653 }
5654
5655 default:
5656 {
5657 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5658 rc = VERR_INVALID_PARAMETER;
5659 break;
5660 }
5661 }
5662 AssertRCReturn(rc, rc);
5663 return rc;
5664}
5665
5666
5667/**
5668 * Queue up a VMWRITE by using the VMCS write cache.
5669 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5670 *
5671 * @param pVCpu The cross context virtual CPU structure.
5672 * @param idxField The VMCS field encoding.
5673 * @param u64Val 16, 32 or 64-bit value.
5674 */
5675VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5676{
5677 AssertPtr(pVCpu);
5678 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5679
5680 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5681 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5682
5683 /* Make sure there are no duplicates. */
5684 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5685 {
5686 if (pCache->Write.aField[i] == idxField)
5687 {
5688 pCache->Write.aFieldVal[i] = u64Val;
5689 return VINF_SUCCESS;
5690 }
5691 }
5692
5693 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5694 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5695 pCache->Write.cValidEntries++;
5696 return VINF_SUCCESS;
5697}
5698#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5699
5700
5701/**
5702 * Sets up the usage of TSC-offsetting and updates the VMCS.
5703 *
5704 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5705 * VMX preemption timer.
5706 *
5707 * @returns VBox status code.
5708 * @param pVM The cross context VM structure.
5709 * @param pVCpu The cross context virtual CPU structure.
5710 *
5711 * @remarks No-long-jump zone!!!
5712 */
5713static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5714{
5715 int rc;
5716 bool fOffsettedTsc;
5717 bool fParavirtTsc;
5718 if (pVM->hm.s.vmx.fUsePreemptTimer)
5719 {
5720 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5721 &fOffsettedTsc, &fParavirtTsc);
5722
5723 /* Make sure the returned values have sane upper and lower boundaries. */
5724 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5725 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5726 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5727 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5728
5729 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5730 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5731 }
5732 else
5733 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5734
5735 /** @todo later optimize this to be done elsewhere and not before every
5736 * VM-entry. */
5737 if (fParavirtTsc)
5738 {
5739 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5740 information before every VM-entry, hence disable it for performance sake. */
5741#if 0
5742 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5743 AssertRC(rc);
5744#endif
5745 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5746 }
5747
5748 if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5749 {
5750 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5751 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5752
5753 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5754 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5755 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5756 }
5757 else
5758 {
5759 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5760 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5761 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5762 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5763 }
5764}
5765
5766
5767#ifdef HMVMX_USE_IEM_EVENT_REFLECTION
5768/**
5769 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5770 * VM-exit interruption info type.
5771 *
5772 * @returns The IEM exception flags.
5773 * @param uVector The event vector.
5774 * @param uVmxVectorType The VMX event type.
5775 *
5776 * @remarks This function currently only constructs flags required for
5777 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5778 * and CR2 aspects of an exception are not included).
5779 */
5780static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5781{
5782 uint32_t fIemXcptFlags;
5783 switch (uVmxVectorType)
5784 {
5785 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5786 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5787 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5788 break;
5789
5790 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5791 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5792 break;
5793
5794 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5795 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5796 break;
5797
5798 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5799 {
5800 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5801 if (uVector == X86_XCPT_BP)
5802 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5803 else if (uVector == X86_XCPT_OF)
5804 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5805 else
5806 {
5807 fIemXcptFlags = 0;
5808 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5809 }
5810 break;
5811 }
5812
5813 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5814 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5815 break;
5816
5817 default:
5818 fIemXcptFlags = 0;
5819 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5820 break;
5821 }
5822 return fIemXcptFlags;
5823}
5824
5825#else
5826/**
5827 * Determines if an exception is a contributory exception.
5828 *
5829 * Contributory exceptions are ones which can cause double-faults unless the
5830 * original exception was a benign exception. Page-fault is intentionally not
5831 * included here as it's a conditional contributory exception.
5832 *
5833 * @returns true if the exception is contributory, false otherwise.
5834 * @param uVector The exception vector.
5835 */
5836DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5837{
5838 switch (uVector)
5839 {
5840 case X86_XCPT_GP:
5841 case X86_XCPT_SS:
5842 case X86_XCPT_NP:
5843 case X86_XCPT_TS:
5844 case X86_XCPT_DE:
5845 return true;
5846 default:
5847 break;
5848 }
5849 return false;
5850}
5851#endif /* HMVMX_USE_IEM_EVENT_REFLECTION */
5852
5853
5854/**
5855 * Sets an event as a pending event to be injected into the guest.
5856 *
5857 * @param pVCpu The cross context virtual CPU structure.
5858 * @param u32IntInfo The VM-entry interruption-information field.
5859 * @param cbInstr The VM-entry instruction length in bytes (for software
5860 * interrupts, exceptions and privileged software
5861 * exceptions).
5862 * @param u32ErrCode The VM-entry exception error code.
5863 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5864 * page-fault.
5865 *
5866 * @remarks Statistics counter assumes this is a guest event being injected or
5867 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5868 * always incremented.
5869 */
5870DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5871 RTGCUINTPTR GCPtrFaultAddress)
5872{
5873 Assert(!pVCpu->hm.s.Event.fPending);
5874 pVCpu->hm.s.Event.fPending = true;
5875 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5876 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5877 pVCpu->hm.s.Event.cbInstr = cbInstr;
5878 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5879}
5880
5881
5882/**
5883 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5884 *
5885 * @param pVCpu The cross context virtual CPU structure.
5886 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5887 * out-of-sync. Make sure to update the required fields
5888 * before using them.
5889 */
5890DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5891{
5892 NOREF(pMixedCtx);
5893 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5894 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5895 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5896 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5897}
5898
5899
5900/**
5901 * Handle a condition that occurred while delivering an event through the guest
5902 * IDT.
5903 *
5904 * @returns Strict VBox status code (i.e. informational status codes too).
5905 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5906 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5907 * to continue execution of the guest which will delivery the \#DF.
5908 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5909 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5910 *
5911 * @param pVCpu The cross context virtual CPU structure.
5912 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5913 * out-of-sync. Make sure to update the required fields
5914 * before using them.
5915 * @param pVmxTransient Pointer to the VMX transient structure.
5916 *
5917 * @remarks No-long-jump zone!!!
5918 */
5919static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5920{
5921 uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5922
5923 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5924 rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5925
5926 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5927 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5928 {
5929 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5930 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5931#ifdef HMVMX_USE_IEM_EVENT_REFLECTION
5932 /*
5933 * If the event was a software interrupt (generated with INT n) or a software exception (generated
5934 * by INT3/INTO) or a privileged software exception (generated by INT1), we can handle the VM-exit
5935 * and continue guest execution which will re-execute the instruction rather than re-injecting the
5936 * exception, as that can cause premature trips to ring-3 before injection and involve TRPM which
5937 * currently has no way of storing that these exceptions were caused by these instructions
5938 * (ICEBP's #DB poses the problem).
5939 */
5940 IEMXCPTRAISE enmRaise;
5941 IEMXCPTRAISEINFO fRaiseInfo;
5942 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5943 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5944 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5945 {
5946 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5947 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5948 }
5949 else if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5950 {
5951 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5952 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5953 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5954 /** @todo Make AssertMsgReturn as just AssertMsg later. */
5955 AssertMsgReturn(uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT,
5956 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5957 uExitVectorType), VERR_VMX_IPE_5);
5958 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5959
5960 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5961 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
5962 {
5963 pVmxTransient->fVectoringPF = true;
5964 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5965 }
5966 }
5967 else
5968 {
5969 /*
5970 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5971 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5972 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
5973 */
5974 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5975 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5976 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5977 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5978 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5979 }
5980
5981 /*
5982 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5983 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5984 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5985 * subsequent VM-entry would fail.
5986 *
5987 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5988 */
5989 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5990 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5991 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
5992 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
5993 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5994 {
5995 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5996 }
5997
5998 switch (enmRaise)
5999 {
6000 case IEMXCPTRAISE_CURRENT_XCPT:
6001 {
6002 Log4(("IDT: vcpu[%RU32] Pending secondary xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n", pVCpu->idCpu,
6003 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
6004 Assert(rcStrict == VINF_SUCCESS);
6005 break;
6006 }
6007
6008 case IEMXCPTRAISE_PREV_EVENT:
6009 {
6010 uint32_t u32ErrCode;
6011 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6012 {
6013 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6014 AssertRCReturn(rc2, rc2);
6015 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6016 }
6017 else
6018 u32ErrCode = 0;
6019
6020 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
6021 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6022 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6023 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
6024
6025 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
6026 pVCpu->hm.s.Event.u32ErrCode));
6027 Assert(rcStrict == VINF_SUCCESS);
6028 break;
6029 }
6030
6031 case IEMXCPTRAISE_REEXEC_INSTR:
6032 Assert(rcStrict == VINF_SUCCESS);
6033 break;
6034
6035 case IEMXCPTRAISE_DOUBLE_FAULT:
6036 {
6037 /*
6038 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
6039 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
6040 */
6041 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6042 {
6043 pVmxTransient->fVectoringDoublePF = true;
6044 Log4(("IDT: vcpu[%RU32] Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
6045 pMixedCtx->cr2));
6046 rcStrict = VINF_SUCCESS;
6047 }
6048 else
6049 {
6050 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6051 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6052 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
6053 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
6054 rcStrict = VINF_HM_DOUBLE_FAULT;
6055 }
6056 break;
6057 }
6058
6059 case IEMXCPTRAISE_TRIPLE_FAULT:
6060 {
6061 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6062 uExitVector));
6063 rcStrict = VINF_EM_RESET;
6064 break;
6065 }
6066
6067 case IEMXCPTRAISE_CPU_HANG:
6068 {
6069 Log4(("IDT: vcpu[%RU32] Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", pVCpu->idCpu, fRaiseInfo));
6070 rcStrict = VERR_EM_GUEST_CPU_HANG;
6071 break;
6072 }
6073
6074 default:
6075 {
6076 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6077 rcStrict = VERR_VMX_IPE_2;
6078 break;
6079 }
6080 }
6081#else
6082 typedef enum
6083 {
6084 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
6085 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
6086 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
6087 VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
6088 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
6089 } VMXREFLECTXCPT;
6090
6091 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
6092 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
6093 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
6094 {
6095 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
6096 {
6097 enmReflect = VMXREFLECTXCPT_XCPT;
6098#ifdef VBOX_STRICT
6099 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
6100 && uExitVector == X86_XCPT_PF)
6101 {
6102 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6103 }
6104#endif
6105 if ( uExitVector == X86_XCPT_PF
6106 && uIdtVector == X86_XCPT_PF)
6107 {
6108 pVmxTransient->fVectoringDoublePF = true;
6109 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6110 }
6111 else if ( uExitVector == X86_XCPT_AC
6112 && uIdtVector == X86_XCPT_AC)
6113 {
6114 enmReflect = VMXREFLECTXCPT_HANG;
6115 Log4(("IDT: Nested #AC - Bad guest\n"));
6116 }
6117 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
6118 && hmR0VmxIsContributoryXcpt(uExitVector)
6119 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
6120 || uIdtVector == X86_XCPT_PF))
6121 {
6122 enmReflect = VMXREFLECTXCPT_DF;
6123 }
6124 else if (uIdtVector == X86_XCPT_DF)
6125 enmReflect = VMXREFLECTXCPT_TF;
6126 }
6127 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
6128 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
6129 {
6130 /*
6131 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
6132 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
6133 */
6134 enmReflect = VMXREFLECTXCPT_XCPT;
6135
6136 if (uExitVector == X86_XCPT_PF)
6137 {
6138 pVmxTransient->fVectoringPF = true;
6139 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6140 }
6141 }
6142 }
6143 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6144 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
6145 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
6146 {
6147 /*
6148 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
6149 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
6150 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
6151 */
6152 enmReflect = VMXREFLECTXCPT_XCPT;
6153 }
6154
6155 /*
6156 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
6157 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
6158 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
6159 *
6160 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
6161 */
6162 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6163 && enmReflect == VMXREFLECTXCPT_XCPT
6164 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
6165 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6166 {
6167 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6168 }
6169
6170 switch (enmReflect)
6171 {
6172 case VMXREFLECTXCPT_XCPT:
6173 {
6174 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6175 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6176 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
6177
6178 uint32_t u32ErrCode = 0;
6179 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6180 {
6181 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6182 AssertRCReturn(rc2, rc2);
6183 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6184 }
6185
6186 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
6187 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6188 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6189 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
6190 rcStrict = VINF_SUCCESS;
6191 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
6192 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
6193
6194 break;
6195 }
6196
6197 case VMXREFLECTXCPT_DF:
6198 {
6199 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6200 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6201 rcStrict = VINF_HM_DOUBLE_FAULT;
6202 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
6203 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
6204
6205 break;
6206 }
6207
6208 case VMXREFLECTXCPT_TF:
6209 {
6210 rcStrict = VINF_EM_RESET;
6211 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6212 uExitVector));
6213 break;
6214 }
6215
6216 case VMXREFLECTXCPT_HANG:
6217 {
6218 rcStrict = VERR_EM_GUEST_CPU_HANG;
6219 break;
6220 }
6221
6222 default:
6223 Assert(rcStrict == VINF_SUCCESS);
6224 break;
6225 }
6226#endif /* HMVMX_USE_IEM_EVENT_REFLECTION */
6227 }
6228 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6229 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6230 && uExitVector != X86_XCPT_DF
6231 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6232 {
6233 /*
6234 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6235 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6236 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6237 */
6238 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6239 {
6240 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
6241 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6242 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6243 }
6244 }
6245
6246 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
6247 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
6248 return rcStrict;
6249}
6250
6251
6252/**
6253 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
6254 *
6255 * @returns VBox status code.
6256 * @param pVCpu The cross context virtual CPU structure.
6257 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6258 * out-of-sync. Make sure to update the required fields
6259 * before using them.
6260 *
6261 * @remarks No-long-jump zone!!!
6262 */
6263static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6264{
6265 NOREF(pMixedCtx);
6266
6267 /*
6268 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
6269 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
6270 */
6271 VMMRZCallRing3Disable(pVCpu);
6272 HM_DISABLE_PREEMPT();
6273
6274 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
6275 {
6276#ifndef DEBUG_bird /** @todo this triggers running bs3-cpu-generated-1.img with --debug-command-line
6277 * and 'dbgc-init' containing:
6278 * sxe "xcpt_de"
6279 * sxe "xcpt_bp"
6280 * sxi "xcpt_gp"
6281 * sxi "xcpt_ss"
6282 * sxi "xcpt_np"
6283 */
6284 /** @todo r=ramshankar: Should be fixed after r119291. */
6285 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
6286#endif
6287 uint32_t uVal = 0;
6288 uint32_t uShadow = 0;
6289 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
6290 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
6291 AssertRCReturn(rc, rc);
6292
6293 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
6294 CPUMSetGuestCR0(pVCpu, uVal);
6295 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
6296 }
6297
6298 HM_RESTORE_PREEMPT();
6299 VMMRZCallRing3Enable(pVCpu);
6300 return VINF_SUCCESS;
6301}
6302
6303
6304/**
6305 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
6306 *
6307 * @returns VBox status code.
6308 * @param pVCpu The cross context virtual CPU structure.
6309 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6310 * out-of-sync. Make sure to update the required fields
6311 * before using them.
6312 *
6313 * @remarks No-long-jump zone!!!
6314 */
6315static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6316{
6317 NOREF(pMixedCtx);
6318
6319 int rc = VINF_SUCCESS;
6320 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
6321 {
6322 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4));
6323 uint32_t uVal = 0;
6324 uint32_t uShadow = 0;
6325 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
6326 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
6327 AssertRCReturn(rc, rc);
6328
6329 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
6330 CPUMSetGuestCR4(pVCpu, uVal);
6331 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6332 }
6333 return rc;
6334}
6335
6336
6337/**
6338 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
6339 *
6340 * @returns VBox status code.
6341 * @param pVCpu The cross context virtual CPU structure.
6342 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6343 * out-of-sync. Make sure to update the required fields
6344 * before using them.
6345 *
6346 * @remarks No-long-jump zone!!!
6347 */
6348static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6349{
6350 int rc = VINF_SUCCESS;
6351 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6352 {
6353 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP));
6354 uint64_t u64Val = 0;
6355 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6356 AssertRCReturn(rc, rc);
6357
6358 pMixedCtx->rip = u64Val;
6359 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6360 }
6361 return rc;
6362}
6363
6364
6365/**
6366 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6367 *
6368 * @returns VBox status code.
6369 * @param pVCpu The cross context virtual CPU structure.
6370 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6371 * out-of-sync. Make sure to update the required fields
6372 * before using them.
6373 *
6374 * @remarks No-long-jump zone!!!
6375 */
6376static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6377{
6378 int rc = VINF_SUCCESS;
6379 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6380 {
6381 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP));
6382 uint64_t u64Val = 0;
6383 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6384 AssertRCReturn(rc, rc);
6385
6386 pMixedCtx->rsp = u64Val;
6387 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6388 }
6389 return rc;
6390}
6391
6392
6393/**
6394 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6395 *
6396 * @returns VBox status code.
6397 * @param pVCpu The cross context virtual CPU structure.
6398 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6399 * out-of-sync. Make sure to update the required fields
6400 * before using them.
6401 *
6402 * @remarks No-long-jump zone!!!
6403 */
6404static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6405{
6406 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6407 {
6408 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS));
6409 uint32_t uVal = 0;
6410 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6411 AssertRCReturn(rc, rc);
6412
6413 pMixedCtx->eflags.u32 = uVal;
6414 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6415 {
6416 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6417 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6418
6419 pMixedCtx->eflags.Bits.u1VM = 0;
6420 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6421 }
6422
6423 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6424 }
6425 return VINF_SUCCESS;
6426}
6427
6428
6429/**
6430 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6431 * guest-CPU context.
6432 */
6433DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6434{
6435 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6436 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6437 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6438 return rc;
6439}
6440
6441
6442/**
6443 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6444 * from the guest-state area in the VMCS.
6445 *
6446 * @param pVCpu The cross context virtual CPU structure.
6447 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6448 * out-of-sync. Make sure to update the required fields
6449 * before using them.
6450 *
6451 * @remarks No-long-jump zone!!!
6452 */
6453static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6454{
6455 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6456 {
6457 uint32_t uIntrState = 0;
6458 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6459 AssertRC(rc);
6460
6461 if (!uIntrState)
6462 {
6463 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6464 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6465
6466 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6467 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6468 }
6469 else
6470 {
6471 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6472 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6473 {
6474 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6475 AssertRC(rc);
6476 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6477 AssertRC(rc);
6478
6479 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6480 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6481 }
6482 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6483 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6484
6485 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6486 {
6487 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6488 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6489 }
6490 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6491 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6492 }
6493
6494 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6495 }
6496}
6497
6498
6499/**
6500 * Saves the guest's activity state.
6501 *
6502 * @returns VBox status code.
6503 * @param pVCpu The cross context virtual CPU structure.
6504 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6505 * out-of-sync. Make sure to update the required fields
6506 * before using them.
6507 *
6508 * @remarks No-long-jump zone!!!
6509 */
6510static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6511{
6512 NOREF(pMixedCtx);
6513 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6514 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6515 return VINF_SUCCESS;
6516}
6517
6518
6519/**
6520 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6521 * the current VMCS into the guest-CPU context.
6522 *
6523 * @returns VBox status code.
6524 * @param pVCpu The cross context virtual CPU structure.
6525 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6526 * out-of-sync. Make sure to update the required fields
6527 * before using them.
6528 *
6529 * @remarks No-long-jump zone!!!
6530 */
6531static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6532{
6533 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6534 {
6535 Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR));
6536 uint32_t u32Val = 0;
6537 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6538 pMixedCtx->SysEnter.cs = u32Val;
6539 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6540 }
6541
6542 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6543 {
6544 Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR));
6545 uint64_t u64Val = 0;
6546 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6547 pMixedCtx->SysEnter.eip = u64Val;
6548 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6549 }
6550 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6551 {
6552 Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR));
6553 uint64_t u64Val = 0;
6554 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6555 pMixedCtx->SysEnter.esp = u64Val;
6556 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6557 }
6558 return VINF_SUCCESS;
6559}
6560
6561
6562/**
6563 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6564 * the CPU back into the guest-CPU context.
6565 *
6566 * @returns VBox status code.
6567 * @param pVCpu The cross context virtual CPU structure.
6568 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6569 * out-of-sync. Make sure to update the required fields
6570 * before using them.
6571 *
6572 * @remarks No-long-jump zone!!!
6573 */
6574static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6575{
6576 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6577 VMMRZCallRing3Disable(pVCpu);
6578 HM_DISABLE_PREEMPT();
6579
6580 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6581 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6582 {
6583 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMM_GUEST_LAZY_MSRS));
6584 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6585 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6586 }
6587
6588 HM_RESTORE_PREEMPT();
6589 VMMRZCallRing3Enable(pVCpu);
6590
6591 return VINF_SUCCESS;
6592}
6593
6594
6595/**
6596 * Saves the auto load/store'd guest MSRs from the current VMCS into
6597 * the guest-CPU context.
6598 *
6599 * @returns VBox status code.
6600 * @param pVCpu The cross context virtual CPU structure.
6601 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6602 * out-of-sync. Make sure to update the required fields
6603 * before using them.
6604 *
6605 * @remarks No-long-jump zone!!!
6606 */
6607static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6608{
6609 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6610 return VINF_SUCCESS;
6611
6612 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS));
6613 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6614 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6615 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6616 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6617 {
6618 switch (pMsr->u32Msr)
6619 {
6620 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6621 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6622 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6623 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6624 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6625 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6626 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6627 break;
6628
6629 default:
6630 {
6631 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6632 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6633 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6634 }
6635 }
6636 }
6637
6638 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6639 return VINF_SUCCESS;
6640}
6641
6642
6643/**
6644 * Saves the guest control registers from the current VMCS into the guest-CPU
6645 * context.
6646 *
6647 * @returns VBox status code.
6648 * @param pVCpu The cross context virtual CPU structure.
6649 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6650 * out-of-sync. Make sure to update the required fields
6651 * before using them.
6652 *
6653 * @remarks No-long-jump zone!!!
6654 */
6655static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6656{
6657 /* Guest CR0. Guest FPU. */
6658 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6659 AssertRCReturn(rc, rc);
6660
6661 /* Guest CR4. */
6662 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6663 AssertRCReturn(rc, rc);
6664
6665 /* Guest CR2 - updated always during the world-switch or in #PF. */
6666 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6667 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6668 {
6669 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3));
6670 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6671 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6672
6673 PVM pVM = pVCpu->CTX_SUFF(pVM);
6674 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6675 || ( pVM->hm.s.fNestedPaging
6676 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6677 {
6678 uint64_t u64Val = 0;
6679 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6680 if (pMixedCtx->cr3 != u64Val)
6681 {
6682 CPUMSetGuestCR3(pVCpu, u64Val);
6683 if (VMMRZCallRing3IsEnabled(pVCpu))
6684 {
6685 PGMUpdateCR3(pVCpu, u64Val);
6686 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6687 }
6688 else
6689 {
6690 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6691 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6692 }
6693 }
6694
6695 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6696 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6697 {
6698 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6699 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6700 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6701 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6702 AssertRCReturn(rc, rc);
6703
6704 if (VMMRZCallRing3IsEnabled(pVCpu))
6705 {
6706 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6707 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6708 }
6709 else
6710 {
6711 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6712 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6713 }
6714 }
6715 }
6716
6717 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6718 }
6719
6720 /*
6721 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6722 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6723 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6724 *
6725 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6726 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6727 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6728 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6729 *
6730 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6731 */
6732 if (VMMRZCallRing3IsEnabled(pVCpu))
6733 {
6734 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6735 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6736
6737 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6738 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6739
6740 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6741 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6742 }
6743
6744 return rc;
6745}
6746
6747
6748/**
6749 * Reads a guest segment register from the current VMCS into the guest-CPU
6750 * context.
6751 *
6752 * @returns VBox status code.
6753 * @param pVCpu The cross context virtual CPU structure.
6754 * @param idxSel Index of the selector in the VMCS.
6755 * @param idxLimit Index of the segment limit in the VMCS.
6756 * @param idxBase Index of the segment base in the VMCS.
6757 * @param idxAccess Index of the access rights of the segment in the VMCS.
6758 * @param pSelReg Pointer to the segment selector.
6759 *
6760 * @remarks No-long-jump zone!!!
6761 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6762 * macro as that takes care of whether to read from the VMCS cache or
6763 * not.
6764 */
6765DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6766 PCPUMSELREG pSelReg)
6767{
6768 NOREF(pVCpu);
6769
6770 uint32_t u32Val = 0;
6771 int rc = VMXReadVmcs32(idxSel, &u32Val);
6772 AssertRCReturn(rc, rc);
6773 pSelReg->Sel = (uint16_t)u32Val;
6774 pSelReg->ValidSel = (uint16_t)u32Val;
6775 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6776
6777 rc = VMXReadVmcs32(idxLimit, &u32Val);
6778 AssertRCReturn(rc, rc);
6779 pSelReg->u32Limit = u32Val;
6780
6781 uint64_t u64Val = 0;
6782 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6783 AssertRCReturn(rc, rc);
6784 pSelReg->u64Base = u64Val;
6785
6786 rc = VMXReadVmcs32(idxAccess, &u32Val);
6787 AssertRCReturn(rc, rc);
6788 pSelReg->Attr.u = u32Val;
6789
6790 /*
6791 * If VT-x marks the segment as unusable, most other bits remain undefined:
6792 * - For CS the L, D and G bits have meaning.
6793 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6794 * - For the remaining data segments no bits are defined.
6795 *
6796 * The present bit and the unusable bit has been observed to be set at the
6797 * same time (the selector was supposed to be invalid as we started executing
6798 * a V8086 interrupt in ring-0).
6799 *
6800 * What should be important for the rest of the VBox code, is that the P bit is
6801 * cleared. Some of the other VBox code recognizes the unusable bit, but
6802 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6803 * safe side here, we'll strip off P and other bits we don't care about. If
6804 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6805 *
6806 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6807 */
6808 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6809 {
6810 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6811
6812 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6813 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6814 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6815
6816 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6817#ifdef DEBUG_bird
6818 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6819 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6820 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6821#endif
6822 }
6823 return VINF_SUCCESS;
6824}
6825
6826
6827#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6828# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6829 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6830 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6831#else
6832# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6833 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6834 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6835#endif
6836
6837
6838/**
6839 * Saves the guest segment registers from the current VMCS into the guest-CPU
6840 * context.
6841 *
6842 * @returns VBox status code.
6843 * @param pVCpu The cross context virtual CPU structure.
6844 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6845 * out-of-sync. Make sure to update the required fields
6846 * before using them.
6847 *
6848 * @remarks No-long-jump zone!!!
6849 */
6850static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6851{
6852 /* Guest segment registers. */
6853 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6854 {
6855 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS));
6856 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6857 AssertRCReturn(rc, rc);
6858
6859 rc = VMXLOCAL_READ_SEG(CS, cs);
6860 rc |= VMXLOCAL_READ_SEG(SS, ss);
6861 rc |= VMXLOCAL_READ_SEG(DS, ds);
6862 rc |= VMXLOCAL_READ_SEG(ES, es);
6863 rc |= VMXLOCAL_READ_SEG(FS, fs);
6864 rc |= VMXLOCAL_READ_SEG(GS, gs);
6865 AssertRCReturn(rc, rc);
6866
6867 /* Restore segment attributes for real-on-v86 mode hack. */
6868 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6869 {
6870 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6871 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6872 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6873 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6874 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6875 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6876 }
6877 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6878 }
6879
6880 return VINF_SUCCESS;
6881}
6882
6883
6884/**
6885 * Saves the guest SS register from the current VMCS into the guest-CPU context.
6886 *
6887 * @returns VBox status code.
6888 * @param pVCpu The cross context virtual CPU structure.
6889 * @remarks No-long-jump zone!!!
6890 */
6891static int hmR0VmxSaveGuestCs(PVMCPU pVCpu)
6892{
6893 /** @todo optimize this? */
6894 return hmR0VmxSaveGuestSegmentRegs(pVCpu, &pVCpu->cpum.GstCtx);
6895}
6896
6897
6898/**
6899 * Saves the guest descriptor table registers and task register from the current
6900 * VMCS into the guest-CPU context.
6901 *
6902 * @returns VBox status code.
6903 * @param pVCpu The cross context virtual CPU structure.
6904 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6905 * out-of-sync. Make sure to update the required fields
6906 * before using them.
6907 *
6908 * @remarks No-long-jump zone!!!
6909 */
6910static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6911{
6912 int rc = VINF_SUCCESS;
6913
6914 /* Guest LDTR. */
6915 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6916 {
6917 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR));
6918 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6919 AssertRCReturn(rc, rc);
6920 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6921 }
6922
6923 /* Guest GDTR. */
6924 uint64_t u64Val = 0;
6925 uint32_t u32Val = 0;
6926 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6927 {
6928 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR));
6929 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6930 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6931 pMixedCtx->gdtr.pGdt = u64Val;
6932 pMixedCtx->gdtr.cbGdt = u32Val;
6933 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6934 }
6935
6936 /* Guest IDTR. */
6937 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6938 {
6939 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR));
6940 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6941 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6942 pMixedCtx->idtr.pIdt = u64Val;
6943 pMixedCtx->idtr.cbIdt = u32Val;
6944 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6945 }
6946
6947 /* Guest TR. */
6948 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6949 {
6950 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR));
6951 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6952 AssertRCReturn(rc, rc);
6953
6954 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6955 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6956 {
6957 rc = VMXLOCAL_READ_SEG(TR, tr);
6958 AssertRCReturn(rc, rc);
6959 }
6960 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6961 }
6962 return rc;
6963}
6964
6965#undef VMXLOCAL_READ_SEG
6966
6967
6968/**
6969 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6970 * context.
6971 *
6972 * @returns VBox status code.
6973 * @param pVCpu The cross context virtual CPU structure.
6974 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6975 * out-of-sync. Make sure to update the required fields
6976 * before using them.
6977 *
6978 * @remarks No-long-jump zone!!!
6979 */
6980static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6981{
6982 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DR7))
6983 {
6984 if (!pVCpu->hm.s.fUsingHyperDR7)
6985 {
6986 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6987 uint32_t u32Val;
6988 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6989 pMixedCtx->dr[7] = u32Val;
6990 }
6991
6992 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DR7);
6993 }
6994 return VINF_SUCCESS;
6995}
6996
6997
6998/**
6999 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
7000 *
7001 * @returns VBox status code.
7002 * @param pVCpu The cross context virtual CPU structure.
7003 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7004 * out-of-sync. Make sure to update the required fields
7005 * before using them.
7006 *
7007 * @remarks No-long-jump zone!!!
7008 */
7009static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7010{
7011 NOREF(pMixedCtx);
7012
7013 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
7014 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
7015 return VINF_SUCCESS;
7016}
7017
7018
7019/**
7020 * Saves the entire guest state from the currently active VMCS into the
7021 * guest-CPU context.
7022 *
7023 * This essentially VMREADs all guest-data.
7024 *
7025 * @returns VBox status code.
7026 * @param pVCpu The cross context virtual CPU structure.
7027 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7028 * out-of-sync. Make sure to update the required fields
7029 * before using them.
7030 */
7031static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7032{
7033 Assert(pVCpu);
7034 Assert(pMixedCtx);
7035
7036 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
7037 return VINF_SUCCESS;
7038
7039 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
7040 again on the ring-3 callback path, there is no real need to. */
7041 if (VMMRZCallRing3IsEnabled(pVCpu))
7042 VMMR0LogFlushDisable(pVCpu);
7043 else
7044 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7045 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
7046
7047 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7048 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7049
7050 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7051 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7052
7053 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7054 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7055
7056 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7057 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7058
7059 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
7060 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7061
7062 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
7063 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7064
7065 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7066 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7067
7068 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7069 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7070
7071 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
7072 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7073
7074 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
7075 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7076
7077 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
7078 ("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
7079 HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
7080
7081 if (VMMRZCallRing3IsEnabled(pVCpu))
7082 VMMR0LogFlushEnable(pVCpu);
7083
7084 return VINF_SUCCESS;
7085}
7086
7087
7088/**
7089 * Saves basic guest registers needed for IEM instruction execution.
7090 *
7091 * @returns VBox status code (OR-able).
7092 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
7093 * @param pMixedCtx Pointer to the CPU context of the guest.
7094 * @param fMemory Whether the instruction being executed operates on
7095 * memory or not. Only CR0 is synced up if clear.
7096 * @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
7097 */
7098static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
7099{
7100 /*
7101 * We assume all general purpose registers other than RSP are available.
7102 *
7103 * - RIP is a must, as it will be incremented or otherwise changed.
7104 * - RFLAGS are always required to figure the CPL.
7105 * - RSP isn't always required, however it's a GPR, so frequently required.
7106 * - SS and CS are the only segment register needed if IEM doesn't do memory
7107 * access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
7108 * - CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
7109 * be required for memory accesses.
7110 *
7111 * Note! Before IEM dispatches an exception, it will call us to sync in everything.
7112 */
7113 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7114 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7115 if (fNeedRsp)
7116 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
7117 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /** @todo Only CS and SS are required here. */
7118 if (!fMemory)
7119 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7120 else
7121 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7122 AssertRCReturn(rc, rc);
7123 return rc;
7124}
7125
7126
7127/**
7128 * Saves guest registers needed for IEM instruction interpretation.
7129 *
7130 * @returns VBox status code (OR-able).
7131 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
7132 */
7133static int hmR0VmxSaveGuestRegsForIemInterpreting(PVMCPU pVCpu)
7134{
7135 /*
7136 * Our goal here is IEM_CPUMCTX_EXTRN_MUST_MASK.
7137 *
7138 * Note! Before IEM dispatches an exception, it will call us to sync in everything.
7139 */
7140#if 0 /* later with CPUMCTX_EXTRN_XXX */
7141 int rc = hmR0VmxSaveGuestRip(pVCpu, &pVCpu->cpum.GstCtx);
7142 rc |= hmR0VmxSaveGuestRflags(pVCpu, &pVCpu->cpum.GstCtx);
7143 rc |= hmR0VmxSaveGuestRsp(pVCpu, &pVCpu->cpum.GstCtx);
7144 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, &pVCpu->cpum.GstCtx); /** @todo Only CS and SS are strictly required here. */
7145 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, &pVCpu->cpum.GstCtx); /** @todo We don't need CR2 here. */
7146 rc |= hmR0VmxSaveGuestApicState(pVCpu, &pVCpu->cpum.GstCtx); /** @todo Only TPR is needed here. */
7147 rc |= hmR0VmxSaveGuestDR7(pVCpu, &pVCpu->cpum.GstCtx);
7148 /* EFER is always up to date. */
7149 AssertRCReturn(rc, rc);
7150 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST - fixme); /** @todo fix me */
7151#else
7152 int rc = hmR0VmxSaveGuestState(pVCpu, &pVCpu->cpum.GstCtx);
7153 AssertRCReturn(rc, rc);
7154 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7155#endif
7156
7157 return rc;
7158}
7159
7160
7161/**
7162 * Ensures that we've got a complete basic guest-context.
7163 *
7164 * This excludes the FPU, SSE, AVX, and similar extended state. The interface
7165 * is for the interpreter.
7166 *
7167 * @returns VBox status code.
7168 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
7169 * @param pMixedCtx Pointer to the guest-CPU context which may have data
7170 * needing to be synced in.
7171 * @thread EMT(pVCpu)
7172 */
7173VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7174{
7175 /* Note! Since this is only applicable to VT-x, the implementation is placed
7176 in the VT-x part of the sources instead of the generic stuff. */
7177 int rc;
7178 PVM pVM = pVCpu->CTX_SUFF(pVM);
7179 if ( pVM->hm.s.vmx.fSupported
7180 && VM_IS_HM_ENABLED(pVM))
7181 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7182 else
7183 rc = VINF_SUCCESS;
7184
7185 /*
7186 * For now, imply that the caller might change everything too. Do this after
7187 * saving the guest state so as to not trigger assertions.
7188 *
7189 * This is required for AMD-V too as it too only selectively re-loads changed
7190 * guest state back in to the VMCB.
7191 */
7192 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7193 return rc;
7194}
7195
7196
7197/**
7198 * Check per-VM and per-VCPU force flag actions that require us to go back to
7199 * ring-3 for one reason or another.
7200 *
7201 * @returns Strict VBox status code (i.e. informational status codes too)
7202 * @retval VINF_SUCCESS if we don't have any actions that require going back to
7203 * ring-3.
7204 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7205 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7206 * interrupts)
7207 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7208 * all EMTs to be in ring-3.
7209 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7210 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7211 * to the EM loop.
7212 *
7213 * @param pVM The cross context VM structure.
7214 * @param pVCpu The cross context virtual CPU structure.
7215 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7216 * out-of-sync. Make sure to update the required fields
7217 * before using them.
7218 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
7219 */
7220static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
7221{
7222 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7223
7224 /*
7225 * Anything pending? Should be more likely than not if we're doing a good job.
7226 */
7227 if ( !fStepping
7228 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7229 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7230 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7231 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7232 return VINF_SUCCESS;
7233
7234 /* We need the control registers now, make sure the guest-CPU context is updated. */
7235 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7236 AssertRCReturn(rc3, rc3);
7237
7238 /* Pending HM CR3 sync. */
7239 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7240 {
7241 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
7242 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
7243 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
7244 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7245 }
7246
7247 /* Pending HM PAE PDPEs. */
7248 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7249 {
7250 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7251 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7252 }
7253
7254 /* Pending PGM C3 sync. */
7255 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7256 {
7257 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
7258 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7259 if (rcStrict2 != VINF_SUCCESS)
7260 {
7261 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7262 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7263 return rcStrict2;
7264 }
7265 }
7266
7267 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7268 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
7269 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7270 {
7271 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7272 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
7273 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7274 return rc2;
7275 }
7276
7277 /* Pending VM request packets, such as hardware interrupts. */
7278 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
7279 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
7280 {
7281 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
7282 return VINF_EM_PENDING_REQUEST;
7283 }
7284
7285 /* Pending PGM pool flushes. */
7286 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7287 {
7288 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
7289 return VINF_PGM_POOL_FLUSH_PENDING;
7290 }
7291
7292 /* Pending DMA requests. */
7293 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
7294 {
7295 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
7296 return VINF_EM_RAW_TO_R3;
7297 }
7298
7299 return VINF_SUCCESS;
7300}
7301
7302
7303/**
7304 * Converts any TRPM trap into a pending HM event. This is typically used when
7305 * entering from ring-3 (not longjmp returns).
7306 *
7307 * @param pVCpu The cross context virtual CPU structure.
7308 */
7309static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7310{
7311 Assert(TRPMHasTrap(pVCpu));
7312 Assert(!pVCpu->hm.s.Event.fPending);
7313
7314 uint8_t uVector;
7315 TRPMEVENT enmTrpmEvent;
7316 RTGCUINT uErrCode;
7317 RTGCUINTPTR GCPtrFaultAddress;
7318 uint8_t cbInstr;
7319
7320 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7321 AssertRC(rc);
7322
7323 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7324 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7325 if (enmTrpmEvent == TRPM_TRAP)
7326 {
7327 switch (uVector)
7328 {
7329 case X86_XCPT_NMI:
7330 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7331 break;
7332
7333 case X86_XCPT_BP:
7334 case X86_XCPT_OF:
7335 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7336 break;
7337
7338 case X86_XCPT_PF:
7339 case X86_XCPT_DF:
7340 case X86_XCPT_TS:
7341 case X86_XCPT_NP:
7342 case X86_XCPT_SS:
7343 case X86_XCPT_GP:
7344 case X86_XCPT_AC:
7345 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7346 RT_FALL_THRU();
7347 default:
7348 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7349 break;
7350 }
7351 }
7352 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7353 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7354 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7355 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7356 else
7357 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7358
7359 rc = TRPMResetTrap(pVCpu);
7360 AssertRC(rc);
7361 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7362 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7363
7364 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7365}
7366
7367
7368/**
7369 * Converts the pending HM event into a TRPM trap.
7370 *
7371 * @param pVCpu The cross context virtual CPU structure.
7372 */
7373static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7374{
7375 Assert(pVCpu->hm.s.Event.fPending);
7376
7377 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7378 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7379 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
7380 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7381
7382 /* If a trap was already pending, we did something wrong! */
7383 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7384
7385 TRPMEVENT enmTrapType;
7386 switch (uVectorType)
7387 {
7388 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7389 enmTrapType = TRPM_HARDWARE_INT;
7390 break;
7391
7392 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7393 enmTrapType = TRPM_SOFTWARE_INT;
7394 break;
7395
7396 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7397 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7398 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7399 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7400 enmTrapType = TRPM_TRAP;
7401 break;
7402
7403 default:
7404 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7405 enmTrapType = TRPM_32BIT_HACK;
7406 break;
7407 }
7408
7409 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7410
7411 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7412 AssertRC(rc);
7413
7414 if (fErrorCodeValid)
7415 TRPMSetErrorCode(pVCpu, uErrorCode);
7416
7417 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7418 && uVector == X86_XCPT_PF)
7419 {
7420 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7421 }
7422 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7423 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7424 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7425 {
7426 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7427 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
7428 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7429 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7430 }
7431
7432 /* Clear any pending events from the VMCS. */
7433 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
7434 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
7435
7436 /* We're now done converting the pending event. */
7437 pVCpu->hm.s.Event.fPending = false;
7438}
7439
7440
7441/**
7442 * Does the necessary state syncing before returning to ring-3 for any reason
7443 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7444 *
7445 * @returns VBox status code.
7446 * @param pVCpu The cross context virtual CPU structure.
7447 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7448 * be out-of-sync. Make sure to update the required
7449 * fields before using them.
7450 * @param fSaveGuestState Whether to save the guest state or not.
7451 *
7452 * @remarks No-long-jmp zone!!!
7453 */
7454static int hmR0VmxLeave(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7455{
7456 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7457 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7458
7459 RTCPUID idCpu = RTMpCpuId();
7460 Log4Func(("HostCpuId=%u\n", idCpu));
7461
7462 /*
7463 * !!! IMPORTANT !!!
7464 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7465 */
7466
7467 /* Save the guest state if necessary. */
7468 if ( fSaveGuestState
7469 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7470 {
7471 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7472 AssertRCReturn(rc, rc);
7473 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7474 }
7475
7476 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
7477 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7478 {
7479 /* We shouldn't reload CR0 without saving it first. */
7480 if (!fSaveGuestState)
7481 {
7482 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7483 AssertRCReturn(rc, rc);
7484 }
7485 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7486 }
7487
7488 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7489#ifdef VBOX_STRICT
7490 if (CPUMIsHyperDebugStateActive(pVCpu))
7491 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7492#endif
7493 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7494 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7495 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7496 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7497
7498#if HC_ARCH_BITS == 64
7499 /* Restore host-state bits that VT-x only restores partially. */
7500 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7501 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7502 {
7503 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7504 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7505 }
7506 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7507#endif
7508
7509 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7510 if (pVCpu->hm.s.vmx.fLazyMsrs)
7511 {
7512 /* We shouldn't reload the guest MSRs without saving it first. */
7513 if (!fSaveGuestState)
7514 {
7515 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7516 AssertRCReturn(rc, rc);
7517 }
7518 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7519 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7520 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7521 }
7522
7523 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7524 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7525
7526 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7527 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7528 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7529 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7530 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7531 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7532 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7533 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7534
7535 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7536
7537 /** @todo This partially defeats the purpose of having preemption hooks.
7538 * The problem is, deregistering the hooks should be moved to a place that
7539 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7540 * context.
7541 */
7542 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7543 {
7544 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7545 AssertRCReturn(rc, rc);
7546
7547 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7548 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7549 }
7550 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7551 NOREF(idCpu);
7552
7553 return VINF_SUCCESS;
7554}
7555
7556
7557/**
7558 * Leaves the VT-x session.
7559 *
7560 * @returns VBox status code.
7561 * @param pVCpu The cross context virtual CPU structure.
7562 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7563 * out-of-sync. Make sure to update the required fields
7564 * before using them.
7565 *
7566 * @remarks No-long-jmp zone!!!
7567 */
7568DECLINLINE(int) hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7569{
7570 HM_DISABLE_PREEMPT();
7571 HMVMX_ASSERT_CPU_SAFE();
7572 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7573 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7574
7575 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7576 and done this from the VMXR0ThreadCtxCallback(). */
7577 if (!pVCpu->hm.s.fLeaveDone)
7578 {
7579 int rc2 = hmR0VmxLeave(pVCpu, pMixedCtx, true /* fSaveGuestState */);
7580 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7581 pVCpu->hm.s.fLeaveDone = true;
7582 }
7583 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7584
7585 /*
7586 * !!! IMPORTANT !!!
7587 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7588 */
7589
7590 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7591 /** @todo Deregistering here means we need to VMCLEAR always
7592 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7593 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7594 VMMR0ThreadCtxHookDisable(pVCpu);
7595
7596 /* Leave HM context. This takes care of local init (term). */
7597 int rc = HMR0LeaveCpu(pVCpu);
7598
7599 HM_RESTORE_PREEMPT();
7600 return rc;
7601}
7602
7603
7604/**
7605 * Does the necessary state syncing before doing a longjmp to ring-3.
7606 *
7607 * @returns VBox status code.
7608 * @param pVCpu The cross context virtual CPU structure.
7609 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7610 * out-of-sync. Make sure to update the required fields
7611 * before using them.
7612 *
7613 * @remarks No-long-jmp zone!!!
7614 */
7615DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7616{
7617 return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7618}
7619
7620
7621/**
7622 * Take necessary actions before going back to ring-3.
7623 *
7624 * An action requires us to go back to ring-3. This function does the necessary
7625 * steps before we can safely return to ring-3. This is not the same as longjmps
7626 * to ring-3, this is voluntary and prepares the guest so it may continue
7627 * executing outside HM (recompiler/IEM).
7628 *
7629 * @returns VBox status code.
7630 * @param pVM The cross context VM structure.
7631 * @param pVCpu The cross context virtual CPU structure.
7632 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7633 * out-of-sync. Make sure to update the required fields
7634 * before using them.
7635 * @param rcExit The reason for exiting to ring-3. Can be
7636 * VINF_VMM_UNKNOWN_RING3_CALL.
7637 */
7638static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7639{
7640 Assert(pVM);
7641 Assert(pVCpu);
7642 Assert(pMixedCtx);
7643 HMVMX_ASSERT_PREEMPT_SAFE();
7644
7645 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7646 {
7647 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7648 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7649 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7650 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7651 }
7652
7653 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7654 VMMRZCallRing3Disable(pVCpu);
7655 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7656
7657 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7658 if (pVCpu->hm.s.Event.fPending)
7659 {
7660 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7661 Assert(!pVCpu->hm.s.Event.fPending);
7662 }
7663
7664 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7665 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7666
7667 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7668 and if we're injecting an event we should have a TRPM trap pending. */
7669 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7670#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7671 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7672#endif
7673
7674 /* Save guest state and restore host state bits. */
7675 int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7676 AssertRCReturn(rc, rc);
7677 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7678 /* Thread-context hooks are unregistered at this point!!! */
7679
7680 /* Sync recompiler state. */
7681 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7682 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7683 | CPUM_CHANGED_LDTR
7684 | CPUM_CHANGED_GDTR
7685 | CPUM_CHANGED_IDTR
7686 | CPUM_CHANGED_TR
7687 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7688 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7689 if ( pVM->hm.s.fNestedPaging
7690 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7691 {
7692 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7693 }
7694
7695 Assert(!pVCpu->hm.s.fClearTrapFlag);
7696
7697 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7698 if (rcExit != VINF_EM_RAW_INTERRUPT)
7699 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7700
7701 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7702
7703 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7704 VMMRZCallRing3RemoveNotification(pVCpu);
7705 VMMRZCallRing3Enable(pVCpu);
7706
7707 return rc;
7708}
7709
7710
7711/**
7712 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7713 * longjump to ring-3 and possibly get preempted.
7714 *
7715 * @returns VBox status code.
7716 * @param pVCpu The cross context virtual CPU structure.
7717 * @param enmOperation The operation causing the ring-3 longjump.
7718 * @param pvUser Opaque pointer to the guest-CPU context. The data
7719 * may be out-of-sync. Make sure to update the required
7720 * fields before using them.
7721 */
7722static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7723{
7724 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7725 {
7726 /*
7727 * !!! IMPORTANT !!!
7728 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7729 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7730 */
7731 VMMRZCallRing3RemoveNotification(pVCpu);
7732 VMMRZCallRing3Disable(pVCpu);
7733 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7734 RTThreadPreemptDisable(&PreemptState);
7735
7736 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7737 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7738
7739#if HC_ARCH_BITS == 64
7740 /* Restore host-state bits that VT-x only restores partially. */
7741 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7742 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7743 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7744 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7745#endif
7746 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7747 if (pVCpu->hm.s.vmx.fLazyMsrs)
7748 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7749
7750 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7751 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7752 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7753 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7754 {
7755 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7756 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7757 }
7758
7759 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7760 VMMR0ThreadCtxHookDisable(pVCpu);
7761 HMR0LeaveCpu(pVCpu);
7762 RTThreadPreemptRestore(&PreemptState);
7763 return VINF_SUCCESS;
7764 }
7765
7766 Assert(pVCpu);
7767 Assert(pvUser);
7768 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7769 HMVMX_ASSERT_PREEMPT_SAFE();
7770
7771 VMMRZCallRing3Disable(pVCpu);
7772 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7773
7774 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7775 enmOperation));
7776
7777 int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7778 AssertRCReturn(rc, rc);
7779
7780 VMMRZCallRing3Enable(pVCpu);
7781 return VINF_SUCCESS;
7782}
7783
7784
7785/**
7786 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7787 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7788 *
7789 * @param pVCpu The cross context virtual CPU structure.
7790 */
7791DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7792{
7793 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7794 {
7795 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7796 {
7797 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7798 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7799 AssertRC(rc);
7800 Log4(("Setup interrupt-window exiting\n"));
7801 }
7802 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7803}
7804
7805
7806/**
7807 * Clears the interrupt-window exiting control in the VMCS.
7808 *
7809 * @param pVCpu The cross context virtual CPU structure.
7810 */
7811DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7812{
7813 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7814 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7815 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7816 AssertRC(rc);
7817 Log4(("Cleared interrupt-window exiting\n"));
7818}
7819
7820
7821/**
7822 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7823 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7824 *
7825 * @param pVCpu The cross context virtual CPU structure.
7826 */
7827DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7828{
7829 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7830 {
7831 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7832 {
7833 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7834 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7835 AssertRC(rc);
7836 Log4(("Setup NMI-window exiting\n"));
7837 }
7838 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7839}
7840
7841
7842/**
7843 * Clears the NMI-window exiting control in the VMCS.
7844 *
7845 * @param pVCpu The cross context virtual CPU structure.
7846 */
7847DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7848{
7849 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7850 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7851 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7852 AssertRC(rc);
7853 Log4(("Cleared NMI-window exiting\n"));
7854}
7855
7856
7857/**
7858 * Evaluates the event to be delivered to the guest and sets it as the pending
7859 * event.
7860 *
7861 * @returns The VT-x guest-interruptibility state.
7862 * @param pVCpu The cross context virtual CPU structure.
7863 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7864 * out-of-sync. Make sure to update the required fields
7865 * before using them.
7866 */
7867static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7868{
7869 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7870 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7871 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7872 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7873 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7874
7875 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7876 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7877 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7878 Assert(!TRPMHasTrap(pVCpu));
7879
7880 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7881 APICUpdatePendingInterrupts(pVCpu);
7882
7883 /*
7884 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7885 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7886 */
7887 /** @todo SMI. SMIs take priority over NMIs. */
7888 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7889 {
7890 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7891 if ( !pVCpu->hm.s.Event.fPending
7892 && !fBlockNmi
7893 && !fBlockSti
7894 && !fBlockMovSS)
7895 {
7896 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7897 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7898 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7899
7900 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7901 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7902 }
7903 else
7904 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7905 }
7906 /*
7907 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7908 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7909 */
7910 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7911 && !pVCpu->hm.s.fSingleInstruction)
7912 {
7913 Assert(!DBGFIsStepping(pVCpu));
7914 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7915 AssertRC(rc);
7916 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7917 if ( !pVCpu->hm.s.Event.fPending
7918 && !fBlockInt
7919 && !fBlockSti
7920 && !fBlockMovSS)
7921 {
7922 uint8_t u8Interrupt;
7923 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7924 if (RT_SUCCESS(rc))
7925 {
7926 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7927 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7928 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7929
7930 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7931 }
7932 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7933 {
7934 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7935 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7936 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7937
7938 /*
7939 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7940 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7941 * need to re-set this force-flag here.
7942 */
7943 }
7944 else
7945 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7946 }
7947 else
7948 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7949 }
7950
7951 return uIntrState;
7952}
7953
7954
7955/**
7956 * Sets a pending-debug exception to be delivered to the guest if the guest is
7957 * single-stepping in the VMCS.
7958 *
7959 * @param pVCpu The cross context virtual CPU structure.
7960 */
7961DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7962{
7963 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7964 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7965 AssertRC(rc);
7966}
7967
7968
7969/**
7970 * Injects any pending events into the guest if the guest is in a state to
7971 * receive them.
7972 *
7973 * @returns Strict VBox status code (i.e. informational status codes too).
7974 * @param pVCpu The cross context virtual CPU structure.
7975 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7976 * out-of-sync. Make sure to update the required fields
7977 * before using them.
7978 * @param uIntrState The VT-x guest-interruptibility state.
7979 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7980 * return VINF_EM_DBG_STEPPED if the event was
7981 * dispatched directly.
7982 */
7983static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7984{
7985 HMVMX_ASSERT_PREEMPT_SAFE();
7986 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7987
7988 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7989 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7990
7991 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7992 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7993 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7994 Assert(!TRPMHasTrap(pVCpu));
7995
7996 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7997 if (pVCpu->hm.s.Event.fPending)
7998 {
7999 /*
8000 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
8001 * pending even while injecting an event and in this case, we want a VM-exit as soon as
8002 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
8003 *
8004 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
8005 */
8006 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8007#ifdef VBOX_STRICT
8008 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8009 {
8010 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
8011 Assert(!fBlockInt);
8012 Assert(!fBlockSti);
8013 Assert(!fBlockMovSS);
8014 }
8015 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8016 {
8017 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
8018 Assert(!fBlockSti);
8019 Assert(!fBlockMovSS);
8020 Assert(!fBlockNmi);
8021 }
8022#endif
8023 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
8024 (uint8_t)uIntType));
8025 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
8026 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
8027 fStepping, &uIntrState);
8028 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
8029
8030 /* Update the interruptibility-state as it could have been changed by
8031 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
8032 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
8033 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
8034
8035 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8036 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
8037 else
8038 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
8039 }
8040
8041 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
8042 if ( fBlockSti
8043 || fBlockMovSS)
8044 {
8045 if (!pVCpu->hm.s.fSingleInstruction)
8046 {
8047 /*
8048 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
8049 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
8050 * See Intel spec. 27.3.4 "Saving Non-Register State".
8051 */
8052 Assert(!DBGFIsStepping(pVCpu));
8053 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8054 AssertRCReturn(rc2, rc2);
8055 if (pMixedCtx->eflags.Bits.u1TF)
8056 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
8057 }
8058 else if (pMixedCtx->eflags.Bits.u1TF)
8059 {
8060 /*
8061 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
8062 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
8063 */
8064 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
8065 uIntrState = 0;
8066 }
8067 }
8068
8069 /*
8070 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
8071 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8072 */
8073 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
8074 AssertRC(rc2);
8075
8076 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8077 NOREF(fBlockMovSS); NOREF(fBlockSti);
8078 return rcStrict;
8079}
8080
8081
8082/**
8083 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
8084 *
8085 * @param pVCpu The cross context virtual CPU structure.
8086 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8087 * out-of-sync. Make sure to update the required fields
8088 * before using them.
8089 */
8090DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8091{
8092 NOREF(pMixedCtx);
8093 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
8094 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
8095}
8096
8097
8098/**
8099 * Injects a double-fault (\#DF) exception into the VM.
8100 *
8101 * @returns Strict VBox status code (i.e. informational status codes too).
8102 * @param pVCpu The cross context virtual CPU structure.
8103 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8104 * out-of-sync. Make sure to update the required fields
8105 * before using them.
8106 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
8107 * and should return VINF_EM_DBG_STEPPED if the event
8108 * is injected directly (register modified by us, not
8109 * by hardware on VM-entry).
8110 * @param puIntrState Pointer to the current guest interruptibility-state.
8111 * This interruptibility-state will be updated if
8112 * necessary. This cannot not be NULL.
8113 */
8114DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
8115{
8116 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
8117 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8118 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8119 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
8120 fStepping, puIntrState);
8121}
8122
8123
8124/**
8125 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
8126 *
8127 * @param pVCpu The cross context virtual CPU structure.
8128 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8129 * out-of-sync. Make sure to update the required fields
8130 * before using them.
8131 */
8132DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8133{
8134 NOREF(pMixedCtx);
8135 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
8136 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8137 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
8138}
8139
8140
8141/**
8142 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
8143 *
8144 * @param pVCpu The cross context virtual CPU structure.
8145 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8146 * out-of-sync. Make sure to update the required fields
8147 * before using them.
8148 * @param cbInstr The value of RIP that is to be pushed on the guest
8149 * stack.
8150 */
8151DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
8152{
8153 NOREF(pMixedCtx);
8154 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
8155 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8156 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
8157}
8158
8159
8160/**
8161 * Injects a general-protection (\#GP) fault into the VM.
8162 *
8163 * @returns Strict VBox status code (i.e. informational status codes too).
8164 * @param pVCpu The cross context virtual CPU structure.
8165 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8166 * out-of-sync. Make sure to update the required fields
8167 * before using them.
8168 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
8169 * mode, i.e. in real-mode it's not valid).
8170 * @param u32ErrorCode The error code associated with the \#GP.
8171 * @param fStepping Whether we're running in
8172 * hmR0VmxRunGuestCodeStep() and should return
8173 * VINF_EM_DBG_STEPPED if the event is injected
8174 * directly (register modified by us, not by
8175 * hardware on VM-entry).
8176 * @param puIntrState Pointer to the current guest interruptibility-state.
8177 * This interruptibility-state will be updated if
8178 * necessary. This cannot not be NULL.
8179 */
8180DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
8181 bool fStepping, uint32_t *puIntrState)
8182{
8183 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
8184 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8185 if (fErrorCodeValid)
8186 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8187 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
8188 fStepping, puIntrState);
8189}
8190
8191
8192#if 0 /* unused */
8193/**
8194 * Sets a general-protection (\#GP) exception as pending-for-injection into the
8195 * VM.
8196 *
8197 * @param pVCpu The cross context virtual CPU structure.
8198 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8199 * out-of-sync. Make sure to update the required fields
8200 * before using them.
8201 * @param u32ErrorCode The error code associated with the \#GP.
8202 */
8203DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
8204{
8205 NOREF(pMixedCtx);
8206 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
8207 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8208 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8209 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
8210}
8211#endif /* unused */
8212
8213
8214/**
8215 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
8216 *
8217 * @param pVCpu The cross context virtual CPU structure.
8218 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8219 * out-of-sync. Make sure to update the required fields
8220 * before using them.
8221 * @param uVector The software interrupt vector number.
8222 * @param cbInstr The value of RIP that is to be pushed on the guest
8223 * stack.
8224 */
8225DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
8226{
8227 NOREF(pMixedCtx);
8228 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
8229 if ( uVector == X86_XCPT_BP
8230 || uVector == X86_XCPT_OF)
8231 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8232 else
8233 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8234 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
8235}
8236
8237
8238/**
8239 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8240 * stack.
8241 *
8242 * @returns Strict VBox status code (i.e. informational status codes too).
8243 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8244 * @param pVM The cross context VM structure.
8245 * @param pMixedCtx Pointer to the guest-CPU context.
8246 * @param uValue The value to push to the guest stack.
8247 */
8248DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
8249{
8250 /*
8251 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8252 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8253 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8254 */
8255 if (pMixedCtx->sp == 1)
8256 return VINF_EM_RESET;
8257 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8258 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
8259 AssertRC(rc);
8260 return rc;
8261}
8262
8263
8264/**
8265 * Injects an event into the guest upon VM-entry by updating the relevant fields
8266 * in the VM-entry area in the VMCS.
8267 *
8268 * @returns Strict VBox status code (i.e. informational status codes too).
8269 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8270 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8271 *
8272 * @param pVCpu The cross context virtual CPU structure.
8273 * @param pMixedCtx Pointer to the guest-CPU context. The data may
8274 * be out-of-sync. Make sure to update the required
8275 * fields before using them.
8276 * @param u64IntInfo The VM-entry interruption-information field.
8277 * @param cbInstr The VM-entry instruction length in bytes (for
8278 * software interrupts, exceptions and privileged
8279 * software exceptions).
8280 * @param u32ErrCode The VM-entry exception error code.
8281 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
8282 * @param puIntrState Pointer to the current guest interruptibility-state.
8283 * This interruptibility-state will be updated if
8284 * necessary. This cannot not be NULL.
8285 * @param fStepping Whether we're running in
8286 * hmR0VmxRunGuestCodeStep() and should return
8287 * VINF_EM_DBG_STEPPED if the event is injected
8288 * directly (register modified by us, not by
8289 * hardware on VM-entry).
8290 *
8291 * @remarks Requires CR0!
8292 */
8293static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
8294 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
8295 uint32_t *puIntrState)
8296{
8297 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8298 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
8299 Assert(puIntrState);
8300 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
8301
8302 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
8303 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
8304
8305#ifdef VBOX_STRICT
8306 /*
8307 * Validate the error-code-valid bit for hardware exceptions.
8308 * No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8309 */
8310 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8311 && !CPUMIsGuestInRealModeEx(pMixedCtx))
8312 {
8313 switch (uVector)
8314 {
8315 case X86_XCPT_PF:
8316 case X86_XCPT_DF:
8317 case X86_XCPT_TS:
8318 case X86_XCPT_NP:
8319 case X86_XCPT_SS:
8320 case X86_XCPT_GP:
8321 case X86_XCPT_AC:
8322 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
8323 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8324 RT_FALL_THRU();
8325 default:
8326 break;
8327 }
8328 }
8329#endif
8330
8331 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
8332 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8333 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
8334
8335 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8336
8337 /* We require CR0 to check if the guest is in real-mode. */
8338 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8339 AssertRCReturn(rc, rc);
8340
8341 /*
8342 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
8343 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
8344 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
8345 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8346 */
8347 if (CPUMIsGuestInRealModeEx(pMixedCtx))
8348 {
8349 PVM pVM = pVCpu->CTX_SUFF(pVM);
8350 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
8351 {
8352 Assert(PDMVmmDevHeapIsEnabled(pVM));
8353 Assert(pVM->hm.s.vmx.pRealModeTSS);
8354
8355 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
8356 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8357 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
8358 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8359 AssertRCReturn(rc, rc);
8360 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
8361
8362 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8363 size_t const cbIdtEntry = sizeof(X86IDTR16);
8364 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
8365 {
8366 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8367 if (uVector == X86_XCPT_DF)
8368 return VINF_EM_RESET;
8369
8370 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
8371 if (uVector == X86_XCPT_GP)
8372 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
8373
8374 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
8375 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
8376 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
8377 fStepping, puIntrState);
8378 }
8379
8380 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8381 uint16_t uGuestIp = pMixedCtx->ip;
8382 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
8383 {
8384 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
8385 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
8386 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8387 }
8388 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
8389 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8390
8391 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8392 X86IDTR16 IdtEntry;
8393 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
8394 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8395 AssertRCReturn(rc, rc);
8396
8397 /* Construct the stack frame for the interrupt/exception handler. */
8398 VBOXSTRICTRC rcStrict;
8399 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
8400 if (rcStrict == VINF_SUCCESS)
8401 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
8402 if (rcStrict == VINF_SUCCESS)
8403 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
8404
8405 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
8406 if (rcStrict == VINF_SUCCESS)
8407 {
8408 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
8409 pMixedCtx->rip = IdtEntry.offSel;
8410 pMixedCtx->cs.Sel = IdtEntry.uSel;
8411 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
8412 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8413 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8414 && uVector == X86_XCPT_PF)
8415 pMixedCtx->cr2 = GCPtrFaultAddress;
8416
8417 /* If any other guest-state bits are changed here, make sure to update
8418 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
8419 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
8420 | HM_CHANGED_GUEST_RIP
8421 | HM_CHANGED_GUEST_RFLAGS
8422 | HM_CHANGED_GUEST_RSP);
8423
8424 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8425 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8426 {
8427 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8428 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8429 Log4(("Clearing inhibition due to STI.\n"));
8430 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
8431 }
8432 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8433 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
8434
8435 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8436 it, if we are returning to ring-3 before executing guest code. */
8437 pVCpu->hm.s.Event.fPending = false;
8438
8439 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8440 if (fStepping)
8441 rcStrict = VINF_EM_DBG_STEPPED;
8442 }
8443 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8444 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8445 return rcStrict;
8446 }
8447
8448 /*
8449 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8450 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8451 */
8452 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8453 }
8454
8455 /* Validate. */
8456 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8457 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8458 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8459
8460 /* Inject. */
8461 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8462 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8463 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8464 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8465
8466 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8467 && uVector == X86_XCPT_PF)
8468 pMixedCtx->cr2 = GCPtrFaultAddress;
8469
8470 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8471 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8472
8473 AssertRCReturn(rc, rc);
8474 return VINF_SUCCESS;
8475}
8476
8477
8478/**
8479 * Clears the interrupt-window exiting control in the VMCS and if necessary
8480 * clears the current event in the VMCS as well.
8481 *
8482 * @returns VBox status code.
8483 * @param pVCpu The cross context virtual CPU structure.
8484 *
8485 * @remarks Use this function only to clear events that have not yet been
8486 * delivered to the guest but are injected in the VMCS!
8487 * @remarks No-long-jump zone!!!
8488 */
8489static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8490{
8491 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8492
8493 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8494 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8495
8496 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8497 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8498}
8499
8500
8501/**
8502 * Enters the VT-x session.
8503 *
8504 * @returns VBox status code.
8505 * @param pVM The cross context VM structure.
8506 * @param pVCpu The cross context virtual CPU structure.
8507 * @param pCpu Pointer to the CPU info struct.
8508 */
8509VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8510{
8511 AssertPtr(pVM);
8512 AssertPtr(pVCpu);
8513 Assert(pVM->hm.s.vmx.fSupported);
8514 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8515 NOREF(pCpu); NOREF(pVM);
8516
8517 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8518 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8519
8520#ifdef VBOX_STRICT
8521 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8522 RTCCUINTREG uHostCR4 = ASMGetCR4();
8523 if (!(uHostCR4 & X86_CR4_VMXE))
8524 {
8525 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8526 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8527 }
8528#endif
8529
8530 /*
8531 * Load the VCPU's VMCS as the current (and active) one.
8532 */
8533 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8534 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8535 if (RT_FAILURE(rc))
8536 return rc;
8537
8538 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8539 pVCpu->hm.s.fLeaveDone = false;
8540 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8541
8542 return VINF_SUCCESS;
8543}
8544
8545
8546/**
8547 * The thread-context callback (only on platforms which support it).
8548 *
8549 * @param enmEvent The thread-context event.
8550 * @param pVCpu The cross context virtual CPU structure.
8551 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8552 * @thread EMT(pVCpu)
8553 */
8554VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8555{
8556 NOREF(fGlobalInit);
8557
8558 switch (enmEvent)
8559 {
8560 case RTTHREADCTXEVENT_OUT:
8561 {
8562 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8563 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8564 VMCPU_ASSERT_EMT(pVCpu);
8565
8566 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8567
8568 /* No longjmps (logger flushes, locks) in this fragile context. */
8569 VMMRZCallRing3Disable(pVCpu);
8570 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8571
8572 /*
8573 * Restore host-state (FPU, debug etc.)
8574 */
8575 if (!pVCpu->hm.s.fLeaveDone)
8576 {
8577 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8578 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8579 hmR0VmxLeave(pVCpu, pMixedCtx, false /* fSaveGuestState */);
8580 pVCpu->hm.s.fLeaveDone = true;
8581 }
8582
8583 /* Leave HM context, takes care of local init (term). */
8584 int rc = HMR0LeaveCpu(pVCpu);
8585 AssertRC(rc); NOREF(rc);
8586
8587 /* Restore longjmp state. */
8588 VMMRZCallRing3Enable(pVCpu);
8589 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8590 break;
8591 }
8592
8593 case RTTHREADCTXEVENT_IN:
8594 {
8595 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8596 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8597 VMCPU_ASSERT_EMT(pVCpu);
8598
8599 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8600 VMMRZCallRing3Disable(pVCpu);
8601 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8602
8603 /* Initialize the bare minimum state required for HM. This takes care of
8604 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8605 int rc = HMR0EnterCpu(pVCpu);
8606 AssertRC(rc);
8607 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8608
8609 /* Load the active VMCS as the current one. */
8610 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8611 {
8612 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8613 AssertRC(rc); NOREF(rc);
8614 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8615 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8616 }
8617 pVCpu->hm.s.fLeaveDone = false;
8618
8619 /* Restore longjmp state. */
8620 VMMRZCallRing3Enable(pVCpu);
8621 break;
8622 }
8623
8624 default:
8625 break;
8626 }
8627}
8628
8629
8630/**
8631 * Saves the host state in the VMCS host-state.
8632 * Sets up the VM-exit MSR-load area.
8633 *
8634 * The CPU state will be loaded from these fields on every successful VM-exit.
8635 *
8636 * @returns VBox status code.
8637 * @param pVM The cross context VM structure.
8638 * @param pVCpu The cross context virtual CPU structure.
8639 *
8640 * @remarks No-long-jump zone!!!
8641 */
8642static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8643{
8644 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8645
8646 int rc = VINF_SUCCESS;
8647 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8648 {
8649 rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8650 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8651
8652 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8653 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8654
8655 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8656 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8657
8658 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8659 }
8660 return rc;
8661}
8662
8663
8664/**
8665 * Saves the host state in the VMCS host-state.
8666 *
8667 * @returns VBox status code.
8668 * @param pVM The cross context VM structure.
8669 * @param pVCpu The cross context virtual CPU structure.
8670 *
8671 * @remarks No-long-jump zone!!!
8672 */
8673VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8674{
8675 AssertPtr(pVM);
8676 AssertPtr(pVCpu);
8677
8678 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8679
8680 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8681 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8682 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8683 return hmR0VmxSaveHostState(pVM, pVCpu);
8684}
8685
8686
8687/**
8688 * Loads the guest state into the VMCS guest-state area.
8689 *
8690 * The will typically be done before VM-entry when the guest-CPU state and the
8691 * VMCS state may potentially be out of sync.
8692 *
8693 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8694 * VM-entry controls.
8695 * Sets up the appropriate VMX non-root function to execute guest code based on
8696 * the guest CPU mode.
8697 *
8698 * @returns VBox strict status code.
8699 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8700 * without unrestricted guest access and the VMMDev is not presently
8701 * mapped (e.g. EFI32).
8702 *
8703 * @param pVM The cross context VM structure.
8704 * @param pVCpu The cross context virtual CPU structure.
8705 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8706 * out-of-sync. Make sure to update the required fields
8707 * before using them.
8708 *
8709 * @remarks No-long-jump zone!!!
8710 */
8711static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8712{
8713 AssertPtr(pVM);
8714 AssertPtr(pVCpu);
8715 AssertPtr(pMixedCtx);
8716 HMVMX_ASSERT_PREEMPT_SAFE();
8717
8718 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8719
8720 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8721
8722 /* Determine real-on-v86 mode. */
8723 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8724 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8725 && CPUMIsGuestInRealModeEx(pMixedCtx))
8726 {
8727 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8728 }
8729
8730 /*
8731 * Load the guest-state into the VMCS.
8732 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8733 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8734 */
8735 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8736 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8737
8738 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8739 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8740 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8741
8742 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8743 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8744 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8745
8746 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8747 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8748
8749 VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8750 if (rcStrict == VINF_SUCCESS)
8751 { /* likely */ }
8752 else
8753 {
8754 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8755 return rcStrict;
8756 }
8757
8758 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8759 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8760 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8761
8762 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8763 determine we don't have to swap EFER after all. */
8764 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8765 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8766
8767 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8768 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8769
8770 rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8771 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8772
8773 /*
8774 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8775 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8776 */
8777 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8778 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8779
8780 /* Clear any unused and reserved bits. */
8781 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2
8782 | HM_CHANGED_GUEST_HWVIRT);
8783
8784 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8785 return rc;
8786}
8787
8788
8789/**
8790 * Loads the state shared between the host and guest into the VMCS.
8791 *
8792 * @param pVM The cross context VM structure.
8793 * @param pVCpu The cross context virtual CPU structure.
8794 * @param pCtx Pointer to the guest-CPU context.
8795 *
8796 * @remarks No-long-jump zone!!!
8797 */
8798static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8799{
8800 NOREF(pVM);
8801
8802 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8803 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8804
8805 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8806 {
8807 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8808 AssertRC(rc);
8809 }
8810
8811 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8812 {
8813 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8814 AssertRC(rc);
8815
8816 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8817 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8818 {
8819 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8820 AssertRC(rc);
8821 }
8822 }
8823
8824 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMM_GUEST_LAZY_MSRS))
8825 {
8826 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8827 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMM_GUEST_LAZY_MSRS);
8828 }
8829
8830 /* Loading CR0, debug state might have changed intercepts, update VMCS. */
8831 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMM_GUEST_XCPT_INTERCEPTS))
8832 {
8833 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8834 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8835 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8836 AssertRC(rc);
8837 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMM_GUEST_XCPT_INTERCEPTS);
8838 }
8839
8840 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8841 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8842}
8843
8844
8845/**
8846 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8847 *
8848 * @returns Strict VBox status code (i.e. informational status codes too).
8849 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8850 * without unrestricted guest access and the VMMDev is not presently
8851 * mapped (e.g. EFI32).
8852 *
8853 * @param pVM The cross context VM structure.
8854 * @param pVCpu The cross context virtual CPU structure.
8855 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8856 * out-of-sync. Make sure to update the required fields
8857 * before using them.
8858 *
8859 * @remarks No-long-jump zone!!!
8860 */
8861static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8862{
8863 HMVMX_ASSERT_PREEMPT_SAFE();
8864 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8865 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8866
8867 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8868#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8869 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8870#endif
8871
8872 /*
8873 * RIP is what changes the most often and hence if it's the only bit needing to be
8874 * updated, we shall handle it early for performance reasons.
8875 */
8876 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8877 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8878 {
8879 rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8880 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8881 { /* likely */}
8882 else
8883 {
8884 AssertMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8885 VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8886 }
8887 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8888 }
8889 else if (HMCPU_CF_VALUE(pVCpu))
8890 {
8891 rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8892 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8893 { /* likely */}
8894 else
8895 {
8896 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8897 ("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8898 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8899 return rcStrict;
8900 }
8901 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8902 }
8903
8904 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8905 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8906 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8907 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8908 return rcStrict;
8909}
8910
8911
8912/**
8913 * Does the preparations before executing guest code in VT-x.
8914 *
8915 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8916 * recompiler/IEM. We must be cautious what we do here regarding committing
8917 * guest-state information into the VMCS assuming we assuredly execute the
8918 * guest in VT-x mode.
8919 *
8920 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8921 * the common-state (TRPM/forceflags), we must undo those changes so that the
8922 * recompiler/IEM can (and should) use them when it resumes guest execution.
8923 * Otherwise such operations must be done when we can no longer exit to ring-3.
8924 *
8925 * @returns Strict VBox status code (i.e. informational status codes too).
8926 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8927 * have been disabled.
8928 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8929 * double-fault into the guest.
8930 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8931 * dispatched directly.
8932 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8933 *
8934 * @param pVM The cross context VM structure.
8935 * @param pVCpu The cross context virtual CPU structure.
8936 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8937 * out-of-sync. Make sure to update the required fields
8938 * before using them.
8939 * @param pVmxTransient Pointer to the VMX transient structure.
8940 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8941 * us ignore some of the reasons for returning to
8942 * ring-3, and return VINF_EM_DBG_STEPPED if event
8943 * dispatching took place.
8944 */
8945static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8946{
8947 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8948
8949#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8950 PGMRZDynMapFlushAutoSet(pVCpu);
8951#endif
8952
8953 /* Check force flag actions that might require us to go back to ring-3. */
8954 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8955 if (rcStrict == VINF_SUCCESS)
8956 { /* FFs doesn't get set all the time. */ }
8957 else
8958 return rcStrict;
8959
8960 /*
8961 * Setup the virtualized-APIC accesses.
8962 *
8963 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8964 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8965 *
8966 * This is the reason we do it here and not in hmR0VmxLoadGuestState().
8967 */
8968 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8969 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8970 && PDMHasApic(pVM))
8971 {
8972 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8973 Assert(u64MsrApicBase);
8974 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8975
8976 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8977
8978 /* Unalias any existing mapping. */
8979 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8980 AssertRCReturn(rc, rc);
8981
8982 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8983 Log4(("hmR0VmxPreRunGuest: VCPU%u: Mapped HC APIC-access page at %#RGp\n", pVCpu->idCpu, GCPhysApicBase));
8984 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8985 AssertRCReturn(rc, rc);
8986
8987 /* Update the per-VCPU cache of the APIC base MSR. */
8988 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8989 }
8990
8991 if (TRPMHasTrap(pVCpu))
8992 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8993 uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8994
8995 /*
8996 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8997 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8998 */
8999 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
9000 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9001 { /* likely */ }
9002 else
9003 {
9004 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
9005 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9006 return rcStrict;
9007 }
9008
9009 /*
9010 * No longjmps to ring-3 from this point on!!!
9011 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
9012 * This also disables flushing of the R0-logger instance (if any).
9013 */
9014 VMMRZCallRing3Disable(pVCpu);
9015
9016 /*
9017 * Load the guest state bits.
9018 *
9019 * We cannot perform longjmps while loading the guest state because we do not preserve the
9020 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
9021 * CPU migration.
9022 *
9023 * If we are injecting events to a real-on-v86 mode guest, we will have to update
9024 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
9025 * Hence, loading of the guest state needs to be done -after- injection of events.
9026 */
9027 rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
9028 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9029 { /* likely */ }
9030 else
9031 {
9032 VMMRZCallRing3Enable(pVCpu);
9033 return rcStrict;
9034 }
9035
9036 /*
9037 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
9038 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
9039 *
9040 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
9041 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
9042 *
9043 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
9044 * executing guest code.
9045 */
9046 pVmxTransient->fEFlags = ASMIntDisableFlags();
9047
9048 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
9049 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
9050 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
9051 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
9052 {
9053 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
9054 {
9055 pVCpu->hm.s.Event.fPending = false;
9056
9057 /*
9058 * We've injected any pending events. This is really the point of no return (to ring-3).
9059 *
9060 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
9061 * returns from this function, so don't enable them here.
9062 */
9063 return VINF_SUCCESS;
9064 }
9065
9066 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
9067 rcStrict = VINF_EM_RAW_INTERRUPT;
9068 }
9069 else
9070 {
9071 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
9072 rcStrict = VINF_EM_RAW_TO_R3;
9073 }
9074
9075 ASMSetFlags(pVmxTransient->fEFlags);
9076 VMMRZCallRing3Enable(pVCpu);
9077
9078 return rcStrict;
9079}
9080
9081
9082/**
9083 * Prepares to run guest code in VT-x and we've committed to doing so. This
9084 * means there is no backing out to ring-3 or anywhere else at this
9085 * point.
9086 *
9087 * @param pVM The cross context VM structure.
9088 * @param pVCpu The cross context virtual CPU structure.
9089 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9090 * out-of-sync. Make sure to update the required fields
9091 * before using them.
9092 * @param pVmxTransient Pointer to the VMX transient structure.
9093 *
9094 * @remarks Called with preemption disabled.
9095 * @remarks No-long-jump zone!!!
9096 */
9097static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9098{
9099 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9100 Assert(VMMR0IsLogFlushDisabled(pVCpu));
9101 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9102
9103 /*
9104 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
9105 */
9106 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
9107 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
9108
9109 if (!CPUMIsGuestFPUStateActive(pVCpu))
9110 {
9111 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
9112 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
9113 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
9114 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
9115 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
9116 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
9117 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9118 }
9119
9120 /*
9121 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
9122 */
9123 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
9124 && pVCpu->hm.s.vmx.cMsrs > 0)
9125 {
9126 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
9127 }
9128
9129 /*
9130 * Load the host state bits as we may've been preempted (only happens when
9131 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
9132 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
9133 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
9134 * See @bugref{8432}.
9135 */
9136 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
9137 {
9138 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
9139 AssertRC(rc);
9140 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
9141 }
9142 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
9143
9144 /*
9145 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
9146 */
9147 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
9148 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
9149 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
9150
9151 /* Store status of the shared guest-host state at the time of VM-entry. */
9152#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
9153 if (CPUMIsGuestInLongModeEx(pMixedCtx))
9154 {
9155 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
9156 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
9157 }
9158 else
9159#endif
9160 {
9161 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
9162 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
9163 }
9164
9165 /*
9166 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
9167 */
9168 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9169 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
9170
9171 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
9172 RTCPUID idCurrentCpu = pCpu->idCpu;
9173 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
9174 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
9175 {
9176 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
9177 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
9178 }
9179
9180 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
9181 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
9182 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
9183 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
9184
9185 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
9186
9187 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
9188 to start executing. */
9189
9190 /*
9191 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
9192 */
9193 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
9194 {
9195 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9196 {
9197 bool fMsrUpdated;
9198 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9199 AssertRC(rc2);
9200 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
9201
9202 rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
9203 &fMsrUpdated);
9204 AssertRC(rc2);
9205 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9206
9207 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
9208 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
9209 }
9210 else
9211 {
9212 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
9213 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9214 }
9215 }
9216
9217 if (pVM->cpum.ro.GuestFeatures.fIbrs)
9218 {
9219 bool fMsrUpdated;
9220 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9221 AssertRC(rc2);
9222 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
9223
9224 rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
9225 &fMsrUpdated);
9226 AssertRC(rc2);
9227 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9228 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
9229 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
9230 }
9231
9232#ifdef VBOX_STRICT
9233 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
9234 hmR0VmxCheckHostEferMsr(pVCpu);
9235 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
9236#endif
9237#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
9238 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
9239 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
9240 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
9241#endif
9242}
9243
9244
9245/**
9246 * Performs some essential restoration of state after running guest code in
9247 * VT-x.
9248 *
9249 * @param pVM The cross context VM structure.
9250 * @param pVCpu The cross context virtual CPU structure.
9251 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9252 * out-of-sync. Make sure to update the required fields
9253 * before using them.
9254 * @param pVmxTransient Pointer to the VMX transient structure.
9255 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
9256 *
9257 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
9258 *
9259 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
9260 * unconditionally when it is safe to do so.
9261 */
9262static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
9263{
9264 NOREF(pVM);
9265 uint64_t uHostTsc = ASMReadTSC();
9266
9267 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9268
9269 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
9270 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
9271 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
9272 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
9273 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
9274 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
9275
9276 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9277 TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TSCOffset);
9278
9279 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
9280 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
9281 Assert(!ASMIntAreEnabled());
9282 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
9283
9284#if HC_ARCH_BITS == 64
9285 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
9286#endif
9287#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
9288 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
9289 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
9290 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
9291#else
9292 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
9293#endif
9294#ifdef VBOX_STRICT
9295 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
9296#endif
9297 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
9298
9299 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
9300 uint32_t uExitReason;
9301 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
9302 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9303 AssertRC(rc);
9304 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
9305 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
9306
9307 if (rcVMRun == VINF_SUCCESS)
9308 {
9309 /*
9310 * Update the VM-exit history array here even if the VM-entry failed due to:
9311 * - Invalid guest state.
9312 * - MSR loading.
9313 * - Machine-check event.
9314 *
9315 * In any of the above cases we will still have a "valid" VM-exit reason
9316 * despite @a fVMEntryFailed being false.
9317 *
9318 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
9319 *
9320 * Note! We don't have CS or RIP at this point. Will probably address that later
9321 * by amending the history entry added here.
9322 */
9323 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FLAGS_AND_TYPE(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
9324 UINT64_MAX, uHostTsc);
9325
9326 if (!pVmxTransient->fVMEntryFailed)
9327 {
9328 /** @todo We can optimize this by only syncing with our force-flags when
9329 * really needed and keeping the VMCS state as it is for most
9330 * VM-exits. */
9331 /* Update the guest interruptibility-state from the VMCS. */
9332 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
9333
9334 /*
9335 * Allow longjmps to ring-3 -after- saving the guest-interruptibility state
9336 * as it's not part of hmR0VmxSaveGuestState() and thus would trigger an assertion
9337 * on the longjmp path to ring-3 while saving the (rest of) the guest state,
9338 * see @bugref{6208#c63}.
9339 */
9340 VMMRZCallRing3Enable(pVCpu);
9341
9342#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
9343 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9344 AssertRC(rc);
9345#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
9346 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9347 AssertRC(rc);
9348#endif
9349
9350 /*
9351 * Sync the TPR shadow with our APIC state.
9352 */
9353 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9354 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
9355 {
9356 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
9357 AssertRC(rc);
9358 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_APIC_STATE);
9359 }
9360
9361 return;
9362 }
9363 }
9364 else
9365 {
9366 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
9367 pVmxTransient->fVMEntryFailed));
9368 }
9369
9370 VMMRZCallRing3Enable(pVCpu);
9371}
9372
9373
9374/**
9375 * Runs the guest code using VT-x the normal way.
9376 *
9377 * @returns VBox status code.
9378 * @param pVM The cross context VM structure.
9379 * @param pVCpu The cross context virtual CPU structure.
9380 * @param pCtx Pointer to the guest-CPU context.
9381 *
9382 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
9383 */
9384static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9385{
9386 VMXTRANSIENT VmxTransient;
9387 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9388 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9389 uint32_t cLoops = 0;
9390
9391 for (;; cLoops++)
9392 {
9393 Assert(!HMR0SuspendPending());
9394 HMVMX_ASSERT_CPU_SAFE();
9395
9396 /* Preparatory work for running guest code, this may force us to return
9397 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
9398 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9399 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
9400 if (rcStrict != VINF_SUCCESS)
9401 break;
9402
9403 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9404 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9405 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9406
9407 /* Restore any residual host-state and save any bits shared between host
9408 and guest into the guest-CPU state. Re-enables interrupts! */
9409 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
9410
9411 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9412 if (RT_SUCCESS(rcRun))
9413 { /* very likely */ }
9414 else
9415 {
9416 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9417 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9418 return rcRun;
9419 }
9420
9421 /* Profile the VM-exit. */
9422 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9423 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9424 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9425 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9426 HMVMX_START_EXIT_DISPATCH_PROF();
9427
9428 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9429
9430 /* Handle the VM-exit. */
9431#ifdef HMVMX_USE_FUNCTION_TABLE
9432 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
9433#else
9434 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
9435#endif
9436 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9437 if (rcStrict == VINF_SUCCESS)
9438 {
9439 if (cLoops <= pVM->hm.s.cMaxResumeLoops)
9440 continue; /* likely */
9441 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9442 rcStrict = VINF_EM_RAW_INTERRUPT;
9443 }
9444 break;
9445 }
9446
9447 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9448 return rcStrict;
9449}
9450
9451
9452
9453/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
9454 * probes.
9455 *
9456 * The following few functions and associated structure contains the bloat
9457 * necessary for providing detailed debug events and dtrace probes as well as
9458 * reliable host side single stepping. This works on the principle of
9459 * "subclassing" the normal execution loop and workers. We replace the loop
9460 * method completely and override selected helpers to add necessary adjustments
9461 * to their core operation.
9462 *
9463 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
9464 * any performance for debug and analysis features.
9465 *
9466 * @{
9467 */
9468
9469/**
9470 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
9471 * the debug run loop.
9472 */
9473typedef struct VMXRUNDBGSTATE
9474{
9475 /** The RIP we started executing at. This is for detecting that we stepped. */
9476 uint64_t uRipStart;
9477 /** The CS we started executing with. */
9478 uint16_t uCsStart;
9479
9480 /** Whether we've actually modified the 1st execution control field. */
9481 bool fModifiedProcCtls : 1;
9482 /** Whether we've actually modified the 2nd execution control field. */
9483 bool fModifiedProcCtls2 : 1;
9484 /** Whether we've actually modified the exception bitmap. */
9485 bool fModifiedXcptBitmap : 1;
9486
9487 /** We desire the modified the CR0 mask to be cleared. */
9488 bool fClearCr0Mask : 1;
9489 /** We desire the modified the CR4 mask to be cleared. */
9490 bool fClearCr4Mask : 1;
9491 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9492 uint32_t fCpe1Extra;
9493 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9494 uint32_t fCpe1Unwanted;
9495 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9496 uint32_t fCpe2Extra;
9497 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9498 uint32_t bmXcptExtra;
9499 /** The sequence number of the Dtrace provider settings the state was
9500 * configured against. */
9501 uint32_t uDtraceSettingsSeqNo;
9502 /** VM-exits to check (one bit per VM-exit). */
9503 uint32_t bmExitsToCheck[3];
9504
9505 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9506 uint32_t fProcCtlsInitial;
9507 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9508 uint32_t fProcCtls2Initial;
9509 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9510 uint32_t bmXcptInitial;
9511} VMXRUNDBGSTATE;
9512AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9513typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9514
9515
9516/**
9517 * Initializes the VMXRUNDBGSTATE structure.
9518 *
9519 * @param pVCpu The cross context virtual CPU structure of the
9520 * calling EMT.
9521 * @param pCtx The CPU register context to go with @a pVCpu.
9522 * @param pDbgState The structure to initialize.
9523 */
9524DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9525{
9526 pDbgState->uRipStart = pCtx->rip;
9527 pDbgState->uCsStart = pCtx->cs.Sel;
9528
9529 pDbgState->fModifiedProcCtls = false;
9530 pDbgState->fModifiedProcCtls2 = false;
9531 pDbgState->fModifiedXcptBitmap = false;
9532 pDbgState->fClearCr0Mask = false;
9533 pDbgState->fClearCr4Mask = false;
9534 pDbgState->fCpe1Extra = 0;
9535 pDbgState->fCpe1Unwanted = 0;
9536 pDbgState->fCpe2Extra = 0;
9537 pDbgState->bmXcptExtra = 0;
9538 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9539 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9540 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9541}
9542
9543
9544/**
9545 * Updates the VMSC fields with changes requested by @a pDbgState.
9546 *
9547 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9548 * immediately before executing guest code, i.e. when interrupts are disabled.
9549 * We don't check status codes here as we cannot easily assert or return in the
9550 * latter case.
9551 *
9552 * @param pVCpu The cross context virtual CPU structure.
9553 * @param pDbgState The debug state.
9554 */
9555DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9556{
9557 /*
9558 * Ensure desired flags in VMCS control fields are set.
9559 * (Ignoring write failure here, as we're committed and it's just debug extras.)
9560 *
9561 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9562 * there should be no stale data in pCtx at this point.
9563 */
9564 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9565 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9566 {
9567 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
9568 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9569 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9570 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9571 pDbgState->fModifiedProcCtls = true;
9572 }
9573
9574 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9575 {
9576 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
9577 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9578 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9579 pDbgState->fModifiedProcCtls2 = true;
9580 }
9581
9582 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9583 {
9584 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
9585 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9586 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9587 pDbgState->fModifiedXcptBitmap = true;
9588 }
9589
9590 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9591 {
9592 pVCpu->hm.s.vmx.u32CR0Mask = 0;
9593 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9594 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9595 }
9596
9597 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9598 {
9599 pVCpu->hm.s.vmx.u32CR4Mask = 0;
9600 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9601 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9602 }
9603}
9604
9605
9606DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9607{
9608 /*
9609 * Restore VM-exit control settings as we may not reenter this function the
9610 * next time around.
9611 */
9612 /* We reload the initial value, trigger what we can of recalculations the
9613 next time around. From the looks of things, that's all that's required atm. */
9614 if (pDbgState->fModifiedProcCtls)
9615 {
9616 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9617 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9618 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9619 AssertRCReturn(rc2, rc2);
9620 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9621 }
9622
9623 /* We're currently the only ones messing with this one, so just restore the
9624 cached value and reload the field. */
9625 if ( pDbgState->fModifiedProcCtls2
9626 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9627 {
9628 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9629 AssertRCReturn(rc2, rc2);
9630 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9631 }
9632
9633 /* If we've modified the exception bitmap, we restore it and trigger
9634 reloading and partial recalculation the next time around. */
9635 if (pDbgState->fModifiedXcptBitmap)
9636 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9637
9638 return rcStrict;
9639}
9640
9641
9642/**
9643 * Configures VM-exit controls for current DBGF and DTrace settings.
9644 *
9645 * This updates @a pDbgState and the VMCS execution control fields to reflect
9646 * the necessary VM-exits demanded by DBGF and DTrace.
9647 *
9648 * @param pVM The cross context VM structure.
9649 * @param pVCpu The cross context virtual CPU structure.
9650 * @param pCtx Pointer to the guest-CPU context.
9651 * @param pDbgState The debug state.
9652 * @param pVmxTransient Pointer to the VMX transient structure. May update
9653 * fUpdateTscOffsettingAndPreemptTimer.
9654 */
9655static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9656 PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9657{
9658 /*
9659 * Take down the dtrace serial number so we can spot changes.
9660 */
9661 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9662 ASMCompilerBarrier();
9663
9664 /*
9665 * We'll rebuild most of the middle block of data members (holding the
9666 * current settings) as we go along here, so start by clearing it all.
9667 */
9668 pDbgState->bmXcptExtra = 0;
9669 pDbgState->fCpe1Extra = 0;
9670 pDbgState->fCpe1Unwanted = 0;
9671 pDbgState->fCpe2Extra = 0;
9672 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9673 pDbgState->bmExitsToCheck[i] = 0;
9674
9675 /*
9676 * Software interrupts (INT XXh) - no idea how to trigger these...
9677 */
9678 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9679 || VBOXVMM_INT_SOFTWARE_ENABLED())
9680 {
9681 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9682 }
9683
9684 /*
9685 * INT3 breakpoints - triggered by #BP exceptions.
9686 */
9687 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9688 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9689
9690 /*
9691 * Exception bitmap and XCPT events+probes.
9692 */
9693 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9694 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9695 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9696
9697 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9698 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9699 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9700 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9701 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9702 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9703 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9704 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9705 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9706 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9707 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9708 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9709 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9710 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9711 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9712 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9713 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9714 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9715
9716 if (pDbgState->bmXcptExtra)
9717 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9718
9719 /*
9720 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9721 *
9722 * Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9723 * So, when adding/changing/removing please don't forget to update it.
9724 *
9725 * Some of the macros are picking up local variables to save horizontal space,
9726 * (being able to see it in a table is the lesser evil here).
9727 */
9728#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9729 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9730 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9731#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9732 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9733 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9734 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9735 } else do { } while (0)
9736#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9737 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9738 { \
9739 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9740 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9741 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9742 } else do { } while (0)
9743#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9744 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9745 { \
9746 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9747 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9748 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9749 } else do { } while (0)
9750#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9751 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9752 { \
9753 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9754 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9755 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9756 } else do { } while (0)
9757
9758 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9759 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9760 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9761 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9762 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9763
9764 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9765 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9766 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9767 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9768 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9769 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9770 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9771 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9772 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9773 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9774 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9775 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9776 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9777 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9778 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9779 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9780 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9781 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9782 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9783 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9784 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9785 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9786 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9787 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9788 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9789 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9790 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9791 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9792 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9793 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9794 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9795 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9796 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9797 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9798 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9799 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9800
9801 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9802 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9803 {
9804 int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9805 rc2 |= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9806 rc2 |= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9807 AssertRC(rc2);
9808
9809#if 0 /** @todo fix me */
9810 pDbgState->fClearCr0Mask = true;
9811 pDbgState->fClearCr4Mask = true;
9812#endif
9813 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9814 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9815 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9816 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9817 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9818 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9819 require clearing here and in the loop if we start using it. */
9820 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9821 }
9822 else
9823 {
9824 if (pDbgState->fClearCr0Mask)
9825 {
9826 pDbgState->fClearCr0Mask = false;
9827 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9828 }
9829 if (pDbgState->fClearCr4Mask)
9830 {
9831 pDbgState->fClearCr4Mask = false;
9832 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9833 }
9834 }
9835 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9836 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9837
9838 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9839 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9840 {
9841 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9842 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9843 }
9844 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9845 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9846
9847 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9848 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9849 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9850 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9851 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9852 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9853 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9854 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9855#if 0 /** @todo too slow, fix handler. */
9856 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9857#endif
9858 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9859
9860 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9861 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9862 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9863 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9864 {
9865 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9866 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9867 }
9868 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9869 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9870 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9871 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9872
9873 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9874 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9875 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9876 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9877 {
9878 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9879 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9880 }
9881 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9882 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9883 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9884 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9885
9886 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9887 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9888 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9889 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9890 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9891 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9892 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9893 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9894 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9895 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9896 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9897 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9898 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9899 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9900 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9901 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9902 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9903 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9904 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9905 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9906 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9907 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9908
9909#undef IS_EITHER_ENABLED
9910#undef SET_ONLY_XBM_IF_EITHER_EN
9911#undef SET_CPE1_XBM_IF_EITHER_EN
9912#undef SET_CPEU_XBM_IF_EITHER_EN
9913#undef SET_CPE2_XBM_IF_EITHER_EN
9914
9915 /*
9916 * Sanitize the control stuff.
9917 */
9918 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9919 if (pDbgState->fCpe2Extra)
9920 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9921 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9922 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9923 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9924 {
9925 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9926 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9927 }
9928
9929 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9930 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9931 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9932 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9933}
9934
9935
9936/**
9937 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9938 * appropriate.
9939 *
9940 * The caller has checked the VM-exit against the
9941 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9942 * already, so we don't have to do that either.
9943 *
9944 * @returns Strict VBox status code (i.e. informational status codes too).
9945 * @param pVM The cross context VM structure.
9946 * @param pVCpu The cross context virtual CPU structure.
9947 * @param pMixedCtx Pointer to the guest-CPU context.
9948 * @param pVmxTransient Pointer to the VMX-transient structure.
9949 * @param uExitReason The VM-exit reason.
9950 *
9951 * @remarks The name of this function is displayed by dtrace, so keep it short
9952 * and to the point. No longer than 33 chars long, please.
9953 */
9954static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9955 PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9956{
9957 /*
9958 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9959 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9960 *
9961 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9962 * does. Must add/change/remove both places. Same ordering, please.
9963 *
9964 * Added/removed events must also be reflected in the next section
9965 * where we dispatch dtrace events.
9966 */
9967 bool fDtrace1 = false;
9968 bool fDtrace2 = false;
9969 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9970 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9971 uint32_t uEventArg = 0;
9972#define SET_EXIT(a_EventSubName) \
9973 do { \
9974 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9975 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9976 } while (0)
9977#define SET_BOTH(a_EventSubName) \
9978 do { \
9979 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9980 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9981 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9982 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9983 } while (0)
9984 switch (uExitReason)
9985 {
9986 case VMX_EXIT_MTF:
9987 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9988
9989 case VMX_EXIT_XCPT_OR_NMI:
9990 {
9991 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9992 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9993 {
9994 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9995 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9996 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9997 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9998 {
9999 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
10000 {
10001 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10002 uEventArg = pVmxTransient->uExitIntErrorCode;
10003 }
10004 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
10005 switch (enmEvent1)
10006 {
10007 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
10008 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
10009 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
10010 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
10011 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
10012 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
10013 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
10014 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
10015 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
10016 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
10017 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
10018 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
10019 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
10020 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
10021 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
10022 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
10023 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
10024 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
10025 default: break;
10026 }
10027 }
10028 else
10029 AssertFailed();
10030 break;
10031
10032 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
10033 uEventArg = idxVector;
10034 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
10035 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
10036 break;
10037 }
10038 break;
10039 }
10040
10041 case VMX_EXIT_TRIPLE_FAULT:
10042 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
10043 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
10044 break;
10045 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
10046 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
10047 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
10048 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
10049 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
10050
10051 /* Instruction specific VM-exits: */
10052 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
10053 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
10054 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
10055 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
10056 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
10057 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
10058 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
10059 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
10060 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
10061 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
10062 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
10063 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
10064 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
10065 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
10066 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
10067 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
10068 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
10069 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
10070 case VMX_EXIT_MOV_CRX:
10071 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10072/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
10073* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
10074 if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
10075 == VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
10076 SET_BOTH(CRX_READ);
10077 else
10078 SET_BOTH(CRX_WRITE);
10079 uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
10080 break;
10081 case VMX_EXIT_MOV_DRX:
10082 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10083 if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
10084 == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
10085 SET_BOTH(DRX_READ);
10086 else
10087 SET_BOTH(DRX_WRITE);
10088 uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
10089 break;
10090 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
10091 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
10092 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
10093 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
10094 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
10095 case VMX_EXIT_XDTR_ACCESS:
10096 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10097 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
10098 {
10099 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
10100 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
10101 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
10102 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
10103 }
10104 break;
10105
10106 case VMX_EXIT_TR_ACCESS:
10107 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10108 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
10109 {
10110 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
10111 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
10112 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
10113 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
10114 }
10115 break;
10116
10117 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
10118 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
10119 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
10120 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
10121 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
10122 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
10123 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
10124 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
10125 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
10126 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
10127 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
10128
10129 /* Events that aren't relevant at this point. */
10130 case VMX_EXIT_EXT_INT:
10131 case VMX_EXIT_INT_WINDOW:
10132 case VMX_EXIT_NMI_WINDOW:
10133 case VMX_EXIT_TPR_BELOW_THRESHOLD:
10134 case VMX_EXIT_PREEMPT_TIMER:
10135 case VMX_EXIT_IO_INSTR:
10136 break;
10137
10138 /* Errors and unexpected events. */
10139 case VMX_EXIT_INIT_SIGNAL:
10140 case VMX_EXIT_SIPI:
10141 case VMX_EXIT_IO_SMI:
10142 case VMX_EXIT_SMI:
10143 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10144 case VMX_EXIT_ERR_MSR_LOAD:
10145 case VMX_EXIT_ERR_MACHINE_CHECK:
10146 break;
10147
10148 default:
10149 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10150 break;
10151 }
10152#undef SET_BOTH
10153#undef SET_EXIT
10154
10155 /*
10156 * Dtrace tracepoints go first. We do them here at once so we don't
10157 * have to copy the guest state saving and stuff a few dozen times.
10158 * Down side is that we've got to repeat the switch, though this time
10159 * we use enmEvent since the probes are a subset of what DBGF does.
10160 */
10161 if (fDtrace1 || fDtrace2)
10162 {
10163 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10164 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10165 switch (enmEvent1)
10166 {
10167 /** @todo consider which extra parameters would be helpful for each probe. */
10168 case DBGFEVENT_END: break;
10169 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
10170 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
10171 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
10172 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
10173 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
10174 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
10175 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
10176 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
10177 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
10178 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
10179 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
10180 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
10181 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
10182 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
10183 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
10184 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
10185 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
10186 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
10187 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10188 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
10189 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
10190 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
10191 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
10192 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
10193 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
10194 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
10195 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
10196 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10197 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10198 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10199 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10200 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
10201 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
10202 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
10203 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
10204 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
10205 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
10206 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
10207 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
10208 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
10209 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
10210 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
10211 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
10212 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
10213 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
10214 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
10215 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
10216 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
10217 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
10218 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
10219 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
10220 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
10221 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
10222 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
10223 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
10224 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
10225 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
10226 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
10227 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
10228 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
10229 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
10230 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
10231 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
10232 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
10233 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
10234 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
10235 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
10236 }
10237 switch (enmEvent2)
10238 {
10239 /** @todo consider which extra parameters would be helpful for each probe. */
10240 case DBGFEVENT_END: break;
10241 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
10242 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
10243 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
10244 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
10245 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
10246 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
10247 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
10248 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
10249 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
10250 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10251 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10252 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10253 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10254 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
10255 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
10256 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
10257 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
10258 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
10259 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
10260 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
10261 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
10262 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
10263 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
10264 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
10265 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
10266 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
10267 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
10268 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
10269 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
10270 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
10271 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
10272 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
10273 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
10274 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
10275 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
10276 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
10277 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
10278 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
10279 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
10280 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
10281 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
10282 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
10283 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
10284 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
10285 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
10286 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
10287 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
10288 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
10289 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
10290 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
10291 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
10292 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
10293 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
10294 }
10295 }
10296
10297 /*
10298 * Fire of the DBGF event, if enabled (our check here is just a quick one,
10299 * the DBGF call will do a full check).
10300 *
10301 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
10302 * Note! If we have to events, we prioritize the first, i.e. the instruction
10303 * one, in order to avoid event nesting.
10304 */
10305 if ( enmEvent1 != DBGFEVENT_END
10306 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
10307 {
10308 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
10309 if (rcStrict != VINF_SUCCESS)
10310 return rcStrict;
10311 }
10312 else if ( enmEvent2 != DBGFEVENT_END
10313 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
10314 {
10315 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
10316 if (rcStrict != VINF_SUCCESS)
10317 return rcStrict;
10318 }
10319
10320 return VINF_SUCCESS;
10321}
10322
10323
10324/**
10325 * Single-stepping VM-exit filtering.
10326 *
10327 * This is preprocessing the VM-exits and deciding whether we've gotten far
10328 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
10329 * handling is performed.
10330 *
10331 * @returns Strict VBox status code (i.e. informational status codes too).
10332 * @param pVM The cross context VM structure.
10333 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
10334 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
10335 * out-of-sync. Make sure to update the required
10336 * fields before using them.
10337 * @param pVmxTransient Pointer to the VMX-transient structure.
10338 * @param uExitReason The VM-exit reason.
10339 * @param pDbgState The debug state.
10340 */
10341DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
10342 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
10343{
10344 /*
10345 * Expensive (saves context) generic dtrace VM-exit probe.
10346 */
10347 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
10348 { /* more likely */ }
10349 else
10350 {
10351 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10352 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10353 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
10354 }
10355
10356 /*
10357 * Check for host NMI, just to get that out of the way.
10358 */
10359 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
10360 { /* normally likely */ }
10361 else
10362 {
10363 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10364 AssertRCReturn(rc2, rc2);
10365 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10366 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10367 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
10368 }
10369
10370 /*
10371 * Check for single stepping event if we're stepping.
10372 */
10373 if (pVCpu->hm.s.fSingleInstruction)
10374 {
10375 switch (uExitReason)
10376 {
10377 case VMX_EXIT_MTF:
10378 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
10379
10380 /* Various events: */
10381 case VMX_EXIT_XCPT_OR_NMI:
10382 case VMX_EXIT_EXT_INT:
10383 case VMX_EXIT_TRIPLE_FAULT:
10384 case VMX_EXIT_INT_WINDOW:
10385 case VMX_EXIT_NMI_WINDOW:
10386 case VMX_EXIT_TASK_SWITCH:
10387 case VMX_EXIT_TPR_BELOW_THRESHOLD:
10388 case VMX_EXIT_APIC_ACCESS:
10389 case VMX_EXIT_EPT_VIOLATION:
10390 case VMX_EXIT_EPT_MISCONFIG:
10391 case VMX_EXIT_PREEMPT_TIMER:
10392
10393 /* Instruction specific VM-exits: */
10394 case VMX_EXIT_CPUID:
10395 case VMX_EXIT_GETSEC:
10396 case VMX_EXIT_HLT:
10397 case VMX_EXIT_INVD:
10398 case VMX_EXIT_INVLPG:
10399 case VMX_EXIT_RDPMC:
10400 case VMX_EXIT_RDTSC:
10401 case VMX_EXIT_RSM:
10402 case VMX_EXIT_VMCALL:
10403 case VMX_EXIT_VMCLEAR:
10404 case VMX_EXIT_VMLAUNCH:
10405 case VMX_EXIT_VMPTRLD:
10406 case VMX_EXIT_VMPTRST:
10407 case VMX_EXIT_VMREAD:
10408 case VMX_EXIT_VMRESUME:
10409 case VMX_EXIT_VMWRITE:
10410 case VMX_EXIT_VMXOFF:
10411 case VMX_EXIT_VMXON:
10412 case VMX_EXIT_MOV_CRX:
10413 case VMX_EXIT_MOV_DRX:
10414 case VMX_EXIT_IO_INSTR:
10415 case VMX_EXIT_RDMSR:
10416 case VMX_EXIT_WRMSR:
10417 case VMX_EXIT_MWAIT:
10418 case VMX_EXIT_MONITOR:
10419 case VMX_EXIT_PAUSE:
10420 case VMX_EXIT_XDTR_ACCESS:
10421 case VMX_EXIT_TR_ACCESS:
10422 case VMX_EXIT_INVEPT:
10423 case VMX_EXIT_RDTSCP:
10424 case VMX_EXIT_INVVPID:
10425 case VMX_EXIT_WBINVD:
10426 case VMX_EXIT_XSETBV:
10427 case VMX_EXIT_RDRAND:
10428 case VMX_EXIT_INVPCID:
10429 case VMX_EXIT_VMFUNC:
10430 case VMX_EXIT_RDSEED:
10431 case VMX_EXIT_XSAVES:
10432 case VMX_EXIT_XRSTORS:
10433 {
10434 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10435 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10436 AssertRCReturn(rc2, rc2);
10437 if ( pMixedCtx->rip != pDbgState->uRipStart
10438 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
10439 return VINF_EM_DBG_STEPPED;
10440 break;
10441 }
10442
10443 /* Errors and unexpected events: */
10444 case VMX_EXIT_INIT_SIGNAL:
10445 case VMX_EXIT_SIPI:
10446 case VMX_EXIT_IO_SMI:
10447 case VMX_EXIT_SMI:
10448 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10449 case VMX_EXIT_ERR_MSR_LOAD:
10450 case VMX_EXIT_ERR_MACHINE_CHECK:
10451 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
10452 break;
10453
10454 default:
10455 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10456 break;
10457 }
10458 }
10459
10460 /*
10461 * Check for debugger event breakpoints and dtrace probes.
10462 */
10463 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
10464 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
10465 {
10466 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10467 if (rcStrict != VINF_SUCCESS)
10468 return rcStrict;
10469 }
10470
10471 /*
10472 * Normal processing.
10473 */
10474#ifdef HMVMX_USE_FUNCTION_TABLE
10475 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
10476#else
10477 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10478#endif
10479}
10480
10481
10482/**
10483 * Single steps guest code using VT-x.
10484 *
10485 * @returns Strict VBox status code (i.e. informational status codes too).
10486 * @param pVM The cross context VM structure.
10487 * @param pVCpu The cross context virtual CPU structure.
10488 * @param pCtx Pointer to the guest-CPU context.
10489 *
10490 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10491 */
10492static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10493{
10494 VMXTRANSIENT VmxTransient;
10495 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10496
10497 /* Set HMCPU indicators. */
10498 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10499 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
10500 pVCpu->hm.s.fDebugWantRdTscExit = false;
10501 pVCpu->hm.s.fUsingDebugLoop = true;
10502
10503 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10504 VMXRUNDBGSTATE DbgState;
10505 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10506 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10507
10508 /*
10509 * The loop.
10510 */
10511 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10512 for (uint32_t cLoops = 0; ; cLoops++)
10513 {
10514 Assert(!HMR0SuspendPending());
10515 HMVMX_ASSERT_CPU_SAFE();
10516 bool fStepping = pVCpu->hm.s.fSingleInstruction;
10517
10518 /*
10519 * Preparatory work for running guest code, this may force us to return
10520 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10521 */
10522 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10523 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10524 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10525 if (rcStrict != VINF_SUCCESS)
10526 break;
10527
10528 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10529 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10530
10531 /*
10532 * Now we can run the guest code.
10533 */
10534 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10535
10536 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10537
10538 /*
10539 * Restore any residual host-state and save any bits shared between host
10540 * and guest into the guest-CPU state. Re-enables interrupts!
10541 */
10542 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
10543
10544 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10545 if (RT_SUCCESS(rcRun))
10546 { /* very likely */ }
10547 else
10548 {
10549 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10550 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10551 return rcRun;
10552 }
10553
10554 /* Profile the VM-exit. */
10555 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10556 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10557 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10558 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10559 HMVMX_START_EXIT_DISPATCH_PROF();
10560
10561 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10562
10563 /*
10564 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10565 */
10566 rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10567 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10568 if (rcStrict != VINF_SUCCESS)
10569 break;
10570 if (cLoops > pVM->hm.s.cMaxResumeLoops)
10571 {
10572 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10573 rcStrict = VINF_EM_RAW_INTERRUPT;
10574 break;
10575 }
10576
10577 /*
10578 * Stepping: Did the RIP change, if so, consider it a single step.
10579 * Otherwise, make sure one of the TFs gets set.
10580 */
10581 if (fStepping)
10582 {
10583 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10584 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10585 AssertRCReturn(rc2, rc2);
10586 if ( pCtx->rip != DbgState.uRipStart
10587 || pCtx->cs.Sel != DbgState.uCsStart)
10588 {
10589 rcStrict = VINF_EM_DBG_STEPPED;
10590 break;
10591 }
10592 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10593 }
10594
10595 /*
10596 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
10597 */
10598 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10599 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10600 }
10601
10602 /*
10603 * Clear the X86_EFL_TF if necessary.
10604 */
10605 if (pVCpu->hm.s.fClearTrapFlag)
10606 {
10607 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10608 AssertRCReturn(rc2, rc2);
10609 pVCpu->hm.s.fClearTrapFlag = false;
10610 pCtx->eflags.Bits.u1TF = 0;
10611 }
10612 /** @todo there seems to be issues with the resume flag when the monitor trap
10613 * flag is pending without being used. Seen early in bios init when
10614 * accessing APIC page in protected mode. */
10615
10616 /*
10617 * Restore VM-exit control settings as we may not reenter this function the
10618 * next time around.
10619 */
10620 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10621
10622 /* Restore HMCPU indicators. */
10623 pVCpu->hm.s.fUsingDebugLoop = false;
10624 pVCpu->hm.s.fDebugWantRdTscExit = false;
10625 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10626
10627 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10628 return rcStrict;
10629}
10630
10631
10632/** @} */
10633
10634
10635/**
10636 * Checks if any expensive dtrace probes are enabled and we should go to the
10637 * debug loop.
10638 *
10639 * @returns true if we should use debug loop, false if not.
10640 */
10641static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10642{
10643 /* It's probably faster to OR the raw 32-bit counter variables together.
10644 Since the variables are in an array and the probes are next to one
10645 another (more or less), we have good locality. So, better read
10646 eight-nine cache lines ever time and only have one conditional, than
10647 128+ conditionals, right? */
10648 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10649 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10650 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10651 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10652 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10653 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10654 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10655 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10656 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10657 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10658 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10659 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10660 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10661 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10662 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10663 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10664 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10665 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10666 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10667 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10668 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10669 ) != 0
10670 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10671 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10672 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10673 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10674 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10675 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10676 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10677 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10678 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10679 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10680 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10681 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10682 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10683 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10684 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10685 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10686 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10687 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10688 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10689 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10690 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10691 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10692 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10693 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10694 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10695 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10696 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10697 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10698 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10699 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10700 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10701 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10702 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10703 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10704 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10705 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10706 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10707 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10708 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10709 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10710 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10711 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10712 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10713 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10714 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10715 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10716 ) != 0
10717 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10718 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10719 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10720 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10721 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10722 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10723 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10724 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10725 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10726 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10727 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10728 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10729 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10730 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10731 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10732 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10733 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10734 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10735 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10736 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10737 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10738 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10739 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10740 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10741 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10742 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10743 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10744 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10745 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10746 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10747 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10748 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10749 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10750 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10751 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10752 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10753 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10754 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10755 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10756 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10757 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10758 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10759 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10760 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10761 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10762 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10763 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10764 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10765 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10766 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10767 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10768 ) != 0;
10769}
10770
10771
10772/**
10773 * Runs the guest code using VT-x.
10774 *
10775 * @returns Strict VBox status code (i.e. informational status codes too).
10776 * @param pVM The cross context VM structure.
10777 * @param pVCpu The cross context virtual CPU structure.
10778 * @param pCtx Pointer to the guest-CPU context.
10779 */
10780VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10781{
10782 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10783 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10784 HMVMX_ASSERT_PREEMPT_SAFE();
10785
10786 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10787
10788 VBOXSTRICTRC rcStrict;
10789 if ( !pVCpu->hm.s.fUseDebugLoop
10790 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10791 && !DBGFIsStepping(pVCpu)
10792 && !pVM->dbgf.ro.cEnabledInt3Breakpoints)
10793 rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10794 else
10795 rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10796
10797 if (rcStrict == VERR_EM_INTERPRETER)
10798 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10799 else if (rcStrict == VINF_EM_RESET)
10800 rcStrict = VINF_EM_TRIPLE_FAULT;
10801
10802 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10803 if (RT_FAILURE(rc2))
10804 {
10805 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10806 rcStrict = rc2;
10807 }
10808 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10809 return rcStrict;
10810}
10811
10812
10813#ifndef HMVMX_USE_FUNCTION_TABLE
10814DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10815{
10816# ifdef DEBUG_ramshankar
10817# define RETURN_EXIT_CALL(a_CallExpr) \
10818 do { \
10819 int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10820 VBOXSTRICTRC rcStrict = a_CallExpr; \
10821 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10822 return rcStrict; \
10823 } while (0)
10824# else
10825# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10826# endif
10827 switch (rcReason)
10828 {
10829 case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10830 case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10831 case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10832 case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10833 case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10834 case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10835 case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10836 case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10837 case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10838 case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10839 case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10840 case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10841 case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10842 case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10843 case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10844 case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10845 case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10846 case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10847 case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10848 case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10849 case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10850 case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10851 case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10852 case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10853 case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10854 case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10855 case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10856 case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10857 case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10858 case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10859 case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10860 case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10861 case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10862 case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10863
10864 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10865 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10866 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10867 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10868 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10869 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10870 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10871 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10872 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10873
10874 case VMX_EXIT_VMCLEAR:
10875 case VMX_EXIT_VMLAUNCH:
10876 case VMX_EXIT_VMPTRLD:
10877 case VMX_EXIT_VMPTRST:
10878 case VMX_EXIT_VMREAD:
10879 case VMX_EXIT_VMRESUME:
10880 case VMX_EXIT_VMWRITE:
10881 case VMX_EXIT_VMXOFF:
10882 case VMX_EXIT_VMXON:
10883 case VMX_EXIT_INVEPT:
10884 case VMX_EXIT_INVVPID:
10885 case VMX_EXIT_VMFUNC:
10886 case VMX_EXIT_XSAVES:
10887 case VMX_EXIT_XRSTORS:
10888 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10889 case VMX_EXIT_ENCLS:
10890 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10891 case VMX_EXIT_PML_FULL:
10892 default:
10893 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10894 }
10895#undef RETURN_EXIT_CALL
10896}
10897#endif /* !HMVMX_USE_FUNCTION_TABLE */
10898
10899
10900#ifdef VBOX_STRICT
10901/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10902# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10903 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10904
10905# define HMVMX_ASSERT_PREEMPT_CPUID() \
10906 do { \
10907 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10908 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10909 } while (0)
10910
10911# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10912 do { \
10913 AssertPtr(pVCpu); \
10914 AssertPtr(pMixedCtx); \
10915 AssertPtr(pVmxTransient); \
10916 Assert(pVmxTransient->fVMEntryFailed == false); \
10917 Assert(ASMIntAreEnabled()); \
10918 HMVMX_ASSERT_PREEMPT_SAFE(); \
10919 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10920 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10921 HMVMX_ASSERT_PREEMPT_SAFE(); \
10922 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10923 HMVMX_ASSERT_PREEMPT_CPUID(); \
10924 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10925 } while (0)
10926
10927# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10928 do { \
10929 Log4Func(("\n")); \
10930 } while (0)
10931#else /* nonstrict builds: */
10932# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10933 do { \
10934 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10935 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10936 } while (0)
10937# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10938#endif
10939
10940
10941/**
10942 * Advances the guest RIP by the specified number of bytes.
10943 *
10944 * @param pVCpu The cross context virtual CPU structure.
10945 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10946 * out-of-sync. Make sure to update the required fields
10947 * before using them.
10948 * @param cbInstr Number of bytes to advance the RIP by.
10949 *
10950 * @remarks No-long-jump zone!!!
10951 */
10952DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10953{
10954 /* Advance the RIP. */
10955 pMixedCtx->rip += cbInstr;
10956 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10957
10958 /* Update interrupt inhibition. */
10959 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10960 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10961 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10962}
10963
10964
10965/**
10966 * Advances the guest RIP after reading it from the VMCS.
10967 *
10968 * @returns VBox status code, no informational status codes.
10969 * @param pVCpu The cross context virtual CPU structure.
10970 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10971 * out-of-sync. Make sure to update the required fields
10972 * before using them.
10973 * @param pVmxTransient Pointer to the VMX transient structure.
10974 *
10975 * @remarks No-long-jump zone!!!
10976 */
10977static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10978{
10979 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10980 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10981 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10982 AssertRCReturn(rc, rc);
10983
10984 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10985
10986 /*
10987 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10988 * pending debug exception field as it takes care of priority of events.
10989 *
10990 * See Intel spec. 32.2.1 "Debug Exceptions".
10991 */
10992 if ( !pVCpu->hm.s.fSingleInstruction
10993 && pMixedCtx->eflags.Bits.u1TF)
10994 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10995
10996 return VINF_SUCCESS;
10997}
10998
10999
11000/**
11001 * Tries to determine what part of the guest-state VT-x has deemed as invalid
11002 * and update error record fields accordingly.
11003 *
11004 * @return VMX_IGS_* return codes.
11005 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
11006 * wrong with the guest state.
11007 *
11008 * @param pVM The cross context VM structure.
11009 * @param pVCpu The cross context virtual CPU structure.
11010 * @param pCtx Pointer to the guest-CPU state.
11011 *
11012 * @remarks This function assumes our cache of the VMCS controls
11013 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
11014 */
11015static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
11016{
11017#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
11018#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
11019 uError = (err); \
11020 break; \
11021 } else do { } while (0)
11022
11023 int rc;
11024 uint32_t uError = VMX_IGS_ERROR;
11025 uint32_t u32Val;
11026 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
11027
11028 do
11029 {
11030 /*
11031 * CR0.
11032 */
11033 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
11034 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
11035 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
11036 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
11037 if (fUnrestrictedGuest)
11038 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
11039
11040 uint32_t u32GuestCR0;
11041 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
11042 AssertRCBreak(rc);
11043 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
11044 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
11045 if ( !fUnrestrictedGuest
11046 && (u32GuestCR0 & X86_CR0_PG)
11047 && !(u32GuestCR0 & X86_CR0_PE))
11048 {
11049 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
11050 }
11051
11052 /*
11053 * CR4.
11054 */
11055 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
11056 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
11057
11058 uint32_t u32GuestCR4;
11059 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
11060 AssertRCBreak(rc);
11061 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
11062 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
11063
11064 /*
11065 * IA32_DEBUGCTL MSR.
11066 */
11067 uint64_t u64Val;
11068 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
11069 AssertRCBreak(rc);
11070 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
11071 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
11072 {
11073 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
11074 }
11075 uint64_t u64DebugCtlMsr = u64Val;
11076
11077#ifdef VBOX_STRICT
11078 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
11079 AssertRCBreak(rc);
11080 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
11081#endif
11082 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
11083
11084 /*
11085 * RIP and RFLAGS.
11086 */
11087 uint32_t u32Eflags;
11088#if HC_ARCH_BITS == 64
11089 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
11090 AssertRCBreak(rc);
11091 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
11092 if ( !fLongModeGuest
11093 || !pCtx->cs.Attr.n.u1Long)
11094 {
11095 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
11096 }
11097 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
11098 * must be identical if the "IA-32e mode guest" VM-entry
11099 * control is 1 and CS.L is 1. No check applies if the
11100 * CPU supports 64 linear-address bits. */
11101
11102 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
11103 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
11104 AssertRCBreak(rc);
11105 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
11106 VMX_IGS_RFLAGS_RESERVED);
11107 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
11108 u32Eflags = u64Val;
11109#else
11110 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
11111 AssertRCBreak(rc);
11112 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
11113 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
11114#endif
11115
11116 if ( fLongModeGuest
11117 || ( fUnrestrictedGuest
11118 && !(u32GuestCR0 & X86_CR0_PE)))
11119 {
11120 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
11121 }
11122
11123 uint32_t u32EntryInfo;
11124 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
11125 AssertRCBreak(rc);
11126 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11127 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11128 {
11129 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
11130 }
11131
11132 /*
11133 * 64-bit checks.
11134 */
11135#if HC_ARCH_BITS == 64
11136 if (fLongModeGuest)
11137 {
11138 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
11139 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
11140 }
11141
11142 if ( !fLongModeGuest
11143 && (u32GuestCR4 & X86_CR4_PCIDE))
11144 {
11145 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
11146 }
11147
11148 /** @todo CR3 field must be such that bits 63:52 and bits in the range
11149 * 51:32 beyond the processor's physical-address width are 0. */
11150
11151 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
11152 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
11153 {
11154 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
11155 }
11156
11157 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
11158 AssertRCBreak(rc);
11159 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
11160
11161 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
11162 AssertRCBreak(rc);
11163 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
11164#endif
11165
11166 /*
11167 * PERF_GLOBAL MSR.
11168 */
11169 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
11170 {
11171 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
11172 AssertRCBreak(rc);
11173 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
11174 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
11175 }
11176
11177 /*
11178 * PAT MSR.
11179 */
11180 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
11181 {
11182 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
11183 AssertRCBreak(rc);
11184 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
11185 for (unsigned i = 0; i < 8; i++)
11186 {
11187 uint8_t u8Val = (u64Val & 0xff);
11188 if ( u8Val != 0 /* UC */
11189 && u8Val != 1 /* WC */
11190 && u8Val != 4 /* WT */
11191 && u8Val != 5 /* WP */
11192 && u8Val != 6 /* WB */
11193 && u8Val != 7 /* UC- */)
11194 {
11195 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
11196 }
11197 u64Val >>= 8;
11198 }
11199 }
11200
11201 /*
11202 * EFER MSR.
11203 */
11204 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
11205 {
11206 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
11207 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
11208 AssertRCBreak(rc);
11209 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
11210 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
11211 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
11212 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
11213 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
11214 HMVMX_CHECK_BREAK( fUnrestrictedGuest
11215 || !(u32GuestCR0 & X86_CR0_PG)
11216 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
11217 VMX_IGS_EFER_LMA_LME_MISMATCH);
11218 }
11219
11220 /*
11221 * Segment registers.
11222 */
11223 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11224 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
11225 if (!(u32Eflags & X86_EFL_VM))
11226 {
11227 /* CS */
11228 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
11229 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
11230 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
11231 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
11232 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
11233 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
11234 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
11235 /* CS cannot be loaded with NULL in protected mode. */
11236 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
11237 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
11238 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
11239 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
11240 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
11241 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
11242 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
11243 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
11244 else
11245 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
11246
11247 /* SS */
11248 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11249 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
11250 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
11251 if ( !(pCtx->cr0 & X86_CR0_PE)
11252 || pCtx->cs.Attr.n.u4Type == 3)
11253 {
11254 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
11255 }
11256 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
11257 {
11258 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
11259 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
11260 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
11261 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
11262 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
11263 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
11264 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
11265 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
11266 }
11267
11268 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
11269 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
11270 {
11271 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
11272 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
11273 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11274 || pCtx->ds.Attr.n.u4Type > 11
11275 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
11276 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
11277 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
11278 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
11279 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
11280 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
11281 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
11282 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11283 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
11284 }
11285 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
11286 {
11287 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
11288 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
11289 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11290 || pCtx->es.Attr.n.u4Type > 11
11291 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
11292 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
11293 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
11294 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
11295 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
11296 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
11297 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
11298 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11299 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
11300 }
11301 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
11302 {
11303 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
11304 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
11305 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11306 || pCtx->fs.Attr.n.u4Type > 11
11307 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
11308 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
11309 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
11310 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
11311 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
11312 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
11313 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
11314 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11315 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
11316 }
11317 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
11318 {
11319 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
11320 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
11321 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11322 || pCtx->gs.Attr.n.u4Type > 11
11323 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
11324 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
11325 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
11326 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
11327 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
11328 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
11329 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
11330 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11331 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
11332 }
11333 /* 64-bit capable CPUs. */
11334#if HC_ARCH_BITS == 64
11335 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11336 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11337 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11338 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11339 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11340 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
11341 VMX_IGS_LONGMODE_SS_BASE_INVALID);
11342 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
11343 VMX_IGS_LONGMODE_DS_BASE_INVALID);
11344 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
11345 VMX_IGS_LONGMODE_ES_BASE_INVALID);
11346#endif
11347 }
11348 else
11349 {
11350 /* V86 mode checks. */
11351 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
11352 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11353 {
11354 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
11355 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
11356 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
11357 }
11358 else
11359 {
11360 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
11361 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
11362 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
11363 }
11364
11365 /* CS */
11366 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
11367 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
11368 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
11369 /* SS */
11370 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
11371 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
11372 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
11373 /* DS */
11374 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
11375 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
11376 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
11377 /* ES */
11378 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
11379 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
11380 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
11381 /* FS */
11382 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
11383 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
11384 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
11385 /* GS */
11386 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
11387 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
11388 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
11389 /* 64-bit capable CPUs. */
11390#if HC_ARCH_BITS == 64
11391 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11392 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11393 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11394 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11395 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11396 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
11397 VMX_IGS_LONGMODE_SS_BASE_INVALID);
11398 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
11399 VMX_IGS_LONGMODE_DS_BASE_INVALID);
11400 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
11401 VMX_IGS_LONGMODE_ES_BASE_INVALID);
11402#endif
11403 }
11404
11405 /*
11406 * TR.
11407 */
11408 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
11409 /* 64-bit capable CPUs. */
11410#if HC_ARCH_BITS == 64
11411 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
11412#endif
11413 if (fLongModeGuest)
11414 {
11415 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
11416 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
11417 }
11418 else
11419 {
11420 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
11421 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
11422 VMX_IGS_TR_ATTR_TYPE_INVALID);
11423 }
11424 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
11425 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
11426 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
11427 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
11428 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11429 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
11430 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11431 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
11432
11433 /*
11434 * GDTR and IDTR.
11435 */
11436#if HC_ARCH_BITS == 64
11437 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
11438 AssertRCBreak(rc);
11439 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
11440
11441 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
11442 AssertRCBreak(rc);
11443 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
11444#endif
11445
11446 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
11447 AssertRCBreak(rc);
11448 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11449
11450 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
11451 AssertRCBreak(rc);
11452 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11453
11454 /*
11455 * Guest Non-Register State.
11456 */
11457 /* Activity State. */
11458 uint32_t u32ActivityState;
11459 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
11460 AssertRCBreak(rc);
11461 HMVMX_CHECK_BREAK( !u32ActivityState
11462 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
11463 VMX_IGS_ACTIVITY_STATE_INVALID);
11464 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
11465 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
11466 uint32_t u32IntrState;
11467 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
11468 AssertRCBreak(rc);
11469 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
11470 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11471 {
11472 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11473 }
11474
11475 /** @todo Activity state and injecting interrupts. Left as a todo since we
11476 * currently don't use activity states but ACTIVE. */
11477
11478 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11479 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11480
11481 /* Guest interruptibility-state. */
11482 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11483 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11484 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
11485 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11486 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11487 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11488 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11489 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11490 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11491 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
11492 {
11493 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11494 {
11495 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11496 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11497 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11498 }
11499 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11500 {
11501 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11502 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11503 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11504 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11505 }
11506 }
11507 /** @todo Assumes the processor is not in SMM. */
11508 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11509 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11510 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11511 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11512 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11513 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11514 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11515 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11516 {
11517 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11518 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11519 }
11520
11521 /* Pending debug exceptions. */
11522#if HC_ARCH_BITS == 64
11523 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11524 AssertRCBreak(rc);
11525 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11526 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11527 u32Val = u64Val; /* For pending debug exceptions checks below. */
11528#else
11529 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11530 AssertRCBreak(rc);
11531 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11532 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11533#endif
11534
11535 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11536 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11537 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11538 {
11539 if ( (u32Eflags & X86_EFL_TF)
11540 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11541 {
11542 /* Bit 14 is PendingDebug.BS. */
11543 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11544 }
11545 if ( !(u32Eflags & X86_EFL_TF)
11546 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11547 {
11548 /* Bit 14 is PendingDebug.BS. */
11549 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11550 }
11551 }
11552
11553 /* VMCS link pointer. */
11554 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11555 AssertRCBreak(rc);
11556 if (u64Val != UINT64_C(0xffffffffffffffff))
11557 {
11558 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11559 /** @todo Bits beyond the processor's physical-address width MBZ. */
11560 /** @todo 32-bit located in memory referenced by value of this field (as a
11561 * physical address) must contain the processor's VMCS revision ID. */
11562 /** @todo SMM checks. */
11563 }
11564
11565 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11566 * not using Nested Paging? */
11567 if ( pVM->hm.s.fNestedPaging
11568 && !fLongModeGuest
11569 && CPUMIsGuestInPAEModeEx(pCtx))
11570 {
11571 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11572 AssertRCBreak(rc);
11573 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11574
11575 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11576 AssertRCBreak(rc);
11577 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11578
11579 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11580 AssertRCBreak(rc);
11581 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11582
11583 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11584 AssertRCBreak(rc);
11585 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11586 }
11587
11588 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11589 if (uError == VMX_IGS_ERROR)
11590 uError = VMX_IGS_REASON_NOT_FOUND;
11591 } while (0);
11592
11593 pVCpu->hm.s.u32HMError = uError;
11594 return uError;
11595
11596#undef HMVMX_ERROR_BREAK
11597#undef HMVMX_CHECK_BREAK
11598}
11599
11600/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11601/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11602/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11603
11604/** @name VM-exit handlers.
11605 * @{
11606 */
11607
11608/**
11609 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11610 */
11611HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11612{
11613 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11614 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11615 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11616 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11617 return VINF_SUCCESS;
11618 return VINF_EM_RAW_INTERRUPT;
11619}
11620
11621
11622/**
11623 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11624 */
11625HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11626{
11627 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11628 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11629
11630 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11631 AssertRCReturn(rc, rc);
11632
11633 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11634 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11635 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11636 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11637
11638 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11639 {
11640 /*
11641 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11642 * anything we inject is not going to cause a VM-exit directly for the event being injected.
11643 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11644 *
11645 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11646 */
11647 VMXDispatchHostNmi();
11648 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11649 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11650 return VINF_SUCCESS;
11651 }
11652
11653 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11654 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11655 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11656 { /* likely */ }
11657 else
11658 {
11659 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11660 rcStrictRc1 = VINF_SUCCESS;
11661 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11662 return rcStrictRc1;
11663 }
11664
11665 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11666 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11667 switch (uIntType)
11668 {
11669 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11670 Assert(uVector == X86_XCPT_DB);
11671 RT_FALL_THRU();
11672 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11673 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11674 RT_FALL_THRU();
11675 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11676 {
11677 /*
11678 * If there's any exception caused as a result of event injection, the resulting
11679 * secondary/final execption will be pending, we shall continue guest execution
11680 * after injecting the event. The page-fault case is complicated and we manually
11681 * handle any currently pending event in hmR0VmxExitXcptPF.
11682 */
11683 if (!pVCpu->hm.s.Event.fPending)
11684 { /* likely */ }
11685 else if (uVector != X86_XCPT_PF)
11686 {
11687 rc = VINF_SUCCESS;
11688 break;
11689 }
11690
11691 switch (uVector)
11692 {
11693 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11694 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11695 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11696 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11697 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11698 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11699
11700 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11701 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11702 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11703 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11704 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11705 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11706 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11707 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11708 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11709 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11710 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11711 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11712 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11713 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11714 default:
11715 {
11716 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11717 AssertRCReturn(rc, rc);
11718
11719 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11720 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11721 {
11722 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11723 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11724 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11725
11726 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11727 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11728 AssertRCReturn(rc, rc);
11729 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11730 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11731 0 /* GCPtrFaultAddress */);
11732 AssertRCReturn(rc, rc);
11733 }
11734 else
11735 {
11736 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11737 pVCpu->hm.s.u32HMError = uVector;
11738 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11739 }
11740 break;
11741 }
11742 }
11743 break;
11744 }
11745
11746 default:
11747 {
11748 pVCpu->hm.s.u32HMError = uExitIntInfo;
11749 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11750 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11751 break;
11752 }
11753 }
11754 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11755 return rc;
11756}
11757
11758
11759/**
11760 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11761 */
11762HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11763{
11764 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11765
11766 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11767 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11768
11769 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11770 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11771 return VINF_SUCCESS;
11772}
11773
11774
11775/**
11776 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11777 */
11778HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11779{
11780 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11781 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11782 {
11783 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11784 HMVMX_RETURN_UNEXPECTED_EXIT();
11785 }
11786
11787 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11788
11789 /*
11790 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11791 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11792 */
11793 uint32_t uIntrState = 0;
11794 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11795 AssertRCReturn(rc, rc);
11796
11797 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11798 if ( fBlockSti
11799 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11800 {
11801 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11802 }
11803
11804 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11805 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11806
11807 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11808 return VINF_SUCCESS;
11809}
11810
11811
11812/**
11813 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11814 */
11815HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11816{
11817 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11818 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11819 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11820}
11821
11822
11823/**
11824 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11825 */
11826HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11827{
11828 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11829 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11830 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11831}
11832
11833
11834/**
11835 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11836 */
11837HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11838{
11839 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11840 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11841 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
11842
11843 /*
11844 * Get the state we need and update the exit history entry.
11845 */
11846 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11847 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11848 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11849 rc |= hmR0VmxSaveGuestCs(pVCpu);
11850 AssertRCReturn(rc, rc);
11851
11852 VBOXSTRICTRC rcStrict;
11853 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu, EMEXIT_MAKE_FLAGS_AND_TYPE(EMEXIT_F_KIND_EM, EMEXITTYPE_CPUID),
11854 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11855 if (!pExitRec)
11856 {
11857 /*
11858 * Regular CPUID instruction execution.
11859 */
11860 PVM pVM = pVCpu->CTX_SUFF(pVM);
11861 rcStrict = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11862 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11863 {
11864 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11865 Assert(pVmxTransient->cbInstr == 2);
11866 }
11867 else
11868 {
11869 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11870 rcStrict = VERR_EM_INTERPRETER;
11871 }
11872 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11873 }
11874 else
11875 {
11876 /*
11877 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
11878 */
11879 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
11880 int rc2 = hmR0VmxSaveGuestRegsForIemInterpreting(pVCpu);
11881 AssertRCReturn(rc2, rc2);
11882
11883 Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11884 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11885
11886 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11887
11888 Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11889 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11890 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11891 }
11892 return VBOXSTRICTRC_TODO(rcStrict);
11893}
11894
11895
11896/**
11897 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11898 */
11899HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11900{
11901 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11902 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11903 AssertRCReturn(rc, rc);
11904
11905 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11906 return VINF_EM_RAW_EMULATE_INSTR;
11907
11908 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11909 HMVMX_RETURN_UNEXPECTED_EXIT();
11910}
11911
11912
11913/**
11914 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11915 */
11916HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11917{
11918 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11919#if 1 /** @todo Needs testing. @bugref{6973} */
11920 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /* Needed for CPL < 0 only, really. */
11921 rc |= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, false /*fNeedRsp*/);
11922 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11923 AssertRCReturn(rc, rc);
11924 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11925 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11926 {
11927 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11928 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11929 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11930 }
11931 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11932 rcStrict = VINF_SUCCESS;
11933 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
11934 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11935 return rcStrict;
11936#else
11937 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11938 AssertRCReturn(rc, rc);
11939
11940 PVM pVM = pVCpu->CTX_SUFF(pVM);
11941 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11942 if (RT_LIKELY(rc == VINF_SUCCESS))
11943 {
11944 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11945 Assert(pVmxTransient->cbInstr == 2);
11946 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11947 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11948 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11949 }
11950 else
11951 rc = VERR_EM_INTERPRETER;
11952 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11953 return rc;
11954#endif
11955}
11956
11957
11958/**
11959 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11960 */
11961HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11962{
11963 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11964#if 1 /** @todo Needs testing. @bugref{6973} */
11965 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /* Needed for CPL < 0 only, really. */
11966 rc |= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, false /*fNeedRsp*/);
11967 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11968 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11969 AssertRCReturn(rc, rc);
11970 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11971 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11972 {
11973 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11974 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11975 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11976 }
11977 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11978 rcStrict = VINF_SUCCESS;
11979 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
11980 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtscp);
11981 return rcStrict;
11982#else
11983 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11984 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11985 AssertRCReturn(rc, rc);
11986
11987 PVM pVM = pVCpu->CTX_SUFF(pVM);
11988 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11989 if (RT_SUCCESS(rc))
11990 {
11991 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11992 Assert(pVmxTransient->cbInstr == 3);
11993 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11994 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11995 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11996 }
11997 else
11998 {
11999 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
12000 rc = VERR_EM_INTERPRETER;
12001 }
12002 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtscp);
12003 return rc;
12004#endif
12005}
12006
12007
12008/**
12009 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
12010 */
12011HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12012{
12013 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12014 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
12015 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12016 AssertRCReturn(rc, rc);
12017
12018 PVM pVM = pVCpu->CTX_SUFF(pVM);
12019 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12020 if (RT_LIKELY(rc == VINF_SUCCESS))
12021 {
12022 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12023 Assert(pVmxTransient->cbInstr == 2);
12024 }
12025 else
12026 {
12027 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
12028 rc = VERR_EM_INTERPRETER;
12029 }
12030 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
12031 return rc;
12032}
12033
12034
12035/**
12036 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
12037 */
12038HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12039{
12040 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12041 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
12042
12043 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
12044 if (EMAreHypercallInstructionsEnabled(pVCpu))
12045 {
12046#if 0
12047 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12048#else
12049 /* Aggressive state sync. for now. */
12050 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12051 rc |= hmR0VmxSaveGuestRflags(pVCpu,pMixedCtx); /* For CPL checks in gimHvHypercall() & gimKvmHypercall() */
12052 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
12053 AssertRCReturn(rc, rc);
12054#endif
12055
12056 /* Perform the hypercall. */
12057 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
12058 if (rcStrict == VINF_SUCCESS)
12059 {
12060 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12061 AssertRCReturn(rc, rc);
12062 }
12063 else
12064 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
12065 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
12066 || RT_FAILURE(rcStrict));
12067
12068 /* If the hypercall changes anything other than guest's general-purpose registers,
12069 we would need to reload the guest changed bits here before VM-entry. */
12070 }
12071 else
12072 Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
12073
12074 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
12075 if (RT_FAILURE(rcStrict))
12076 {
12077 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
12078 rcStrict = VINF_SUCCESS;
12079 }
12080
12081 return rcStrict;
12082}
12083
12084
12085/**
12086 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
12087 */
12088HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12089{
12090 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12091 PVM pVM = pVCpu->CTX_SUFF(pVM);
12092 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
12093
12094 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12095 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12096 AssertRCReturn(rc, rc);
12097
12098 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
12099 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
12100 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12101 else
12102 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
12103 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
12104 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
12105 return rcStrict;
12106}
12107
12108
12109/**
12110 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
12111 */
12112HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12113{
12114 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12115 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12116 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12117 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12118 AssertRCReturn(rc, rc);
12119
12120 PVM pVM = pVCpu->CTX_SUFF(pVM);
12121 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12122 if (RT_LIKELY(rc == VINF_SUCCESS))
12123 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12124 else
12125 {
12126 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
12127 rc = VERR_EM_INTERPRETER;
12128 }
12129 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
12130 return rc;
12131}
12132
12133
12134/**
12135 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
12136 */
12137HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12138{
12139 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12140 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12141 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12142 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12143 AssertRCReturn(rc, rc);
12144
12145 PVM pVM = pVCpu->CTX_SUFF(pVM);
12146 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12147 rc = VBOXSTRICTRC_VAL(rc2);
12148 if (RT_LIKELY( rc == VINF_SUCCESS
12149 || rc == VINF_EM_HALT))
12150 {
12151 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12152 AssertRCReturn(rc3, rc3);
12153
12154 if ( rc == VINF_EM_HALT
12155 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
12156 {
12157 rc = VINF_SUCCESS;
12158 }
12159 }
12160 else
12161 {
12162 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
12163 rc = VERR_EM_INTERPRETER;
12164 }
12165 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
12166 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
12167 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
12168 return rc;
12169}
12170
12171
12172/**
12173 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
12174 */
12175HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12176{
12177 /*
12178 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
12179 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
12180 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
12181 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
12182 */
12183 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12184 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12185 HMVMX_RETURN_UNEXPECTED_EXIT();
12186}
12187
12188
12189/**
12190 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
12191 */
12192HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12193{
12194 /*
12195 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
12196 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
12197 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
12198 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
12199 */
12200 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12201 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12202 HMVMX_RETURN_UNEXPECTED_EXIT();
12203}
12204
12205
12206/**
12207 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
12208 */
12209HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12210{
12211 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
12212 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12213 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12214 HMVMX_RETURN_UNEXPECTED_EXIT();
12215}
12216
12217
12218/**
12219 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
12220 */
12221HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12222{
12223 /*
12224 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
12225 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
12226 * See Intel spec. 25.3 "Other Causes of VM-exits".
12227 */
12228 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12229 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12230 HMVMX_RETURN_UNEXPECTED_EXIT();
12231}
12232
12233
12234/**
12235 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
12236 * VM-exit.
12237 */
12238HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12239{
12240 /*
12241 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
12242 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
12243 *
12244 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
12245 * See Intel spec. "23.8 Restrictions on VMX operation".
12246 */
12247 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12248 return VINF_SUCCESS;
12249}
12250
12251
12252/**
12253 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
12254 * VM-exit.
12255 */
12256HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12257{
12258 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12259 return VINF_EM_RESET;
12260}
12261
12262
12263/**
12264 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
12265 */
12266HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12267{
12268 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12269 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
12270
12271 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12272 AssertRCReturn(rc, rc);
12273
12274 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
12275 rc = VINF_SUCCESS;
12276 else
12277 rc = VINF_EM_HALT;
12278
12279 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
12280 if (rc != VINF_SUCCESS)
12281 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
12282 return rc;
12283}
12284
12285
12286/**
12287 * VM-exit handler for instructions that result in a \#UD exception delivered to
12288 * the guest.
12289 */
12290HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12291{
12292 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12293 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
12294 return VINF_SUCCESS;
12295}
12296
12297
12298/**
12299 * VM-exit handler for expiry of the VMX preemption timer.
12300 */
12301HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12302{
12303 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12304
12305 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
12306 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12307
12308 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
12309 PVM pVM = pVCpu->CTX_SUFF(pVM);
12310 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
12311 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
12312 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
12313}
12314
12315
12316/**
12317 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
12318 */
12319HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12320{
12321 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12322
12323 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12324 rc |= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, false /*fNeedRsp*/);
12325 rc |= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
12326 AssertRCReturn(rc, rc);
12327
12328 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
12329 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12330
12331 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
12332
12333 return rcStrict;
12334}
12335
12336
12337/**
12338 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
12339 */
12340HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12341{
12342 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12343 /** @todo Use VM-exit instruction information. */
12344 return VERR_EM_INTERPRETER;
12345}
12346
12347
12348/**
12349 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
12350 * Error VM-exit.
12351 */
12352HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12353{
12354 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12355 AssertRCReturn(rc, rc);
12356
12357 rc = hmR0VmxCheckVmcsCtls(pVCpu);
12358 AssertRCReturn(rc, rc);
12359
12360 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12361 NOREF(uInvalidReason);
12362
12363#ifdef VBOX_STRICT
12364 uint32_t uIntrState;
12365 RTHCUINTREG uHCReg;
12366 uint64_t u64Val;
12367 uint32_t u32Val;
12368
12369 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
12370 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
12371 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
12372 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
12373 AssertRCReturn(rc, rc);
12374
12375 Log4(("uInvalidReason %u\n", uInvalidReason));
12376 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
12377 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
12378 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
12379 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
12380
12381 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
12382 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
12383 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
12384 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
12385 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
12386 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
12387 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
12388 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
12389 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
12390 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
12391 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
12392 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
12393#else
12394 NOREF(pVmxTransient);
12395#endif
12396
12397 hmR0DumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12398 return VERR_VMX_INVALID_GUEST_STATE;
12399}
12400
12401
12402/**
12403 * VM-exit handler for VM-entry failure due to an MSR-load
12404 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
12405 */
12406HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12407{
12408 NOREF(pVmxTransient);
12409 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12410 HMVMX_RETURN_UNEXPECTED_EXIT();
12411}
12412
12413
12414/**
12415 * VM-exit handler for VM-entry failure due to a machine-check event
12416 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
12417 */
12418HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12419{
12420 NOREF(pVmxTransient);
12421 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12422 HMVMX_RETURN_UNEXPECTED_EXIT();
12423}
12424
12425
12426/**
12427 * VM-exit handler for all undefined reasons. Should never ever happen.. in
12428 * theory.
12429 */
12430HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12431{
12432 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
12433 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
12434 return VERR_VMX_UNDEFINED_EXIT_CODE;
12435}
12436
12437
12438/**
12439 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
12440 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
12441 * Conditional VM-exit.
12442 */
12443HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12444{
12445 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12446
12447 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
12448 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
12449 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
12450 return VERR_EM_INTERPRETER;
12451 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12452 HMVMX_RETURN_UNEXPECTED_EXIT();
12453}
12454
12455
12456/**
12457 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
12458 */
12459HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12460{
12461 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12462
12463 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
12464 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
12465 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
12466 return VERR_EM_INTERPRETER;
12467 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12468 HMVMX_RETURN_UNEXPECTED_EXIT();
12469}
12470
12471
12472/**
12473 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
12474 */
12475HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12476{
12477 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12478
12479 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
12480 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12481 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12482 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12483 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12484 {
12485 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12486 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12487 }
12488 AssertRCReturn(rc, rc);
12489 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
12490
12491#ifdef VBOX_STRICT
12492 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
12493 {
12494 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
12495 && pMixedCtx->ecx != MSR_K6_EFER)
12496 {
12497 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12498 pMixedCtx->ecx));
12499 HMVMX_RETURN_UNEXPECTED_EXIT();
12500 }
12501 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12502 {
12503 VMXMSREXITREAD enmRead;
12504 VMXMSREXITWRITE enmWrite;
12505 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12506 AssertRCReturn(rc2, rc2);
12507 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
12508 {
12509 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12510 HMVMX_RETURN_UNEXPECTED_EXIT();
12511 }
12512 }
12513 }
12514#endif
12515
12516 PVM pVM = pVCpu->CTX_SUFF(pVM);
12517 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12518 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
12519 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
12520 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
12521 if (RT_SUCCESS(rc))
12522 {
12523 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12524 Assert(pVmxTransient->cbInstr == 2);
12525 }
12526 return rc;
12527}
12528
12529
12530/**
12531 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12532 */
12533HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12534{
12535 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12536 PVM pVM = pVCpu->CTX_SUFF(pVM);
12537 int rc = VINF_SUCCESS;
12538
12539 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
12540 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12541 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12542 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12543 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12544 {
12545 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12546 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12547 }
12548 AssertRCReturn(rc, rc);
12549 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
12550
12551 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12552 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
12553 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12554
12555 if (RT_SUCCESS(rc))
12556 {
12557 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12558
12559 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
12560 if ( pMixedCtx->ecx == MSR_IA32_APICBASE
12561 || ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
12562 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
12563 {
12564 /*
12565 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12566 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
12567 * EMInterpretWrmsr() changes it.
12568 */
12569 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_APIC_STATE);
12570 }
12571 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12572 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12573 else if (pMixedCtx->ecx == MSR_K6_EFER)
12574 {
12575 /*
12576 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12577 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12578 * the other bits as well, SCE and NXE. See @bugref{7368}.
12579 */
12580 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
12581 }
12582
12583 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12584 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12585 {
12586 switch (pMixedCtx->ecx)
12587 {
12588 /*
12589 * For SYSENTER CS, EIP, ESP MSRs, we set both the flags here so we don't accidentally
12590 * overwrite the changed guest-CPU context value while going to ring-3, see @bufref{8745}.
12591 */
12592 case MSR_IA32_SYSENTER_CS:
12593 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
12594 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
12595 break;
12596 case MSR_IA32_SYSENTER_EIP:
12597 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
12598 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
12599 break;
12600 case MSR_IA32_SYSENTER_ESP:
12601 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
12602 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
12603 break;
12604 case MSR_K8_FS_BASE: RT_FALL_THRU();
12605 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
12606 case MSR_K6_EFER: /* already handled above */ break;
12607 default:
12608 {
12609 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12610 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12611 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12612 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMM_GUEST_LAZY_MSRS);
12613 break;
12614 }
12615 }
12616 }
12617#ifdef VBOX_STRICT
12618 else
12619 {
12620 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12621 switch (pMixedCtx->ecx)
12622 {
12623 case MSR_IA32_SYSENTER_CS:
12624 case MSR_IA32_SYSENTER_EIP:
12625 case MSR_IA32_SYSENTER_ESP:
12626 case MSR_K8_FS_BASE:
12627 case MSR_K8_GS_BASE:
12628 {
12629 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12630 HMVMX_RETURN_UNEXPECTED_EXIT();
12631 }
12632
12633 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12634 default:
12635 {
12636 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12637 {
12638 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12639 if (pMixedCtx->ecx != MSR_K6_EFER)
12640 {
12641 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12642 pMixedCtx->ecx));
12643 HMVMX_RETURN_UNEXPECTED_EXIT();
12644 }
12645 }
12646
12647 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12648 {
12649 VMXMSREXITREAD enmRead;
12650 VMXMSREXITWRITE enmWrite;
12651 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12652 AssertRCReturn(rc2, rc2);
12653 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12654 {
12655 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12656 HMVMX_RETURN_UNEXPECTED_EXIT();
12657 }
12658 }
12659 break;
12660 }
12661 }
12662 }
12663#endif /* VBOX_STRICT */
12664 }
12665 return rc;
12666}
12667
12668
12669/**
12670 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12671 */
12672HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12673{
12674 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12675
12676 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12677 return VINF_EM_RAW_INTERRUPT;
12678}
12679
12680
12681/**
12682 * VM-exit handler for when the TPR value is lowered below the specified
12683 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12684 */
12685HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12686{
12687 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12688 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12689
12690 /*
12691 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12692 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12693 */
12694 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12695 return VINF_SUCCESS;
12696}
12697
12698
12699/**
12700 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12701 * VM-exit.
12702 *
12703 * @retval VINF_SUCCESS when guest execution can continue.
12704 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12705 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12706 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12707 * interpreter.
12708 */
12709HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12710{
12711 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12712 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12713 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12714 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12715 AssertRCReturn(rc, rc);
12716
12717 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12718 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12719 PVM pVM = pVCpu->CTX_SUFF(pVM);
12720 VBOXSTRICTRC rcStrict;
12721 rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, true /*fNeedRsp*/);
12722 switch (uAccessType)
12723 {
12724 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12725 {
12726 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12727 AssertRCReturn(rc, rc);
12728
12729 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12730 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12731 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12732 AssertMsg( rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE
12733 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12734 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12735 {
12736 case 0: /* CR0 */
12737 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12738 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12739 break;
12740 case 2: /* CR2 */
12741 /* Nothing to do here, CR2 it's not part of the VMCS. */
12742 break;
12743 case 3: /* CR3 */
12744 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
12745 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12746 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12747 break;
12748 case 4: /* CR4 */
12749 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12750 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12751 VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12752 break;
12753 case 8: /* CR8 */
12754 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12755 /* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12756 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_APIC_STATE);
12757 break;
12758 default:
12759 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12760 break;
12761 }
12762
12763 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12764 break;
12765 }
12766
12767 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12768 {
12769 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12770 AssertRCReturn(rc, rc);
12771
12772 Assert( !pVM->hm.s.fNestedPaging
12773 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12774 || pVCpu->hm.s.fUsingDebugLoop
12775 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12776
12777 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12778 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12779 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12780
12781 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12782 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12783 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12784 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12785 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12786 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12787 VBOXSTRICTRC_VAL(rcStrict)));
12788 if (VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification) == X86_GREG_xSP)
12789 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RSP);
12790 break;
12791 }
12792
12793 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12794 {
12795 AssertRCReturn(rc, rc);
12796 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12797 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12798 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12799 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12800 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12801 break;
12802 }
12803
12804 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12805 {
12806 AssertRCReturn(rc, rc);
12807 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12808 VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12809 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE,
12810 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12811 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12812 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12813 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12814 break;
12815 }
12816
12817 default:
12818 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12819 VERR_VMX_UNEXPECTED_EXCEPTION);
12820 }
12821
12822 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12823 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12824 NOREF(pVM);
12825 return rcStrict;
12826}
12827
12828
12829/**
12830 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12831 * VM-exit.
12832 */
12833HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12834{
12835 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12836 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12837 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
12838
12839 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12840 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12841 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12842 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12843 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12844 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12845 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12846 AssertRCReturn(rc, rc);
12847
12848 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12849 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12850 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12851 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12852 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12853 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12854 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12855 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12856 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12857
12858 /*
12859 * Update exit history to see if this exit can be optimized.
12860 */
12861 VBOXSTRICTRC rcStrict;
12862 PCEMEXITREC pExitRec = NULL;
12863 if ( !fGstStepping
12864 && !fDbgStepping)
12865 pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12866 !fIOString
12867 ? !fIOWrite
12868 ? EMEXIT_MAKE_FLAGS_AND_TYPE(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_READ)
12869 : EMEXIT_MAKE_FLAGS_AND_TYPE(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_WRITE)
12870 : !fIOWrite
12871 ? EMEXIT_MAKE_FLAGS_AND_TYPE(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_STR_READ)
12872 : EMEXIT_MAKE_FLAGS_AND_TYPE(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_STR_WRITE),
12873 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12874 if (!pExitRec)
12875 {
12876 /* I/O operation lookup arrays. */
12877 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12878 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12879
12880 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12881 uint32_t const cbInstr = pVmxTransient->cbInstr;
12882 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12883 PVM pVM = pVCpu->CTX_SUFF(pVM);
12884 if (fIOString)
12885 {
12886 /*
12887 * INS/OUTS - I/O String instruction.
12888 *
12889 * Use instruction-information if available, otherwise fall back on
12890 * interpreting the instruction.
12891 */
12892 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12893 fIOWrite ? 'w' : 'r'));
12894 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12895 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12896 {
12897 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12898 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12899 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12900 AssertRCReturn(rc2, rc2);
12901 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12902 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12903 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12904 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12905 if (fIOWrite)
12906 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12907 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12908 else
12909 {
12910 /*
12911 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12912 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12913 * See Intel Instruction spec. for "INS".
12914 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12915 */
12916 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12917 }
12918 }
12919 else
12920 {
12921 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12922 int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12923 AssertRCReturn(rc2, rc2);
12924 rcStrict = IEMExecOne(pVCpu);
12925 }
12926 /** @todo IEM needs to be setting these flags somehow. */
12927 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12928 fUpdateRipAlready = true;
12929 }
12930 else
12931 {
12932 /*
12933 * IN/OUT - I/O instruction.
12934 */
12935 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12936 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12937 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12938 if (fIOWrite)
12939 {
12940 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12941 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12942 }
12943 else
12944 {
12945 uint32_t u32Result = 0;
12946 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12947 if (IOM_SUCCESS(rcStrict))
12948 {
12949 /* Save result of I/O IN instr. in AL/AX/EAX. */
12950 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12951 }
12952 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12953 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12954 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12955 }
12956 }
12957
12958 if (IOM_SUCCESS(rcStrict))
12959 {
12960 if (!fUpdateRipAlready)
12961 {
12962 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12963 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12964 }
12965
12966 /*
12967 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12968 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12969 */
12970 if (fIOString)
12971 {
12972 /** @todo Single-step for INS/OUTS with REP prefix? */
12973 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12974 }
12975 else if ( !fDbgStepping
12976 && fGstStepping)
12977 {
12978 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12979 }
12980
12981 /*
12982 * If any I/O breakpoints are armed, we need to check if one triggered
12983 * and take appropriate action.
12984 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12985 */
12986 int rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12987 AssertRCReturn(rc2, rc2);
12988
12989 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12990 * execution engines about whether hyper BPs and such are pending. */
12991 uint32_t const uDr7 = pMixedCtx->dr[7];
12992 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12993 && X86_DR7_ANY_RW_IO(uDr7)
12994 && (pMixedCtx->cr4 & X86_CR4_DE))
12995 || DBGFBpIsHwIoArmed(pVM)))
12996 {
12997 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12998
12999 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
13000 VMMRZCallRing3Disable(pVCpu);
13001 HM_DISABLE_PREEMPT();
13002
13003 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
13004
13005 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
13006 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
13007 {
13008 /* Raise #DB. */
13009 if (fIsGuestDbgActive)
13010 ASMSetDR6(pMixedCtx->dr[6]);
13011 if (pMixedCtx->dr[7] != uDr7)
13012 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
13013
13014 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
13015 }
13016 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
13017 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
13018 else if ( rcStrict2 != VINF_SUCCESS
13019 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
13020 rcStrict = rcStrict2;
13021 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
13022
13023 HM_RESTORE_PREEMPT();
13024 VMMRZCallRing3Enable(pVCpu);
13025 }
13026 }
13027
13028#ifdef VBOX_STRICT
13029 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
13030 Assert(!fIOWrite);
13031 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
13032 Assert(fIOWrite);
13033 else
13034 {
13035# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
13036 * statuses, that the VMM device and some others may return. See
13037 * IOM_SUCCESS() for guidance. */
13038 AssertMsg( RT_FAILURE(rcStrict)
13039 || rcStrict == VINF_SUCCESS
13040 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
13041 || rcStrict == VINF_EM_DBG_BREAKPOINT
13042 || rcStrict == VINF_EM_RAW_GUEST_TRAP
13043 || rcStrict == VINF_EM_RAW_TO_R3
13044 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13045# endif
13046 }
13047#endif
13048 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
13049 }
13050 else
13051 {
13052 /*
13053 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
13054 */
13055 int rc2 = hmR0VmxSaveGuestRegsForIemInterpreting(pVCpu);
13056 AssertRCReturn(rc2, rc2);
13057
13058 Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
13059 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13060 VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification) ? "REP " : "",
13061 fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
13062
13063 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
13064
13065 Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
13066 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13067 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
13068 }
13069 return rcStrict;
13070}
13071
13072
13073/**
13074 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
13075 * VM-exit.
13076 */
13077HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13078{
13079 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13080
13081 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
13082 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13083 AssertRCReturn(rc, rc);
13084 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
13085 {
13086 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
13087 AssertRCReturn(rc, rc);
13088 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
13089 {
13090 uint32_t uErrCode;
13091 RTGCUINTPTR GCPtrFaultAddress;
13092 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
13093 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
13094 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
13095 if (fErrorCodeValid)
13096 {
13097 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
13098 AssertRCReturn(rc, rc);
13099 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
13100 }
13101 else
13102 uErrCode = 0;
13103
13104 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
13105 && uVector == X86_XCPT_PF)
13106 GCPtrFaultAddress = pMixedCtx->cr2;
13107 else
13108 GCPtrFaultAddress = 0;
13109
13110 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
13111 0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
13112
13113 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
13114 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
13115 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13116 }
13117 }
13118
13119 /* Fall back to the interpreter to emulate the task-switch. */
13120 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
13121 return VERR_EM_INTERPRETER;
13122}
13123
13124
13125/**
13126 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
13127 */
13128HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13129{
13130 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13131 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
13132 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
13133 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
13134 AssertRCReturn(rc, rc);
13135 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
13136 return VINF_EM_DBG_STEPPED;
13137}
13138
13139
13140/**
13141 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
13142 */
13143HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13144{
13145 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13146
13147 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
13148
13149 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13150 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13151 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13152 {
13153 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
13154 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13155 {
13156 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
13157 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13158 }
13159 }
13160 else
13161 {
13162 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13163 rcStrict1 = VINF_SUCCESS;
13164 return rcStrict1;
13165 }
13166
13167#if 0
13168 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
13169 * just sync the whole thing. */
13170 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13171#else
13172 /* Aggressive state sync. for now. */
13173 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13174 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13175 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13176#endif
13177 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13178 AssertRCReturn(rc, rc);
13179
13180 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
13181 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
13182 VBOXSTRICTRC rcStrict2;
13183 switch (uAccessType)
13184 {
13185 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
13186 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
13187 {
13188 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
13189 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
13190 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
13191
13192 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
13193 GCPhys &= PAGE_BASE_GC_MASK;
13194 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
13195 PVM pVM = pVCpu->CTX_SUFF(pVM);
13196 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
13197 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
13198
13199 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
13200 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
13201 CPUMCTX2CORE(pMixedCtx), GCPhys);
13202 Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
13203 if ( rcStrict2 == VINF_SUCCESS
13204 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13205 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
13206 {
13207 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13208 | HM_CHANGED_GUEST_RSP
13209 | HM_CHANGED_GUEST_RFLAGS
13210 | HM_CHANGED_GUEST_APIC_STATE);
13211 rcStrict2 = VINF_SUCCESS;
13212 }
13213 break;
13214 }
13215
13216 default:
13217 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
13218 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
13219 break;
13220 }
13221
13222 if (rcStrict2 != VINF_SUCCESS)
13223 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
13224 return rcStrict2;
13225}
13226
13227
13228/**
13229 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
13230 * VM-exit.
13231 */
13232HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13233{
13234 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13235
13236 /* We should -not- get this VM-exit if the guest's debug registers were active. */
13237 if (pVmxTransient->fWasGuestDebugStateActive)
13238 {
13239 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
13240 HMVMX_RETURN_UNEXPECTED_EXIT();
13241 }
13242
13243 if ( !pVCpu->hm.s.fSingleInstruction
13244 && !pVmxTransient->fWasHyperDebugStateActive)
13245 {
13246 Assert(!DBGFIsStepping(pVCpu));
13247 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
13248
13249 /* Don't intercept MOV DRx any more. */
13250 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
13251 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
13252 AssertRCReturn(rc, rc);
13253
13254 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
13255 VMMRZCallRing3Disable(pVCpu);
13256 HM_DISABLE_PREEMPT();
13257
13258 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
13259 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
13260 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
13261
13262 HM_RESTORE_PREEMPT();
13263 VMMRZCallRing3Enable(pVCpu);
13264
13265#ifdef VBOX_WITH_STATISTICS
13266 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13267 AssertRCReturn(rc, rc);
13268 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
13269 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
13270 else
13271 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
13272#endif
13273 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
13274 return VINF_SUCCESS;
13275 }
13276
13277 /*
13278 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
13279 * Update the segment registers and DR7 from the CPU.
13280 */
13281 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13282 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13283 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13284 AssertRCReturn(rc, rc);
13285 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13286
13287 PVM pVM = pVCpu->CTX_SUFF(pVM);
13288 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
13289 {
13290 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
13291 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
13292 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
13293 if (RT_SUCCESS(rc))
13294 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
13295 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
13296 }
13297 else
13298 {
13299 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
13300 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
13301 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
13302 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
13303 }
13304
13305 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
13306 if (RT_SUCCESS(rc))
13307 {
13308 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13309 AssertRCReturn(rc2, rc2);
13310 return VINF_SUCCESS;
13311 }
13312 return rc;
13313}
13314
13315
13316/**
13317 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
13318 * Conditional VM-exit.
13319 */
13320HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13321{
13322 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13323 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
13324
13325 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13326 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13327 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13328 {
13329 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
13330 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
13331 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13332 {
13333 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
13334 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13335 }
13336 }
13337 else
13338 {
13339 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13340 rcStrict1 = VINF_SUCCESS;
13341 return rcStrict1;
13342 }
13343
13344 /*
13345 * Get sufficent state and update the exit history entry.
13346 */
13347 RTGCPHYS GCPhys = 0;
13348 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
13349
13350#if 0
13351 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
13352#else
13353 /* Aggressive state sync. for now. */
13354 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13355 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13356 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13357#endif
13358 AssertRCReturn(rc, rc);
13359
13360 VBOXSTRICTRC rcStrict;
13361 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu, EMEXIT_MAKE_FLAGS_AND_TYPE(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO),
13362 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
13363 if (!pExitRec)
13364 {
13365 /*
13366 * If we succeed, resume guest execution.
13367 * If we fail in interpreting the instruction because we couldn't get the guest physical address
13368 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
13369 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
13370 * weird case. See @bugref{6043}.
13371 */
13372 PVM pVM = pVCpu->CTX_SUFF(pVM);
13373 rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
13374 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
13375 if ( rcStrict == VINF_SUCCESS
13376 || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
13377 || rcStrict == VERR_PAGE_NOT_PRESENT)
13378 {
13379 /* Successfully handled MMIO operation. */
13380 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13381 | HM_CHANGED_GUEST_RSP
13382 | HM_CHANGED_GUEST_RFLAGS
13383 | HM_CHANGED_GUEST_APIC_STATE);
13384 rcStrict = VINF_SUCCESS;
13385 }
13386 }
13387 else
13388 {
13389 /*
13390 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
13391 */
13392 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
13393 int rc2 = hmR0VmxSaveGuestRegsForIemInterpreting(pVCpu);
13394 AssertRCReturn(rc2, rc2);
13395
13396 Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
13397 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
13398
13399 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
13400
13401 Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
13402 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13403 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
13404 }
13405 return VBOXSTRICTRC_TODO(rcStrict);
13406}
13407
13408
13409/**
13410 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
13411 * VM-exit.
13412 */
13413HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13414{
13415 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13416 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
13417
13418 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13419 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13420 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13421 {
13422 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
13423 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13424 Log4(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
13425 }
13426 else
13427 {
13428 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13429 rcStrict1 = VINF_SUCCESS;
13430 return rcStrict1;
13431 }
13432
13433 RTGCPHYS GCPhys = 0;
13434 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
13435 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13436#if 0
13437 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
13438#else
13439 /* Aggressive state sync. for now. */
13440 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13441 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13442 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13443#endif
13444 AssertRCReturn(rc, rc);
13445
13446 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
13447 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
13448
13449 RTGCUINT uErrorCode = 0;
13450 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
13451 uErrorCode |= X86_TRAP_PF_ID;
13452 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
13453 uErrorCode |= X86_TRAP_PF_RW;
13454 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
13455 uErrorCode |= X86_TRAP_PF_P;
13456
13457 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
13458
13459 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
13460 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13461
13462 /* Handle the pagefault trap for the nested shadow table. */
13463 PVM pVM = pVCpu->CTX_SUFF(pVM);
13464 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
13465 TRPMResetTrap(pVCpu);
13466
13467 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
13468 if ( rcStrict2 == VINF_SUCCESS
13469 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13470 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
13471 {
13472 /* Successfully synced our nested page tables. */
13473 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
13474 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13475 | HM_CHANGED_GUEST_RSP
13476 | HM_CHANGED_GUEST_RFLAGS);
13477 return VINF_SUCCESS;
13478 }
13479
13480 Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
13481 return rcStrict2;
13482}
13483
13484/** @} */
13485
13486/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13487/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
13488/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13489
13490/** @name VM-exit exception handlers.
13491 * @{
13492 */
13493
13494/**
13495 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
13496 */
13497static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13498{
13499 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13500 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13501
13502 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13503 AssertRCReturn(rc, rc);
13504
13505 if (!(pMixedCtx->cr0 & X86_CR0_NE))
13506 {
13507 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13508 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13509
13510 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13511 * provides VM-exit instruction length. If this causes problem later,
13512 * disassemble the instruction like it's done on AMD-V. */
13513 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13514 AssertRCReturn(rc2, rc2);
13515 return rc;
13516 }
13517
13518 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13519 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13520 return rc;
13521}
13522
13523
13524/**
13525 * VM-exit exception handler for \#BP (Breakpoint exception).
13526 */
13527static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13528{
13529 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13530 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13531
13532 /** @todo Try optimize this by not saving the entire guest state unless
13533 * really needed. */
13534 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13535 AssertRCReturn(rc, rc);
13536
13537 PVM pVM = pVCpu->CTX_SUFF(pVM);
13538 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
13539 if (rc == VINF_EM_RAW_GUEST_TRAP)
13540 {
13541 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13542 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13543 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13544 AssertRCReturn(rc, rc);
13545
13546 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13547 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13548 }
13549
13550 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
13551 return rc;
13552}
13553
13554
13555/**
13556 * VM-exit exception handler for \#AC (alignment check exception).
13557 */
13558static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13559{
13560 RT_NOREF_PV(pMixedCtx);
13561 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13562
13563 /*
13564 * Re-inject it. We'll detect any nesting before getting here.
13565 */
13566 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13567 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13568 AssertRCReturn(rc, rc);
13569 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13570
13571 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13572 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13573 return VINF_SUCCESS;
13574}
13575
13576
13577/**
13578 * VM-exit exception handler for \#DB (Debug exception).
13579 */
13580static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13581{
13582 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13583 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13584 Log6(("XcptDB\n"));
13585
13586 /*
13587 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13588 * for processing.
13589 */
13590 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13591 AssertRCReturn(rc, rc);
13592
13593 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13594 uint64_t uDR6 = X86_DR6_INIT_VAL;
13595 uDR6 |= ( pVmxTransient->uExitQualification
13596 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
13597
13598 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13599 if (rc == VINF_EM_RAW_GUEST_TRAP)
13600 {
13601 /*
13602 * The exception was for the guest. Update DR6, DR7.GD and
13603 * IA32_DEBUGCTL.LBR before forwarding it.
13604 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13605 */
13606 VMMRZCallRing3Disable(pVCpu);
13607 HM_DISABLE_PREEMPT();
13608
13609 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
13610 pMixedCtx->dr[6] |= uDR6;
13611 if (CPUMIsGuestDebugStateActive(pVCpu))
13612 ASMSetDR6(pMixedCtx->dr[6]);
13613
13614 HM_RESTORE_PREEMPT();
13615 VMMRZCallRing3Enable(pVCpu);
13616
13617 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13618 AssertRCReturn(rc, rc);
13619
13620 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13621 pMixedCtx->dr[7] &= ~X86_DR7_GD;
13622
13623 /* Paranoia. */
13624 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13625 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
13626
13627 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
13628 AssertRCReturn(rc, rc);
13629
13630 /*
13631 * Raise #DB in the guest.
13632 *
13633 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
13634 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP (INT1) and not the
13635 * regular #DB. Thus it -may- trigger different handling in the CPU (like skipped DPL checks), see @bugref{6398}.
13636 *
13637 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of Intel 386,
13638 * see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
13639 */
13640 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13641 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13642 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13643 AssertRCReturn(rc, rc);
13644 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13645 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13646 return VINF_SUCCESS;
13647 }
13648
13649 /*
13650 * Not a guest trap, must be a hypervisor related debug event then.
13651 * Update DR6 in case someone is interested in it.
13652 */
13653 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13654 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13655 CPUMSetHyperDR6(pVCpu, uDR6);
13656
13657 return rc;
13658}
13659
13660/**
13661 * VM-exit exception handler for \#GP (General-protection exception).
13662 *
13663 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13664 */
13665static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13666{
13667 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13668 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13669
13670 int rc;
13671 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13672 { /* likely */ }
13673 else
13674 {
13675#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13676 Assert(pVCpu->hm.s.fUsingDebugLoop);
13677#endif
13678 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13679 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13680 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13681 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13682 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13683 AssertRCReturn(rc, rc);
13684 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13685 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13686 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13687 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13688 return rc;
13689 }
13690
13691 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13692 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13693
13694 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13695 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13696 AssertRCReturn(rc, rc);
13697
13698 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13699 uint32_t cbOp = 0;
13700 PVM pVM = pVCpu->CTX_SUFF(pVM);
13701 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13702 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13703 if (RT_SUCCESS(rc))
13704 {
13705 rc = VINF_SUCCESS;
13706 Assert(cbOp == pDis->cbInstr);
13707 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13708 switch (pDis->pCurInstr->uOpcode)
13709 {
13710 case OP_CLI:
13711 {
13712 pMixedCtx->eflags.Bits.u1IF = 0;
13713 pMixedCtx->eflags.Bits.u1RF = 0;
13714 pMixedCtx->rip += pDis->cbInstr;
13715 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13716 if ( !fDbgStepping
13717 && pMixedCtx->eflags.Bits.u1TF)
13718 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13719 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13720 break;
13721 }
13722
13723 case OP_STI:
13724 {
13725 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13726 pMixedCtx->eflags.Bits.u1IF = 1;
13727 pMixedCtx->eflags.Bits.u1RF = 0;
13728 pMixedCtx->rip += pDis->cbInstr;
13729 if (!fOldIF)
13730 {
13731 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13732 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13733 }
13734 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13735 if ( !fDbgStepping
13736 && pMixedCtx->eflags.Bits.u1TF)
13737 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13738 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13739 break;
13740 }
13741
13742 case OP_HLT:
13743 {
13744 rc = VINF_EM_HALT;
13745 pMixedCtx->rip += pDis->cbInstr;
13746 pMixedCtx->eflags.Bits.u1RF = 0;
13747 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13748 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13749 break;
13750 }
13751
13752 case OP_POPF:
13753 {
13754 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13755 uint32_t cbParm;
13756 uint32_t uMask;
13757 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13758 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13759 {
13760 cbParm = 4;
13761 uMask = 0xffffffff;
13762 }
13763 else
13764 {
13765 cbParm = 2;
13766 uMask = 0xffff;
13767 }
13768
13769 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13770 RTGCPTR GCPtrStack = 0;
13771 X86EFLAGS Eflags;
13772 Eflags.u32 = 0;
13773 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13774 &GCPtrStack);
13775 if (RT_SUCCESS(rc))
13776 {
13777 Assert(sizeof(Eflags.u32) >= cbParm);
13778 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13779 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13780 }
13781 if (RT_FAILURE(rc))
13782 {
13783 rc = VERR_EM_INTERPRETER;
13784 break;
13785 }
13786 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13787 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13788 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13789 pMixedCtx->esp += cbParm;
13790 pMixedCtx->esp &= uMask;
13791 pMixedCtx->rip += pDis->cbInstr;
13792 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13793 | HM_CHANGED_GUEST_RSP
13794 | HM_CHANGED_GUEST_RFLAGS);
13795 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13796 POPF restores EFLAGS.TF. */
13797 if ( !fDbgStepping
13798 && fGstStepping)
13799 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13800 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13801 break;
13802 }
13803
13804 case OP_PUSHF:
13805 {
13806 uint32_t cbParm;
13807 uint32_t uMask;
13808 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13809 {
13810 cbParm = 4;
13811 uMask = 0xffffffff;
13812 }
13813 else
13814 {
13815 cbParm = 2;
13816 uMask = 0xffff;
13817 }
13818
13819 /* Get the stack pointer & push the contents of eflags onto the stack. */
13820 RTGCPTR GCPtrStack = 0;
13821 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13822 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13823 if (RT_FAILURE(rc))
13824 {
13825 rc = VERR_EM_INTERPRETER;
13826 break;
13827 }
13828 X86EFLAGS Eflags = pMixedCtx->eflags;
13829 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13830 Eflags.Bits.u1RF = 0;
13831 Eflags.Bits.u1VM = 0;
13832
13833 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13834 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13835 {
13836 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13837 rc = VERR_EM_INTERPRETER;
13838 break;
13839 }
13840 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13841 pMixedCtx->esp -= cbParm;
13842 pMixedCtx->esp &= uMask;
13843 pMixedCtx->rip += pDis->cbInstr;
13844 pMixedCtx->eflags.Bits.u1RF = 0;
13845 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13846 | HM_CHANGED_GUEST_RSP
13847 | HM_CHANGED_GUEST_RFLAGS);
13848 if ( !fDbgStepping
13849 && pMixedCtx->eflags.Bits.u1TF)
13850 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13851 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13852 break;
13853 }
13854
13855 case OP_IRET:
13856 {
13857 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13858 * instruction reference. */
13859 RTGCPTR GCPtrStack = 0;
13860 uint32_t uMask = 0xffff;
13861 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13862 uint16_t aIretFrame[3];
13863 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13864 {
13865 rc = VERR_EM_INTERPRETER;
13866 break;
13867 }
13868 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13869 &GCPtrStack);
13870 if (RT_SUCCESS(rc))
13871 {
13872 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13873 PGMACCESSORIGIN_HM));
13874 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13875 }
13876 if (RT_FAILURE(rc))
13877 {
13878 rc = VERR_EM_INTERPRETER;
13879 break;
13880 }
13881 pMixedCtx->eip = 0;
13882 pMixedCtx->ip = aIretFrame[0];
13883 pMixedCtx->cs.Sel = aIretFrame[1];
13884 pMixedCtx->cs.ValidSel = aIretFrame[1];
13885 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13886 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13887 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13888 pMixedCtx->sp += sizeof(aIretFrame);
13889 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13890 | HM_CHANGED_GUEST_SEGMENT_REGS
13891 | HM_CHANGED_GUEST_RSP
13892 | HM_CHANGED_GUEST_RFLAGS);
13893 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13894 if ( !fDbgStepping
13895 && fGstStepping)
13896 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13897 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13898 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13899 break;
13900 }
13901
13902 case OP_INT:
13903 {
13904 uint16_t uVector = pDis->Param1.uValue & 0xff;
13905 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13906 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13907 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13908 break;
13909 }
13910
13911 case OP_INTO:
13912 {
13913 if (pMixedCtx->eflags.Bits.u1OF)
13914 {
13915 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13916 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13917 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13918 }
13919 else
13920 {
13921 pMixedCtx->eflags.Bits.u1RF = 0;
13922 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13923 }
13924 break;
13925 }
13926
13927 default:
13928 {
13929 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13930 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13931 EMCODETYPE_SUPERVISOR);
13932 rc = VBOXSTRICTRC_VAL(rc2);
13933 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13934 /** @todo We have to set pending-debug exceptions here when the guest is
13935 * single-stepping depending on the instruction that was interpreted. */
13936 Log4(("#GP rc=%Rrc\n", rc));
13937 break;
13938 }
13939 }
13940 }
13941 else
13942 rc = VERR_EM_INTERPRETER;
13943
13944 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13945 ("#GP Unexpected rc=%Rrc\n", rc));
13946 return rc;
13947}
13948
13949
13950/**
13951 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13952 * the exception reported in the VMX transient structure back into the VM.
13953 *
13954 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13955 * up-to-date.
13956 */
13957static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13958{
13959 RT_NOREF_PV(pMixedCtx);
13960 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13961#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13962 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13963 ("uVector=%#04x u32XcptBitmap=%#010RX32\n",
13964 VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13965#endif
13966
13967 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13968 hmR0VmxCheckExitDueToEventDelivery(). */
13969 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13970 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13971 AssertRCReturn(rc, rc);
13972 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13973
13974#ifdef DEBUG_ramshankar
13975 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13976 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13977 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13978#endif
13979
13980 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13981 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13982 return VINF_SUCCESS;
13983}
13984
13985
13986/**
13987 * VM-exit exception handler for \#PF (Page-fault exception).
13988 */
13989static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13990{
13991 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13992 PVM pVM = pVCpu->CTX_SUFF(pVM);
13993 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13994 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13995 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13996 AssertRCReturn(rc, rc);
13997
13998 if (!pVM->hm.s.fNestedPaging)
13999 { /* likely */ }
14000 else
14001 {
14002#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
14003 Assert(pVCpu->hm.s.fUsingDebugLoop);
14004#endif
14005 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
14006 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
14007 {
14008 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14009 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
14010 }
14011 else
14012 {
14013 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
14014 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
14015 Log4(("Pending #DF due to vectoring #PF. NP\n"));
14016 }
14017 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
14018 return rc;
14019 }
14020
14021 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
14022 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
14023 if (pVmxTransient->fVectoringPF)
14024 {
14025 Assert(pVCpu->hm.s.Event.fPending);
14026 return VINF_EM_RAW_INJECT_TRPM_EVENT;
14027 }
14028
14029 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
14030 AssertRCReturn(rc, rc);
14031
14032 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
14033 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
14034
14035 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
14036 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
14037 (RTGCPTR)pVmxTransient->uExitQualification);
14038
14039 Log4(("#PF: rc=%Rrc\n", rc));
14040 if (rc == VINF_SUCCESS)
14041 {
14042#if 0
14043 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
14044 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
14045 * memory? We don't update the whole state here... */
14046 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
14047 | HM_CHANGED_GUEST_RSP
14048 | HM_CHANGED_GUEST_RFLAGS
14049 | HM_CHANGED_GUEST_APIC_STATE);
14050#else
14051 /*
14052 * This is typically a shadow page table sync or a MMIO instruction. But we may have
14053 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
14054 */
14055 /** @todo take advantage of CPUM changed flags instead of brute forcing. */
14056 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
14057#endif
14058 TRPMResetTrap(pVCpu);
14059 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
14060 return rc;
14061 }
14062
14063 if (rc == VINF_EM_RAW_GUEST_TRAP)
14064 {
14065 if (!pVmxTransient->fVectoringDoublePF)
14066 {
14067 /* It's a guest page fault and needs to be reflected to the guest. */
14068 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
14069 TRPMResetTrap(pVCpu);
14070 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
14071 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14072 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
14073 }
14074 else
14075 {
14076 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
14077 TRPMResetTrap(pVCpu);
14078 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
14079 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
14080 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
14081 }
14082
14083 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
14084 return VINF_SUCCESS;
14085 }
14086
14087 TRPMResetTrap(pVCpu);
14088 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
14089 return rc;
14090}
14091
14092/** @} */
14093
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette