VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 59073

Last change on this file since 59073 was 59073, checked in by vboxsync, 9 years ago

VMM: VINF_EM_DBG_EVENT and DBGFEventGenericWithArg implementation.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 572.1 KB
Line 
1/* $Id: HMVMXR0.cpp 59073 2015-12-10 12:48:03Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2015 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#include <iprt/x86.h>
24#include <iprt/asm-amd64-x86.h>
25#include <iprt/thread.h>
26
27#include <VBox/vmm/pdmapi.h>
28#include <VBox/vmm/dbgf.h>
29#include <VBox/vmm/iem.h>
30#include <VBox/vmm/iom.h>
31#include <VBox/vmm/selm.h>
32#include <VBox/vmm/tm.h>
33#include <VBox/vmm/gim.h>
34#ifdef VBOX_WITH_REM
35# include <VBox/vmm/rem.h>
36#endif
37#include "HMInternal.h"
38#include <VBox/vmm/vm.h>
39#include "HMVMXR0.h"
40#include "dtrace/VBoxVMM.h"
41
42#ifdef DEBUG_ramshankar
43# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
44# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
45# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
46# define HMVMX_ALWAYS_CHECK_GUEST_STATE
47# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
48# define HMVMX_ALWAYS_TRAP_PF
49# define HMVMX_ALWAYS_SWAP_FPU_STATE
50# define HMVMX_ALWAYS_FLUSH_TLB
51# define HMVMX_ALWAYS_SWAP_EFER
52#endif
53
54
55/*********************************************************************************************************************************
56* Defined Constants And Macros *
57*********************************************************************************************************************************/
58/** Use the function table. */
59#define HMVMX_USE_FUNCTION_TABLE
60
61/** Determine which tagged-TLB flush handler to use. */
62#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
63#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
64#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
65#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
66
67/** @name Updated-guest-state flags.
68 * @{ */
69#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
70#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
71#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
72#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
73#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
74#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
75#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
76#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
77#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
78#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
79#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
80#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
81#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
82#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
83#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
84#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
85#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
86#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
87#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
88#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
89#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
90 | HMVMX_UPDATED_GUEST_RSP \
91 | HMVMX_UPDATED_GUEST_RFLAGS \
92 | HMVMX_UPDATED_GUEST_CR0 \
93 | HMVMX_UPDATED_GUEST_CR3 \
94 | HMVMX_UPDATED_GUEST_CR4 \
95 | HMVMX_UPDATED_GUEST_GDTR \
96 | HMVMX_UPDATED_GUEST_IDTR \
97 | HMVMX_UPDATED_GUEST_LDTR \
98 | HMVMX_UPDATED_GUEST_TR \
99 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
100 | HMVMX_UPDATED_GUEST_DEBUG \
101 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
102 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
103 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
104 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
105 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
106 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
107 | HMVMX_UPDATED_GUEST_INTR_STATE \
108 | HMVMX_UPDATED_GUEST_APIC_STATE)
109/** @} */
110
111/** @name
112 * Flags to skip redundant reads of some common VMCS fields that are not part of
113 * the guest-CPU state but are in the transient structure.
114 */
115#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
116#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
117#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
118#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
119#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
120#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
122/** @} */
123
124/** @name
125 * States of the VMCS.
126 *
127 * This does not reflect all possible VMCS states but currently only those
128 * needed for maintaining the VMCS consistently even when thread-context hooks
129 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
130 */
131#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
132#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
133#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
134/** @} */
135
136/**
137 * Exception bitmap mask for real-mode guests (real-on-v86).
138 *
139 * We need to intercept all exceptions manually except:
140 * - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
141 * - \#DB handled in hmR0VmxLoadSharedDebugState().
142 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
143 * support.
144 */
145#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
146 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
147 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
148 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
149 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
150 /* RT_BIT(X86_XCPT_MF) always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
151 | RT_BIT(X86_XCPT_XF))
152
153/**
154 * Exception bitmap mask for all contributory exceptions.
155 *
156 * Page fault is deliberately excluded here as it's conditional as to whether
157 * it's contributory or benign. Page faults are handled separately.
158 */
159#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
160 | RT_BIT(X86_XCPT_DE))
161
162/** Maximum VM-instruction error number. */
163#define HMVMX_INSTR_ERROR_MAX 28
164
165/** Profiling macro. */
166#ifdef HM_PROFILE_EXIT_DISPATCH
167# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
168# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
169#else
170# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
171# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
172#endif
173
174/** Assert that preemption is disabled or covered by thread-context hooks. */
175#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
176 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
177
178/** Assert that we haven't migrated CPUs when thread-context hooks are not
179 * used. */
180#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
181 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
182 ("Illegal migration! Entered on CPU %u Current %u\n", \
183 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
184
185/** Helper macro for VM-exit handlers called unexpectedly. */
186#define HMVMX_RETURN_UNEXPECTED_EXIT() \
187 do { \
188 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
189 return VERR_VMX_UNEXPECTED_EXIT; \
190 } while (0)
191
192
193/*********************************************************************************************************************************
194* Structures and Typedefs *
195*********************************************************************************************************************************/
196/**
197 * VMX transient state.
198 *
199 * A state structure for holding miscellaneous information across
200 * VMX non-root operation and restored after the transition.
201 */
202typedef struct VMXTRANSIENT
203{
204 /** The host's rflags/eflags. */
205 RTCCUINTREG fEFlags;
206#if HC_ARCH_BITS == 32
207 uint32_t u32Alignment0;
208#endif
209 /** The guest's TPR value used for TPR shadowing. */
210 uint8_t u8GuestTpr;
211 /** Alignment. */
212 uint8_t abAlignment0[7];
213
214 /** The basic VM-exit reason. */
215 uint16_t uExitReason;
216 /** Alignment. */
217 uint16_t u16Alignment0;
218 /** The VM-exit interruption error code. */
219 uint32_t uExitIntErrorCode;
220 /** The VM-exit exit code qualification. */
221 uint64_t uExitQualification;
222
223 /** The VM-exit interruption-information field. */
224 uint32_t uExitIntInfo;
225 /** The VM-exit instruction-length field. */
226 uint32_t cbInstr;
227 /** The VM-exit instruction-information field. */
228 union
229 {
230 /** Plain unsigned int representation. */
231 uint32_t u;
232 /** INS and OUTS information. */
233 struct
234 {
235 uint32_t u6Reserved0 : 7;
236 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
237 uint32_t u3AddrSize : 3;
238 uint32_t u5Reserved1 : 5;
239 /** The segment register (X86_SREG_XXX). */
240 uint32_t iSegReg : 3;
241 uint32_t uReserved2 : 14;
242 } StrIo;
243 } ExitInstrInfo;
244 /** Whether the VM-entry failed or not. */
245 bool fVMEntryFailed;
246 /** Alignment. */
247 uint8_t abAlignment1[3];
248
249 /** The VM-entry interruption-information field. */
250 uint32_t uEntryIntInfo;
251 /** The VM-entry exception error code field. */
252 uint32_t uEntryXcptErrorCode;
253 /** The VM-entry instruction length field. */
254 uint32_t cbEntryInstr;
255
256 /** IDT-vectoring information field. */
257 uint32_t uIdtVectoringInfo;
258 /** IDT-vectoring error code. */
259 uint32_t uIdtVectoringErrorCode;
260
261 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
262 uint32_t fVmcsFieldsRead;
263
264 /** Whether the guest FPU was active at the time of VM-exit. */
265 bool fWasGuestFPUStateActive;
266 /** Whether the guest debug state was active at the time of VM-exit. */
267 bool fWasGuestDebugStateActive;
268 /** Whether the hyper debug state was active at the time of VM-exit. */
269 bool fWasHyperDebugStateActive;
270 /** Whether TSC-offsetting should be setup before VM-entry. */
271 bool fUpdateTscOffsettingAndPreemptTimer;
272 /** Whether the VM-exit was caused by a page-fault during delivery of a
273 * contributory exception or a page-fault. */
274 bool fVectoringDoublePF;
275 /** Whether the VM-exit was caused by a page-fault during delivery of an
276 * external interrupt or NMI. */
277 bool fVectoringPF;
278} VMXTRANSIENT;
279AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
280AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
281AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
282AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
283AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
284/** Pointer to VMX transient state. */
285typedef VMXTRANSIENT *PVMXTRANSIENT;
286
287
288/**
289 * MSR-bitmap read permissions.
290 */
291typedef enum VMXMSREXITREAD
292{
293 /** Reading this MSR causes a VM-exit. */
294 VMXMSREXIT_INTERCEPT_READ = 0xb,
295 /** Reading this MSR does not cause a VM-exit. */
296 VMXMSREXIT_PASSTHRU_READ
297} VMXMSREXITREAD;
298/** Pointer to MSR-bitmap read permissions. */
299typedef VMXMSREXITREAD* PVMXMSREXITREAD;
300
301/**
302 * MSR-bitmap write permissions.
303 */
304typedef enum VMXMSREXITWRITE
305{
306 /** Writing to this MSR causes a VM-exit. */
307 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
308 /** Writing to this MSR does not cause a VM-exit. */
309 VMXMSREXIT_PASSTHRU_WRITE
310} VMXMSREXITWRITE;
311/** Pointer to MSR-bitmap write permissions. */
312typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
313
314
315/**
316 * VMX VM-exit handler.
317 *
318 * @returns Strict VBox status code (i.e. informational status codes too).
319 * @param pVCpu The cross context virtual CPU structure.
320 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
321 * out-of-sync. Make sure to update the required
322 * fields before using them.
323 * @param pVmxTransient Pointer to the VMX-transient structure.
324 */
325#ifndef HMVMX_USE_FUNCTION_TABLE
326typedef DECLINLINE(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
327#else
328typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
329/** Pointer to VM-exit handler. */
330typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
331#endif
332
333/**
334 * VMX VM-exit handler, non-strict status code.
335 *
336 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
337 *
338 * @returns VBox status code, no informational status code returned.
339 * @param pVCpu The cross context virtual CPU structure.
340 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
341 * out-of-sync. Make sure to update the required
342 * fields before using them.
343 * @param pVmxTransient Pointer to the VMX-transient structure.
344 *
345 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
346 * use of that status code will be replaced with VINF_EM_SOMETHING
347 * later when switching over to IEM.
348 */
349#ifndef HMVMX_USE_FUNCTION_TABLE
350typedef DECLINLINE(int) FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
351#else
352typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
353#endif
354
355
356/*********************************************************************************************************************************
357* Internal Functions *
358*********************************************************************************************************************************/
359static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
360static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
361static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
362static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
363 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
364 bool fStepping, uint32_t *puIntState);
365#if HC_ARCH_BITS == 32
366static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
367#endif
368#ifndef HMVMX_USE_FUNCTION_TABLE
369DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
370# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
371# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
372#else
373# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
374# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
375#endif
376
377
378/** @name VM-exit handlers.
379 * @{
380 */
381static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
382static FNVMXEXITHANDLER hmR0VmxExitExtInt;
383static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
384static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
385static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
386static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
387static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
388static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
389static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
390static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
391static FNVMXEXITHANDLER hmR0VmxExitCpuid;
392static FNVMXEXITHANDLER hmR0VmxExitGetsec;
393static FNVMXEXITHANDLER hmR0VmxExitHlt;
394static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
395static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
396static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
397static FNVMXEXITHANDLER hmR0VmxExitVmcall;
398static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
399static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
400static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
401static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
402static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
403static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
404static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
405static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
406static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
407static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
408static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
409static FNVMXEXITHANDLER hmR0VmxExitMwait;
410static FNVMXEXITHANDLER hmR0VmxExitMtf;
411static FNVMXEXITHANDLER hmR0VmxExitMonitor;
412static FNVMXEXITHANDLER hmR0VmxExitPause;
413static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
414static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
415static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
416static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
417static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
418static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
419static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
420static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
421static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
422static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
423static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
424static FNVMXEXITHANDLER hmR0VmxExitRdrand;
425static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
426/** @} */
427
428static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
429static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
430static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
431static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
432static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
433static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
434static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
435static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
436static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
437
438
439/*********************************************************************************************************************************
440* Global Variables *
441*********************************************************************************************************************************/
442#ifdef HMVMX_USE_FUNCTION_TABLE
443
444/**
445 * VMX_EXIT dispatch table.
446 */
447static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
448{
449 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
450 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
451 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
452 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
453 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
454 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
455 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
456 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
457 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
458 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
459 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
460 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
461 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
462 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
463 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
464 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
465 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
466 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
467 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
468 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
469 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
470 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
471 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
472 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
473 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
474 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
475 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
476 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
477 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
478 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
479 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
480 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
481 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
482 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
483 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
484 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
485 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
486 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
487 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
488 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
489 /* 40 UNDEFINED */ hmR0VmxExitPause,
490 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
491 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
492 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
493 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
494 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
495 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
496 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
497 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
498 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
499 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
500 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
501 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
502 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
503 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
504 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
505 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
506 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
507 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
508 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
509 /* 60 VMX_EXIT_RESERVED_60 */ hmR0VmxExitErrUndefined,
510 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
511 /* 62 VMX_EXIT_RESERVED_62 */ hmR0VmxExitErrUndefined,
512 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
513 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
514};
515#endif /* HMVMX_USE_FUNCTION_TABLE */
516
517#ifdef VBOX_STRICT
518static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
519{
520 /* 0 */ "(Not Used)",
521 /* 1 */ "VMCALL executed in VMX root operation.",
522 /* 2 */ "VMCLEAR with invalid physical address.",
523 /* 3 */ "VMCLEAR with VMXON pointer.",
524 /* 4 */ "VMLAUNCH with non-clear VMCS.",
525 /* 5 */ "VMRESUME with non-launched VMCS.",
526 /* 6 */ "VMRESUME after VMXOFF",
527 /* 7 */ "VM-entry with invalid control fields.",
528 /* 8 */ "VM-entry with invalid host state fields.",
529 /* 9 */ "VMPTRLD with invalid physical address.",
530 /* 10 */ "VMPTRLD with VMXON pointer.",
531 /* 11 */ "VMPTRLD with incorrect revision identifier.",
532 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
533 /* 13 */ "VMWRITE to read-only VMCS component.",
534 /* 14 */ "(Not Used)",
535 /* 15 */ "VMXON executed in VMX root operation.",
536 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
537 /* 17 */ "VM-entry with non-launched executing VMCS.",
538 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
539 /* 19 */ "VMCALL with non-clear VMCS.",
540 /* 20 */ "VMCALL with invalid VM-exit control fields.",
541 /* 21 */ "(Not Used)",
542 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
543 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
544 /* 24 */ "VMCALL with invalid SMM-monitor features.",
545 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
546 /* 26 */ "VM-entry with events blocked by MOV SS.",
547 /* 27 */ "(Not Used)",
548 /* 28 */ "Invalid operand to INVEPT/INVVPID."
549};
550#endif /* VBOX_STRICT */
551
552
553
554/**
555 * Updates the VM's last error record.
556 *
557 * If there was a VMX instruction error, reads the error data from the VMCS and
558 * updates VCPU's last error record as well.
559 *
560 * @param pVM The cross context VM structure.
561 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
562 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
563 * VERR_VMX_INVALID_VMCS_FIELD.
564 * @param rc The error code.
565 */
566static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
567{
568 AssertPtr(pVM);
569 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
570 || rc == VERR_VMX_UNABLE_TO_START_VM)
571 {
572 AssertPtrReturnVoid(pVCpu);
573 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
574 }
575 pVM->hm.s.lLastError = rc;
576}
577
578
579/**
580 * Reads the VM-entry interruption-information field from the VMCS into the VMX
581 * transient structure.
582 *
583 * @returns VBox status code.
584 * @param pVmxTransient Pointer to the VMX transient structure.
585 *
586 * @remarks No-long-jump zone!!!
587 */
588DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
589{
590 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
591 AssertRCReturn(rc, rc);
592 return VINF_SUCCESS;
593}
594
595
596/**
597 * Reads the VM-entry exception error code field from the VMCS into
598 * the VMX transient structure.
599 *
600 * @returns VBox status code.
601 * @param pVmxTransient Pointer to the VMX transient structure.
602 *
603 * @remarks No-long-jump zone!!!
604 */
605DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
606{
607 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
608 AssertRCReturn(rc, rc);
609 return VINF_SUCCESS;
610}
611
612
613/**
614 * Reads the VM-entry exception error code field from the VMCS into
615 * the VMX transient structure.
616 *
617 * @returns VBox status code.
618 * @param pVmxTransient Pointer to the VMX transient structure.
619 *
620 * @remarks No-long-jump zone!!!
621 */
622DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
623{
624 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
625 AssertRCReturn(rc, rc);
626 return VINF_SUCCESS;
627}
628
629
630/**
631 * Reads the VM-exit interruption-information field from the VMCS into the VMX
632 * transient structure.
633 *
634 * @returns VBox status code.
635 * @param pVmxTransient Pointer to the VMX transient structure.
636 */
637DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
638{
639 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
640 {
641 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
642 AssertRCReturn(rc, rc);
643 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
644 }
645 return VINF_SUCCESS;
646}
647
648
649/**
650 * Reads the VM-exit interruption error code from the VMCS into the VMX
651 * transient structure.
652 *
653 * @returns VBox status code.
654 * @param pVmxTransient Pointer to the VMX transient structure.
655 */
656DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
657{
658 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
659 {
660 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
661 AssertRCReturn(rc, rc);
662 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
663 }
664 return VINF_SUCCESS;
665}
666
667
668/**
669 * Reads the VM-exit instruction length field from the VMCS into the VMX
670 * transient structure.
671 *
672 * @returns VBox status code.
673 * @param pVmxTransient Pointer to the VMX transient structure.
674 */
675DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
676{
677 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
678 {
679 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
680 AssertRCReturn(rc, rc);
681 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
682 }
683 return VINF_SUCCESS;
684}
685
686
687/**
688 * Reads the VM-exit instruction-information field from the VMCS into
689 * the VMX transient structure.
690 *
691 * @returns VBox status code.
692 * @param pVmxTransient Pointer to the VMX transient structure.
693 */
694DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
695{
696 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
697 {
698 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
699 AssertRCReturn(rc, rc);
700 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
701 }
702 return VINF_SUCCESS;
703}
704
705
706/**
707 * Reads the exit code qualification from the VMCS into the VMX transient
708 * structure.
709 *
710 * @returns VBox status code.
711 * @param pVCpu The cross context virtual CPU structure of the
712 * calling EMT. (Required for the VMCS cache case.)
713 * @param pVmxTransient Pointer to the VMX transient structure.
714 */
715DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
716{
717 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
718 {
719 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
720 AssertRCReturn(rc, rc);
721 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
722 }
723 return VINF_SUCCESS;
724}
725
726
727/**
728 * Reads the IDT-vectoring information field from the VMCS into the VMX
729 * transient structure.
730 *
731 * @returns VBox status code.
732 * @param pVmxTransient Pointer to the VMX transient structure.
733 *
734 * @remarks No-long-jump zone!!!
735 */
736DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
737{
738 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
739 {
740 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
741 AssertRCReturn(rc, rc);
742 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
743 }
744 return VINF_SUCCESS;
745}
746
747
748/**
749 * Reads the IDT-vectoring error code from the VMCS into the VMX
750 * transient structure.
751 *
752 * @returns VBox status code.
753 * @param pVmxTransient Pointer to the VMX transient structure.
754 */
755DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
756{
757 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
758 {
759 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
760 AssertRCReturn(rc, rc);
761 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
762 }
763 return VINF_SUCCESS;
764}
765
766
767/**
768 * Enters VMX root mode operation on the current CPU.
769 *
770 * @returns VBox status code.
771 * @param pVM The cross context VM structure. Can be
772 * NULL, after a resume.
773 * @param HCPhysCpuPage Physical address of the VMXON region.
774 * @param pvCpuPage Pointer to the VMXON region.
775 */
776static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
777{
778 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
779 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
780 Assert(pvCpuPage);
781 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
782
783 if (pVM)
784 {
785 /* Write the VMCS revision dword to the VMXON region. */
786 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
787 }
788
789 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
790 RTCCUINTREG fEFlags = ASMIntDisableFlags();
791
792 /* Enable the VMX bit in CR4 if necessary. */
793 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
794
795 /* Enter VMX root mode. */
796 int rc = VMXEnable(HCPhysCpuPage);
797 if (RT_FAILURE(rc))
798 {
799 if (!(uOldCr4 & X86_CR4_VMXE))
800 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
801
802 if (pVM)
803 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
804 }
805
806 /* Restore interrupts. */
807 ASMSetFlags(fEFlags);
808 return rc;
809}
810
811
812/**
813 * Exits VMX root mode operation on the current CPU.
814 *
815 * @returns VBox status code.
816 */
817static int hmR0VmxLeaveRootMode(void)
818{
819 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
820
821 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
822 RTCCUINTREG fEFlags = ASMIntDisableFlags();
823
824 /* If we're for some reason not in VMX root mode, then don't leave it. */
825 RTCCUINTREG uHostCR4 = ASMGetCR4();
826
827 int rc;
828 if (uHostCR4 & X86_CR4_VMXE)
829 {
830 /* Exit VMX root mode and clear the VMX bit in CR4. */
831 VMXDisable();
832 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
833 rc = VINF_SUCCESS;
834 }
835 else
836 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
837
838 /* Restore interrupts. */
839 ASMSetFlags(fEFlags);
840 return rc;
841}
842
843
844/**
845 * Allocates and maps one physically contiguous page. The allocated page is
846 * zero'd out. (Used by various VT-x structures).
847 *
848 * @returns IPRT status code.
849 * @param pMemObj Pointer to the ring-0 memory object.
850 * @param ppVirt Where to store the virtual address of the
851 * allocation.
852 * @param pHCPhys Where to store the physical address of the
853 * allocation.
854 */
855DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
856{
857 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
858 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
859 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
860
861 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
862 if (RT_FAILURE(rc))
863 return rc;
864 *ppVirt = RTR0MemObjAddress(*pMemObj);
865 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
866 ASMMemZero32(*ppVirt, PAGE_SIZE);
867 return VINF_SUCCESS;
868}
869
870
871/**
872 * Frees and unmaps an allocated physical page.
873 *
874 * @param pMemObj Pointer to the ring-0 memory object.
875 * @param ppVirt Where to re-initialize the virtual address of
876 * allocation as 0.
877 * @param pHCPhys Where to re-initialize the physical address of the
878 * allocation as 0.
879 */
880DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
881{
882 AssertPtr(pMemObj);
883 AssertPtr(ppVirt);
884 AssertPtr(pHCPhys);
885 if (*pMemObj != NIL_RTR0MEMOBJ)
886 {
887 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
888 AssertRC(rc);
889 *pMemObj = NIL_RTR0MEMOBJ;
890 *ppVirt = 0;
891 *pHCPhys = 0;
892 }
893}
894
895
896/**
897 * Worker function to free VT-x related structures.
898 *
899 * @returns IPRT status code.
900 * @param pVM The cross context VM structure.
901 */
902static void hmR0VmxStructsFree(PVM pVM)
903{
904 for (VMCPUID i = 0; i < pVM->cCpus; i++)
905 {
906 PVMCPU pVCpu = &pVM->aCpus[i];
907 AssertPtr(pVCpu);
908
909 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
910 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
911
912 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
913 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
914
915 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
916 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
917 }
918
919 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
920#ifdef VBOX_WITH_CRASHDUMP_MAGIC
921 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
922#endif
923}
924
925
926/**
927 * Worker function to allocate VT-x related VM structures.
928 *
929 * @returns IPRT status code.
930 * @param pVM The cross context VM structure.
931 */
932static int hmR0VmxStructsAlloc(PVM pVM)
933{
934 /*
935 * Initialize members up-front so we can cleanup properly on allocation failure.
936 */
937#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
938 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
939 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
940 pVM->hm.s.vmx.HCPhys##a_Name = 0;
941
942#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
943 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
944 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
945 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
946
947#ifdef VBOX_WITH_CRASHDUMP_MAGIC
948 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
949#endif
950 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
951
952 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
953 for (VMCPUID i = 0; i < pVM->cCpus; i++)
954 {
955 PVMCPU pVCpu = &pVM->aCpus[i];
956 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
957 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
958 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
959 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
960 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
961 }
962#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
963#undef VMXLOCAL_INIT_VM_MEMOBJ
964
965 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
966 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
967 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
968 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
969
970 /*
971 * Allocate all the VT-x structures.
972 */
973 int rc = VINF_SUCCESS;
974#ifdef VBOX_WITH_CRASHDUMP_MAGIC
975 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
976 if (RT_FAILURE(rc))
977 goto cleanup;
978 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
979 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
980#endif
981
982 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
983 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
984 {
985 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
986 &pVM->hm.s.vmx.HCPhysApicAccess);
987 if (RT_FAILURE(rc))
988 goto cleanup;
989 }
990
991 /*
992 * Initialize per-VCPU VT-x structures.
993 */
994 for (VMCPUID i = 0; i < pVM->cCpus; i++)
995 {
996 PVMCPU pVCpu = &pVM->aCpus[i];
997 AssertPtr(pVCpu);
998
999 /* Allocate the VM control structure (VMCS). */
1000 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1001 if (RT_FAILURE(rc))
1002 goto cleanup;
1003
1004 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
1005 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1006 {
1007 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1008 &pVCpu->hm.s.vmx.HCPhysVirtApic);
1009 if (RT_FAILURE(rc))
1010 goto cleanup;
1011 }
1012
1013 /*
1014 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1015 * transparent accesses of specific MSRs.
1016 *
1017 * If the condition for enabling MSR bitmaps changes here, don't forget to
1018 * update HMAreMsrBitmapsAvailable().
1019 */
1020 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1021 {
1022 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1023 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1024 if (RT_FAILURE(rc))
1025 goto cleanup;
1026 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1027 }
1028
1029 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1030 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1031 if (RT_FAILURE(rc))
1032 goto cleanup;
1033
1034 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1035 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1036 if (RT_FAILURE(rc))
1037 goto cleanup;
1038 }
1039
1040 return VINF_SUCCESS;
1041
1042cleanup:
1043 hmR0VmxStructsFree(pVM);
1044 return rc;
1045}
1046
1047
1048/**
1049 * Does global VT-x initialization (called during module initialization).
1050 *
1051 * @returns VBox status code.
1052 */
1053VMMR0DECL(int) VMXR0GlobalInit(void)
1054{
1055#ifdef HMVMX_USE_FUNCTION_TABLE
1056 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1057# ifdef VBOX_STRICT
1058 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1059 Assert(g_apfnVMExitHandlers[i]);
1060# endif
1061#endif
1062 return VINF_SUCCESS;
1063}
1064
1065
1066/**
1067 * Does global VT-x termination (called during module termination).
1068 */
1069VMMR0DECL(void) VMXR0GlobalTerm()
1070{
1071 /* Nothing to do currently. */
1072}
1073
1074
1075/**
1076 * Sets up and activates VT-x on the current CPU.
1077 *
1078 * @returns VBox status code.
1079 * @param pCpu Pointer to the global CPU info struct.
1080 * @param pVM The cross context VM structure. Can be
1081 * NULL after a host resume operation.
1082 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1083 * fEnabledByHost is @c true).
1084 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1085 * @a fEnabledByHost is @c true).
1086 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1087 * enable VT-x on the host.
1088 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1089 */
1090VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1091 void *pvMsrs)
1092{
1093 Assert(pCpu);
1094 Assert(pvMsrs);
1095 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1096
1097 /* Enable VT-x if it's not already enabled by the host. */
1098 if (!fEnabledByHost)
1099 {
1100 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1101 if (RT_FAILURE(rc))
1102 return rc;
1103 }
1104
1105 /*
1106 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1107 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1108 */
1109 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1110 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1111 {
1112 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1113 pCpu->fFlushAsidBeforeUse = false;
1114 }
1115 else
1116 pCpu->fFlushAsidBeforeUse = true;
1117
1118 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1119 ++pCpu->cTlbFlushes;
1120
1121 return VINF_SUCCESS;
1122}
1123
1124
1125/**
1126 * Deactivates VT-x on the current CPU.
1127 *
1128 * @returns VBox status code.
1129 * @param pCpu Pointer to the global CPU info struct.
1130 * @param pvCpuPage Pointer to the VMXON region.
1131 * @param HCPhysCpuPage Physical address of the VMXON region.
1132 *
1133 * @remarks This function should never be called when SUPR0EnableVTx() or
1134 * similar was used to enable VT-x on the host.
1135 */
1136VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1137{
1138 NOREF(pCpu);
1139 NOREF(pvCpuPage);
1140 NOREF(HCPhysCpuPage);
1141
1142 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1143 return hmR0VmxLeaveRootMode();
1144}
1145
1146
1147/**
1148 * Sets the permission bits for the specified MSR in the MSR bitmap.
1149 *
1150 * @param pVCpu The cross context virtual CPU structure.
1151 * @param uMsr The MSR value.
1152 * @param enmRead Whether reading this MSR causes a VM-exit.
1153 * @param enmWrite Whether writing this MSR causes a VM-exit.
1154 */
1155static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1156{
1157 int32_t iBit;
1158 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1159
1160 /*
1161 * Layout:
1162 * 0x000 - 0x3ff - Low MSR read bits
1163 * 0x400 - 0x7ff - High MSR read bits
1164 * 0x800 - 0xbff - Low MSR write bits
1165 * 0xc00 - 0xfff - High MSR write bits
1166 */
1167 if (uMsr <= 0x00001FFF)
1168 iBit = uMsr;
1169 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1170 {
1171 iBit = uMsr - UINT32_C(0xC0000000);
1172 pbMsrBitmap += 0x400;
1173 }
1174 else
1175 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1176
1177 Assert(iBit <= 0x1fff);
1178 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1179 ASMBitSet(pbMsrBitmap, iBit);
1180 else
1181 ASMBitClear(pbMsrBitmap, iBit);
1182
1183 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1184 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1185 else
1186 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1187}
1188
1189
1190#ifdef VBOX_STRICT
1191/**
1192 * Gets the permission bits for the specified MSR in the MSR bitmap.
1193 *
1194 * @returns VBox status code.
1195 * @retval VINF_SUCCESS if the specified MSR is found.
1196 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1197 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1198 *
1199 * @param pVCpu The cross context virtual CPU structure.
1200 * @param uMsr The MSR.
1201 * @param penmRead Where to store the read permissions.
1202 * @param penmWrite Where to store the write permissions.
1203 */
1204static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1205{
1206 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1207 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1208 int32_t iBit;
1209 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1210
1211 /* See hmR0VmxSetMsrPermission() for the layout. */
1212 if (uMsr <= 0x00001FFF)
1213 iBit = uMsr;
1214 else if ( uMsr >= 0xC0000000
1215 && uMsr <= 0xC0001FFF)
1216 {
1217 iBit = (uMsr - 0xC0000000);
1218 pbMsrBitmap += 0x400;
1219 }
1220 else
1221 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1222
1223 Assert(iBit <= 0x1fff);
1224 if (ASMBitTest(pbMsrBitmap, iBit))
1225 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1226 else
1227 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1228
1229 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1230 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1231 else
1232 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1233 return VINF_SUCCESS;
1234}
1235#endif /* VBOX_STRICT */
1236
1237
1238/**
1239 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1240 * area.
1241 *
1242 * @returns VBox status code.
1243 * @param pVCpu The cross context virtual CPU structure.
1244 * @param cMsrs The number of MSRs.
1245 */
1246DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1247{
1248 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1249 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1250 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1251 {
1252 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1253 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1254 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1255 }
1256
1257 /* Update number of guest MSRs to load/store across the world-switch. */
1258 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1259 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1260
1261 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1262 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1263
1264 /* Update the VCPU's copy of the MSR count. */
1265 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1266
1267 return VINF_SUCCESS;
1268}
1269
1270
1271/**
1272 * Adds a new (or updates the value of an existing) guest/host MSR
1273 * pair to be swapped during the world-switch as part of the
1274 * auto-load/store MSR area in the VMCS.
1275 *
1276 * @returns VBox status code.
1277 * @param pVCpu The cross context virtual CPU structure.
1278 * @param uMsr The MSR.
1279 * @param uGuestMsrValue Value of the guest MSR.
1280 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1281 * necessary.
1282 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1283 * its value was updated. Optional, can be NULL.
1284 */
1285static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1286 bool *pfAddedAndUpdated)
1287{
1288 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1289 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1290 uint32_t i;
1291 for (i = 0; i < cMsrs; i++)
1292 {
1293 if (pGuestMsr->u32Msr == uMsr)
1294 break;
1295 pGuestMsr++;
1296 }
1297
1298 bool fAdded = false;
1299 if (i == cMsrs)
1300 {
1301 ++cMsrs;
1302 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1303 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1304
1305 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1306 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1307 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1308
1309 fAdded = true;
1310 }
1311
1312 /* Update the MSR values in the auto-load/store MSR area. */
1313 pGuestMsr->u32Msr = uMsr;
1314 pGuestMsr->u64Value = uGuestMsrValue;
1315
1316 /* Create/update the MSR slot in the host MSR area. */
1317 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1318 pHostMsr += i;
1319 pHostMsr->u32Msr = uMsr;
1320
1321 /*
1322 * Update the host MSR only when requested by the caller AND when we're
1323 * adding it to the auto-load/store area. Otherwise, it would have been
1324 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1325 */
1326 bool fUpdatedMsrValue = false;
1327 if ( fAdded
1328 && fUpdateHostMsr)
1329 {
1330 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1331 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1332 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1333 fUpdatedMsrValue = true;
1334 }
1335
1336 if (pfAddedAndUpdated)
1337 *pfAddedAndUpdated = fUpdatedMsrValue;
1338 return VINF_SUCCESS;
1339}
1340
1341
1342/**
1343 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1344 * auto-load/store MSR area in the VMCS.
1345 *
1346 * @returns VBox status code.
1347 * @param pVCpu The cross context virtual CPU structure.
1348 * @param uMsr The MSR.
1349 */
1350static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1351{
1352 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1353 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1354 for (uint32_t i = 0; i < cMsrs; i++)
1355 {
1356 /* Find the MSR. */
1357 if (pGuestMsr->u32Msr == uMsr)
1358 {
1359 /* If it's the last MSR, simply reduce the count. */
1360 if (i == cMsrs - 1)
1361 {
1362 --cMsrs;
1363 break;
1364 }
1365
1366 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1367 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1368 pLastGuestMsr += cMsrs - 1;
1369 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1370 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1371
1372 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1373 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1374 pLastHostMsr += cMsrs - 1;
1375 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1376 pHostMsr->u64Value = pLastHostMsr->u64Value;
1377 --cMsrs;
1378 break;
1379 }
1380 pGuestMsr++;
1381 }
1382
1383 /* Update the VMCS if the count changed (meaning the MSR was found). */
1384 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1385 {
1386 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1387 AssertRCReturn(rc, rc);
1388
1389 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1390 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1391 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1392
1393 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1394 return VINF_SUCCESS;
1395 }
1396
1397 return VERR_NOT_FOUND;
1398}
1399
1400
1401/**
1402 * Checks if the specified guest MSR is part of the auto-load/store area in
1403 * the VMCS.
1404 *
1405 * @returns true if found, false otherwise.
1406 * @param pVCpu The cross context virtual CPU structure.
1407 * @param uMsr The MSR to find.
1408 */
1409static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1410{
1411 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1412 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1413
1414 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1415 {
1416 if (pGuestMsr->u32Msr == uMsr)
1417 return true;
1418 }
1419 return false;
1420}
1421
1422
1423/**
1424 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1425 *
1426 * @param pVCpu The cross context virtual CPU structure.
1427 *
1428 * @remarks No-long-jump zone!!!
1429 */
1430static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1431{
1432 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1433 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1434 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1435 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1436
1437 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1438 {
1439 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1440
1441 /*
1442 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1443 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1444 */
1445 if (pHostMsr->u32Msr == MSR_K6_EFER)
1446 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1447 else
1448 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1449 }
1450
1451 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1452}
1453
1454
1455#if HC_ARCH_BITS == 64
1456/**
1457 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1458 * perform lazy restoration of the host MSRs while leaving VT-x.
1459 *
1460 * @param pVCpu The cross context virtual CPU structure.
1461 *
1462 * @remarks No-long-jump zone!!!
1463 */
1464static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1465{
1466 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1467
1468 /*
1469 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1470 */
1471 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1472 {
1473 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1474 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1475 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1476 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1477 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1478 }
1479}
1480
1481
1482/**
1483 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1484 * lazily while leaving VT-x.
1485 *
1486 * @returns true if it does, false otherwise.
1487 * @param pVCpu The cross context virtual CPU structure.
1488 * @param uMsr The MSR to check.
1489 */
1490static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1491{
1492 NOREF(pVCpu);
1493 switch (uMsr)
1494 {
1495 case MSR_K8_LSTAR:
1496 case MSR_K6_STAR:
1497 case MSR_K8_SF_MASK:
1498 case MSR_K8_KERNEL_GS_BASE:
1499 return true;
1500 }
1501 return false;
1502}
1503
1504
1505/**
1506 * Saves a set of guest MSRs back into the guest-CPU context.
1507 *
1508 * @param pVCpu The cross context virtual CPU structure.
1509 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1510 * out-of-sync. Make sure to update the required fields
1511 * before using them.
1512 *
1513 * @remarks No-long-jump zone!!!
1514 */
1515static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1516{
1517 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1518 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1519
1520 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1521 {
1522 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1523 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1524 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1525 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1526 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1527 }
1528}
1529
1530
1531/**
1532 * Loads a set of guests MSRs to allow read/passthru to the guest.
1533 *
1534 * The name of this function is slightly confusing. This function does NOT
1535 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1536 * common prefix for functions dealing with "lazy restoration" of the shared
1537 * MSRs.
1538 *
1539 * @param pVCpu The cross context virtual CPU structure.
1540 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1541 * out-of-sync. Make sure to update the required fields
1542 * before using them.
1543 *
1544 * @remarks No-long-jump zone!!!
1545 */
1546static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1547{
1548 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1549 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1550
1551#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1552 do { \
1553 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1554 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1555 else \
1556 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1557 } while (0)
1558
1559 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1560 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1561 {
1562 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1563 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1564 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1565 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1566 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1567 }
1568 else
1569 {
1570 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1571 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1572 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1573 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1574 }
1575
1576#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1577}
1578
1579
1580/**
1581 * Performs lazy restoration of the set of host MSRs if they were previously
1582 * loaded with guest MSR values.
1583 *
1584 * @param pVCpu The cross context virtual CPU structure.
1585 *
1586 * @remarks No-long-jump zone!!!
1587 * @remarks The guest MSRs should have been saved back into the guest-CPU
1588 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1589 */
1590static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1591{
1592 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1593 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1594
1595 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1596 {
1597 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1598 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1599 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1600 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1601 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1602 }
1603 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1604}
1605#endif /* HC_ARCH_BITS == 64 */
1606
1607
1608/**
1609 * Verifies that our cached values of the VMCS controls are all
1610 * consistent with what's actually present in the VMCS.
1611 *
1612 * @returns VBox status code.
1613 * @param pVCpu The cross context virtual CPU structure.
1614 */
1615static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1616{
1617 uint32_t u32Val;
1618 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1619 AssertRCReturn(rc, rc);
1620 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1621 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1622
1623 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1624 AssertRCReturn(rc, rc);
1625 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1626 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1627
1628 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1629 AssertRCReturn(rc, rc);
1630 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1631 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1632
1633 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1634 AssertRCReturn(rc, rc);
1635 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1636 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1637
1638 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1639 {
1640 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1641 AssertRCReturn(rc, rc);
1642 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1643 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1644 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1645 }
1646
1647 return VINF_SUCCESS;
1648}
1649
1650
1651#ifdef VBOX_STRICT
1652/**
1653 * Verifies that our cached host EFER value has not changed
1654 * since we cached it.
1655 *
1656 * @param pVCpu The cross context virtual CPU structure.
1657 */
1658static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1659{
1660 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1661
1662 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1663 {
1664 uint64_t u64Val;
1665 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, &u64Val);
1666 AssertRC(rc);
1667
1668 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1669 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1670 }
1671}
1672
1673
1674/**
1675 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1676 * VMCS are correct.
1677 *
1678 * @param pVCpu The cross context virtual CPU structure.
1679 */
1680static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1681{
1682 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1683
1684 /* Verify MSR counts in the VMCS are what we think it should be. */
1685 uint32_t cMsrs;
1686 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1687 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1688
1689 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1690 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1691
1692 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1693 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1694
1695 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1696 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1697 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1698 {
1699 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1700 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1701 pGuestMsr->u32Msr, cMsrs));
1702
1703 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1704 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1705 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1706
1707 /* Verify that the permissions are as expected in the MSR bitmap. */
1708 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1709 {
1710 VMXMSREXITREAD enmRead;
1711 VMXMSREXITWRITE enmWrite;
1712 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1713 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1714 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1715 {
1716 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1717 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1718 }
1719 else
1720 {
1721 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1722 pGuestMsr->u32Msr, cMsrs));
1723 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1724 pGuestMsr->u32Msr, cMsrs));
1725 }
1726 }
1727 }
1728}
1729#endif /* VBOX_STRICT */
1730
1731
1732/**
1733 * Flushes the TLB using EPT.
1734 *
1735 * @returns VBox status code.
1736 * @param pVCpu The cross context virtual CPU structure of the calling
1737 * EMT. Can be NULL depending on @a enmFlush.
1738 * @param enmFlush Type of flush.
1739 *
1740 * @remarks Caller is responsible for making sure this function is called only
1741 * when NestedPaging is supported and providing @a enmFlush that is
1742 * supported by the CPU.
1743 * @remarks Can be called with interrupts disabled.
1744 */
1745static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1746{
1747 uint64_t au64Descriptor[2];
1748 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1749 au64Descriptor[0] = 0;
1750 else
1751 {
1752 Assert(pVCpu);
1753 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1754 }
1755 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1756
1757 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1758 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1759 rc));
1760 if ( RT_SUCCESS(rc)
1761 && pVCpu)
1762 {
1763 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1764 }
1765}
1766
1767
1768/**
1769 * Flushes the TLB using VPID.
1770 *
1771 * @returns VBox status code.
1772 * @param pVM The cross context VM structure.
1773 * @param pVCpu The cross context virtual CPU structure of the calling
1774 * EMT. Can be NULL depending on @a enmFlush.
1775 * @param enmFlush Type of flush.
1776 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1777 * on @a enmFlush).
1778 *
1779 * @remarks Can be called with interrupts disabled.
1780 */
1781static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1782{
1783 NOREF(pVM);
1784 AssertPtr(pVM);
1785 Assert(pVM->hm.s.vmx.fVpid);
1786
1787 uint64_t au64Descriptor[2];
1788 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1789 {
1790 au64Descriptor[0] = 0;
1791 au64Descriptor[1] = 0;
1792 }
1793 else
1794 {
1795 AssertPtr(pVCpu);
1796 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1797 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1798 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1799 au64Descriptor[1] = GCPtr;
1800 }
1801
1802 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1803 AssertMsg(rc == VINF_SUCCESS,
1804 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1805 if ( RT_SUCCESS(rc)
1806 && pVCpu)
1807 {
1808 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1809 }
1810}
1811
1812
1813/**
1814 * Invalidates a guest page by guest virtual address. Only relevant for
1815 * EPT/VPID, otherwise there is nothing really to invalidate.
1816 *
1817 * @returns VBox status code.
1818 * @param pVM The cross context VM structure.
1819 * @param pVCpu The cross context virtual CPU structure.
1820 * @param GCVirt Guest virtual address of the page to invalidate.
1821 */
1822VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1823{
1824 AssertPtr(pVM);
1825 AssertPtr(pVCpu);
1826 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1827
1828 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1829 if (!fFlushPending)
1830 {
1831 /*
1832 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1833 * See @bugref{6043} and @bugref{6177}.
1834 *
1835 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1836 * function maybe called in a loop with individual addresses.
1837 */
1838 if (pVM->hm.s.vmx.fVpid)
1839 {
1840 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1841 {
1842 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1843 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1844 }
1845 else
1846 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1847 }
1848 else if (pVM->hm.s.fNestedPaging)
1849 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1850 }
1851
1852 return VINF_SUCCESS;
1853}
1854
1855
1856/**
1857 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1858 * otherwise there is nothing really to invalidate.
1859 *
1860 * @returns VBox status code.
1861 * @param pVM The cross context VM structure.
1862 * @param pVCpu The cross context virtual CPU structure.
1863 * @param GCPhys Guest physical address of the page to invalidate.
1864 */
1865VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1866{
1867 NOREF(pVM); NOREF(GCPhys);
1868 LogFlowFunc(("%RGp\n", GCPhys));
1869
1870 /*
1871 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1872 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1873 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1874 */
1875 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1876 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1877 return VINF_SUCCESS;
1878}
1879
1880
1881/**
1882 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1883 * case where neither EPT nor VPID is supported by the CPU.
1884 *
1885 * @param pVM The cross context VM structure.
1886 * @param pVCpu The cross context virtual CPU structure.
1887 * @param pCpu Pointer to the global HM struct.
1888 *
1889 * @remarks Called with interrupts disabled.
1890 */
1891static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1892{
1893 AssertPtr(pVCpu);
1894 AssertPtr(pCpu);
1895 NOREF(pVM);
1896
1897 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1898
1899 Assert(pCpu->idCpu != NIL_RTCPUID);
1900 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1901 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1902 pVCpu->hm.s.fForceTLBFlush = false;
1903 return;
1904}
1905
1906
1907/**
1908 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1909 *
1910 * @param pVM The cross context VM structure.
1911 * @param pVCpu The cross context virtual CPU structure.
1912 * @param pCpu Pointer to the global HM CPU struct.
1913 * @remarks All references to "ASID" in this function pertains to "VPID" in
1914 * Intel's nomenclature. The reason is, to avoid confusion in compare
1915 * statements since the host-CPU copies are named "ASID".
1916 *
1917 * @remarks Called with interrupts disabled.
1918 */
1919static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1920{
1921#ifdef VBOX_WITH_STATISTICS
1922 bool fTlbFlushed = false;
1923# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1924# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1925 if (!fTlbFlushed) \
1926 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1927 } while (0)
1928#else
1929# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1930# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1931#endif
1932
1933 AssertPtr(pVM);
1934 AssertPtr(pCpu);
1935 AssertPtr(pVCpu);
1936 Assert(pCpu->idCpu != NIL_RTCPUID);
1937
1938 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1939 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1940 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1941
1942 /*
1943 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1944 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1945 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1946 */
1947 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1948 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1949 {
1950 ++pCpu->uCurrentAsid;
1951 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1952 {
1953 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1954 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1955 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1956 }
1957
1958 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1959 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1960 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1961
1962 /*
1963 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1964 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1965 */
1966 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1967 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1968 HMVMX_SET_TAGGED_TLB_FLUSHED();
1969 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1970 }
1971
1972 /* Check for explicit TLB flushes. */
1973 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1974 {
1975 /*
1976 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1977 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1978 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1979 * but not guest-physical mappings.
1980 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1981 */
1982 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1983 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1984 HMVMX_SET_TAGGED_TLB_FLUSHED();
1985 }
1986
1987 pVCpu->hm.s.fForceTLBFlush = false;
1988 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1989
1990 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1991 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1992 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1993 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1994 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1995 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
1996 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
1997 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1998 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1999
2000 /* Update VMCS with the VPID. */
2001 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2002 AssertRC(rc);
2003
2004#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2005}
2006
2007
2008/**
2009 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2010 *
2011 * @returns VBox status code.
2012 * @param pVM The cross context VM structure.
2013 * @param pVCpu The cross context virtual CPU structure.
2014 * @param pCpu Pointer to the global HM CPU struct.
2015 *
2016 * @remarks Called with interrupts disabled.
2017 */
2018static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2019{
2020 AssertPtr(pVM);
2021 AssertPtr(pVCpu);
2022 AssertPtr(pCpu);
2023 Assert(pCpu->idCpu != NIL_RTCPUID);
2024 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2025 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2026
2027 /*
2028 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2029 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2030 */
2031 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2032 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2033 {
2034 pVCpu->hm.s.fForceTLBFlush = true;
2035 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2036 }
2037
2038 /* Check for explicit TLB flushes. */
2039 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2040 {
2041 pVCpu->hm.s.fForceTLBFlush = true;
2042 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2043 }
2044
2045 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2046 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2047
2048 if (pVCpu->hm.s.fForceTLBFlush)
2049 {
2050 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2051 pVCpu->hm.s.fForceTLBFlush = false;
2052 }
2053}
2054
2055
2056/**
2057 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2058 *
2059 * @returns VBox status code.
2060 * @param pVM The cross context VM structure.
2061 * @param pVCpu The cross context virtual CPU structure.
2062 * @param pCpu Pointer to the global HM CPU struct.
2063 *
2064 * @remarks Called with interrupts disabled.
2065 */
2066static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2067{
2068 AssertPtr(pVM);
2069 AssertPtr(pVCpu);
2070 AssertPtr(pCpu);
2071 Assert(pCpu->idCpu != NIL_RTCPUID);
2072 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2073 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2074
2075 /*
2076 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2077 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2078 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2079 */
2080 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2081 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2082 {
2083 pVCpu->hm.s.fForceTLBFlush = true;
2084 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2085 }
2086
2087 /* Check for explicit TLB flushes. */
2088 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2089 {
2090 /*
2091 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2092 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2093 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2094 */
2095 pVCpu->hm.s.fForceTLBFlush = true;
2096 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2097 }
2098
2099 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2100 if (pVCpu->hm.s.fForceTLBFlush)
2101 {
2102 ++pCpu->uCurrentAsid;
2103 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2104 {
2105 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2106 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2107 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2108 }
2109
2110 pVCpu->hm.s.fForceTLBFlush = false;
2111 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2112 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2113 if (pCpu->fFlushAsidBeforeUse)
2114 {
2115 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2116 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2117 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2118 {
2119 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2120 pCpu->fFlushAsidBeforeUse = false;
2121 }
2122 else
2123 {
2124 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2125 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2126 }
2127 }
2128 }
2129
2130 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2131 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2132 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2133 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2134 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2135 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2136 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2137
2138 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2139 AssertRC(rc);
2140}
2141
2142
2143/**
2144 * Flushes the guest TLB entry based on CPU capabilities.
2145 *
2146 * @param pVCpu The cross context virtual CPU structure.
2147 * @param pCpu Pointer to the global HM CPU struct.
2148 */
2149DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2150{
2151#ifdef HMVMX_ALWAYS_FLUSH_TLB
2152 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2153#endif
2154 PVM pVM = pVCpu->CTX_SUFF(pVM);
2155 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2156 {
2157 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2158 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2159 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2160 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2161 default:
2162 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2163 break;
2164 }
2165
2166 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2167}
2168
2169
2170/**
2171 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2172 * TLB entries from the host TLB before VM-entry.
2173 *
2174 * @returns VBox status code.
2175 * @param pVM The cross context VM structure.
2176 */
2177static int hmR0VmxSetupTaggedTlb(PVM pVM)
2178{
2179 /*
2180 * Determine optimal flush type for Nested Paging.
2181 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2182 * guest execution (see hmR3InitFinalizeR0()).
2183 */
2184 if (pVM->hm.s.fNestedPaging)
2185 {
2186 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2187 {
2188 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2189 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2190 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2191 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2192 else
2193 {
2194 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2195 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2196 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2197 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2198 }
2199
2200 /* Make sure the write-back cacheable memory type for EPT is supported. */
2201 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2202 {
2203 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2204 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2205 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2206 }
2207
2208 /* EPT requires a page-walk length of 4. */
2209 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2210 {
2211 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2212 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2213 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2214 }
2215 }
2216 else
2217 {
2218 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2219 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2220 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2221 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2222 }
2223 }
2224
2225 /*
2226 * Determine optimal flush type for VPID.
2227 */
2228 if (pVM->hm.s.vmx.fVpid)
2229 {
2230 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2231 {
2232 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2233 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2234 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2235 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2236 else
2237 {
2238 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2239 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2240 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2241 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2242 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2243 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2244 pVM->hm.s.vmx.fVpid = false;
2245 }
2246 }
2247 else
2248 {
2249 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2250 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2251 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2252 pVM->hm.s.vmx.fVpid = false;
2253 }
2254 }
2255
2256 /*
2257 * Setup the handler for flushing tagged-TLBs.
2258 */
2259 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2260 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2261 else if (pVM->hm.s.fNestedPaging)
2262 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2263 else if (pVM->hm.s.vmx.fVpid)
2264 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2265 else
2266 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2267 return VINF_SUCCESS;
2268}
2269
2270
2271/**
2272 * Sets up pin-based VM-execution controls in the VMCS.
2273 *
2274 * @returns VBox status code.
2275 * @param pVM The cross context VM structure.
2276 * @param pVCpu The cross context virtual CPU structure.
2277 */
2278static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2279{
2280 AssertPtr(pVM);
2281 AssertPtr(pVCpu);
2282
2283 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2284 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2285
2286 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2287 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2288
2289 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2290 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2291
2292 /* Enable the VMX preemption timer. */
2293 if (pVM->hm.s.vmx.fUsePreemptTimer)
2294 {
2295 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2296 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2297 }
2298
2299 if ((val & zap) != val)
2300 {
2301 LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2302 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2303 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2304 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2305 }
2306
2307 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2308 AssertRCReturn(rc, rc);
2309
2310 pVCpu->hm.s.vmx.u32PinCtls = val;
2311 return rc;
2312}
2313
2314
2315/**
2316 * Sets up processor-based VM-execution controls in the VMCS.
2317 *
2318 * @returns VBox status code.
2319 * @param pVM The cross context VM structure.
2320 * @param pVCpu The cross context virtual CPU structure.
2321 */
2322static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2323{
2324 AssertPtr(pVM);
2325 AssertPtr(pVCpu);
2326
2327 int rc = VERR_INTERNAL_ERROR_5;
2328 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2329 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2330
2331 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2332 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2333 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2334 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2335 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2336 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2337 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2338
2339 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2340 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2341 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2342 {
2343 LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2344 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2345 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2346 }
2347
2348 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2349 if (!pVM->hm.s.fNestedPaging)
2350 {
2351 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2352 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2353 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2354 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2355 }
2356
2357 /* Use TPR shadowing if supported by the CPU. */
2358 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2359 {
2360 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2361 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2362 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2363 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2364 AssertRCReturn(rc, rc);
2365
2366 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2367 /* CR8 writes cause a VM-exit based on TPR threshold. */
2368 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2369 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2370 }
2371 else
2372 {
2373 /*
2374 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2375 * Set this control only for 64-bit guests.
2376 */
2377 if (pVM->hm.s.fAllow64BitGuests)
2378 {
2379 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2380 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2381 }
2382 }
2383
2384 /* Use MSR-bitmaps if supported by the CPU. */
2385 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2386 {
2387 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2388
2389 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2390 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2391 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2392 AssertRCReturn(rc, rc);
2393
2394 /*
2395 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2396 * automatically using dedicated fields in the VMCS.
2397 */
2398 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2399 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2400 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2401 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2402 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2403
2404#if HC_ARCH_BITS == 64
2405 /*
2406 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2407 */
2408 if (pVM->hm.s.fAllow64BitGuests)
2409 {
2410 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2411 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2412 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2413 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2414 }
2415#endif
2416 }
2417
2418 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2419 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2420 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2421
2422 if ((val & zap) != val)
2423 {
2424 LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2425 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2426 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2427 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2428 }
2429
2430 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2431 AssertRCReturn(rc, rc);
2432
2433 pVCpu->hm.s.vmx.u32ProcCtls = val;
2434
2435 /*
2436 * Secondary processor-based VM-execution controls.
2437 */
2438 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2439 {
2440 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2441 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2442
2443 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2444 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2445
2446 if (pVM->hm.s.fNestedPaging)
2447 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2448 else
2449 {
2450 /*
2451 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2452 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2453 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2454 */
2455 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2456 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2457 }
2458
2459 if (pVM->hm.s.vmx.fVpid)
2460 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2461
2462 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2463 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2464
2465 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2466 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2467 * done dynamically. */
2468 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2469 {
2470 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2471 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2472 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2473 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2474 AssertRCReturn(rc, rc);
2475 }
2476
2477 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2478 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2479
2480 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2481 && pVM->hm.s.vmx.cPleGapTicks
2482 && pVM->hm.s.vmx.cPleWindowTicks)
2483 {
2484 val |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2485
2486 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2487 AssertRCReturn(rc, rc);
2488
2489 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2490 AssertRCReturn(rc, rc);
2491 }
2492
2493 if ((val & zap) != val)
2494 {
2495 LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2496 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2497 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2498 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2499 }
2500
2501 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2502 AssertRCReturn(rc, rc);
2503
2504 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2505 }
2506 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2507 {
2508 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2509 "available\n"));
2510 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2511 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2512 }
2513
2514 return VINF_SUCCESS;
2515}
2516
2517
2518/**
2519 * Sets up miscellaneous (everything other than Pin & Processor-based
2520 * VM-execution) control fields in the VMCS.
2521 *
2522 * @returns VBox status code.
2523 * @param pVM The cross context VM structure.
2524 * @param pVCpu The cross context virtual CPU structure.
2525 */
2526static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2527{
2528 NOREF(pVM);
2529 AssertPtr(pVM);
2530 AssertPtr(pVCpu);
2531
2532 int rc = VERR_GENERAL_FAILURE;
2533
2534 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2535#if 0
2536 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2537 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2538 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2539
2540 /*
2541 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2542 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2543 * We thus use the exception bitmap to control it rather than use both.
2544 */
2545 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2546 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2547
2548 /** @todo Explore possibility of using IO-bitmaps. */
2549 /* All IO & IOIO instructions cause VM-exits. */
2550 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2551 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2552
2553 /* Initialize the MSR-bitmap area. */
2554 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2555 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2556 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2557#endif
2558
2559 /* Setup MSR auto-load/store area. */
2560 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2561 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2562 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2563 AssertRCReturn(rc, rc);
2564 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2565 AssertRCReturn(rc, rc);
2566
2567 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2568 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2569 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2570 AssertRCReturn(rc, rc);
2571
2572 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2573 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2574 AssertRCReturn(rc, rc);
2575
2576 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2577#if 0
2578 /* Setup debug controls */
2579 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2580 AssertRCReturn(rc, rc);
2581 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2582 AssertRCReturn(rc, rc);
2583#endif
2584
2585 return rc;
2586}
2587
2588
2589/**
2590 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2591 *
2592 * @returns VBox status code.
2593 * @param pVM The cross context VM structure.
2594 * @param pVCpu The cross context virtual CPU structure.
2595 */
2596static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2597{
2598 AssertPtr(pVM);
2599 AssertPtr(pVCpu);
2600
2601 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2602
2603 uint32_t u32XcptBitmap = pVCpu->hm.s.fGIMTrapXcptUD ? RT_BIT(X86_XCPT_UD) : 0;
2604
2605 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2606 u32XcptBitmap |= RT_BIT_32(X86_XCPT_AC);
2607
2608 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2609 and writes, and because recursive #DBs can cause the CPU hang, we must always
2610 intercept #DB. */
2611 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2612
2613 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2614 if (!pVM->hm.s.fNestedPaging)
2615 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2616
2617 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2618 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2619 AssertRCReturn(rc, rc);
2620 return rc;
2621}
2622
2623
2624/**
2625 * Sets up the initial guest-state mask. The guest-state mask is consulted
2626 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2627 * for the nested virtualization case (as it would cause a VM-exit).
2628 *
2629 * @param pVCpu The cross context virtual CPU structure.
2630 */
2631static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2632{
2633 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2634 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2635 return VINF_SUCCESS;
2636}
2637
2638
2639/**
2640 * Does per-VM VT-x initialization.
2641 *
2642 * @returns VBox status code.
2643 * @param pVM The cross context VM structure.
2644 */
2645VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2646{
2647 LogFlowFunc(("pVM=%p\n", pVM));
2648
2649 int rc = hmR0VmxStructsAlloc(pVM);
2650 if (RT_FAILURE(rc))
2651 {
2652 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2653 return rc;
2654 }
2655
2656 return VINF_SUCCESS;
2657}
2658
2659
2660/**
2661 * Does per-VM VT-x termination.
2662 *
2663 * @returns VBox status code.
2664 * @param pVM The cross context VM structure.
2665 */
2666VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2667{
2668 LogFlowFunc(("pVM=%p\n", pVM));
2669
2670#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2671 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2672 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2673#endif
2674 hmR0VmxStructsFree(pVM);
2675 return VINF_SUCCESS;
2676}
2677
2678
2679/**
2680 * Sets up the VM for execution under VT-x.
2681 * This function is only called once per-VM during initialization.
2682 *
2683 * @returns VBox status code.
2684 * @param pVM The cross context VM structure.
2685 */
2686VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2687{
2688 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2689 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2690
2691 LogFlowFunc(("pVM=%p\n", pVM));
2692
2693 /*
2694 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2695 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2696 */
2697 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2698 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2699 || !pVM->hm.s.vmx.pRealModeTSS))
2700 {
2701 LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2702 return VERR_INTERNAL_ERROR;
2703 }
2704
2705 /* Initialize these always, see hmR3InitFinalizeR0().*/
2706 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2707 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2708
2709 /* Setup the tagged-TLB flush handlers. */
2710 int rc = hmR0VmxSetupTaggedTlb(pVM);
2711 if (RT_FAILURE(rc))
2712 {
2713 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2714 return rc;
2715 }
2716
2717 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2718 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2719#if HC_ARCH_BITS == 64
2720 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2721 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2722 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2723 {
2724 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2725 }
2726#endif
2727
2728 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2729 {
2730 PVMCPU pVCpu = &pVM->aCpus[i];
2731 AssertPtr(pVCpu);
2732 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2733
2734 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2735 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2736
2737 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2738 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2739 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2740
2741 /* Set revision dword at the beginning of the VMCS structure. */
2742 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2743
2744 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2745 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2746 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2747 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2748
2749 /* Load this VMCS as the current VMCS. */
2750 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2751 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2752 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2753
2754 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2755 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2756 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2757
2758 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2759 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2760 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2761
2762 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2763 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2764 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2765
2766 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2767 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2768 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2769
2770 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2771 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2772 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2773
2774#if HC_ARCH_BITS == 32
2775 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2776 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2777 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2778#endif
2779
2780 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2781 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2782 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2783 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2784
2785 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2786
2787 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2788 }
2789
2790 return VINF_SUCCESS;
2791}
2792
2793
2794/**
2795 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2796 * the VMCS.
2797 *
2798 * @returns VBox status code.
2799 * @param pVM The cross context VM structure.
2800 * @param pVCpu The cross context virtual CPU structure.
2801 */
2802DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2803{
2804 NOREF(pVM); NOREF(pVCpu);
2805
2806 RTCCUINTREG uReg = ASMGetCR0();
2807 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2808 AssertRCReturn(rc, rc);
2809
2810 uReg = ASMGetCR3();
2811 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2812 AssertRCReturn(rc, rc);
2813
2814 uReg = ASMGetCR4();
2815 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2816 AssertRCReturn(rc, rc);
2817 return rc;
2818}
2819
2820
2821#if HC_ARCH_BITS == 64
2822/**
2823 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2824 * requirements. See hmR0VmxSaveHostSegmentRegs().
2825 */
2826# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2827 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2828 { \
2829 bool fValidSelector = true; \
2830 if ((selValue) & X86_SEL_LDT) \
2831 { \
2832 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2833 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2834 } \
2835 if (fValidSelector) \
2836 { \
2837 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2838 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2839 } \
2840 (selValue) = 0; \
2841 }
2842#endif
2843
2844
2845/**
2846 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2847 * the host-state area in the VMCS.
2848 *
2849 * @returns VBox status code.
2850 * @param pVM The cross context VM structure.
2851 * @param pVCpu The cross context virtual CPU structure.
2852 */
2853DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2854{
2855 int rc = VERR_INTERNAL_ERROR_5;
2856
2857#if HC_ARCH_BITS == 64
2858 /*
2859 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2860 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2861 */
2862 AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2863 ("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2864#endif
2865
2866 /*
2867 * Host DS, ES, FS and GS segment registers.
2868 */
2869#if HC_ARCH_BITS == 64
2870 RTSEL uSelDS = ASMGetDS();
2871 RTSEL uSelES = ASMGetES();
2872 RTSEL uSelFS = ASMGetFS();
2873 RTSEL uSelGS = ASMGetGS();
2874#else
2875 RTSEL uSelDS = 0;
2876 RTSEL uSelES = 0;
2877 RTSEL uSelFS = 0;
2878 RTSEL uSelGS = 0;
2879#endif
2880
2881 /* Recalculate which host-state bits need to be manually restored. */
2882 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2883
2884 /*
2885 * Host CS and SS segment registers.
2886 */
2887 RTSEL uSelCS = ASMGetCS();
2888 RTSEL uSelSS = ASMGetSS();
2889
2890 /*
2891 * Host TR segment register.
2892 */
2893 RTSEL uSelTR = ASMGetTR();
2894
2895#if HC_ARCH_BITS == 64
2896 /*
2897 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2898 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2899 */
2900 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2901 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2902 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2903 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2904# undef VMXLOCAL_ADJUST_HOST_SEG
2905#endif
2906
2907 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2908 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2909 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2910 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2911 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2912 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2913 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2914 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2915 Assert(uSelCS);
2916 Assert(uSelTR);
2917
2918 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2919#if 0
2920 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2921 Assert(uSelSS != 0);
2922#endif
2923
2924 /* Write these host selector fields into the host-state area in the VMCS. */
2925 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2926 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2927#if HC_ARCH_BITS == 64
2928 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2929 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2930 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2931 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2932#else
2933 NOREF(uSelDS);
2934 NOREF(uSelES);
2935 NOREF(uSelFS);
2936 NOREF(uSelGS);
2937#endif
2938 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2939
2940 /*
2941 * Host GDTR and IDTR.
2942 */
2943 RTGDTR Gdtr;
2944 RTIDTR Idtr;
2945 RT_ZERO(Gdtr);
2946 RT_ZERO(Idtr);
2947 ASMGetGDTR(&Gdtr);
2948 ASMGetIDTR(&Idtr);
2949 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2950 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2951
2952#if HC_ARCH_BITS == 64
2953 /*
2954 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2955 * maximum limit (0xffff) on every VM-exit.
2956 */
2957 if (Gdtr.cbGdt != 0xffff)
2958 {
2959 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2960 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2961 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2962 }
2963
2964 /*
2965 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
2966 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
2967 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
2968 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
2969 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
2970 * hosts where we are pretty sure it won't cause trouble.
2971 */
2972# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
2973 if (Idtr.cbIdt < 0x0fff)
2974# else
2975 if (Idtr.cbIdt != 0xffff)
2976# endif
2977 {
2978 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
2979 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2980 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2981 }
2982#endif
2983
2984 /*
2985 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2986 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2987 */
2988 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
2989 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
2990 VERR_VMX_INVALID_HOST_STATE);
2991
2992 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2993#if HC_ARCH_BITS == 64
2994 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
2995
2996 /*
2997 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2998 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2999 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3000 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3001 *
3002 * [1] See Intel spec. 3.5 "System Descriptor Types".
3003 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3004 */
3005 Assert(pDesc->System.u4Type == 11);
3006 if ( pDesc->System.u16LimitLow != 0x67
3007 || pDesc->System.u4LimitHigh)
3008 {
3009 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3010 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3011 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3012 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3013 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3014
3015 /* Store the GDTR here as we need it while restoring TR. */
3016 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3017 }
3018#else
3019 NOREF(pVM);
3020 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3021#endif
3022 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3023 AssertRCReturn(rc, rc);
3024
3025 /*
3026 * Host FS base and GS base.
3027 */
3028#if HC_ARCH_BITS == 64
3029 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3030 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3031 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
3032 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
3033
3034 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3035 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3036 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3037 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3038 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3039#endif
3040 return rc;
3041}
3042
3043
3044/**
3045 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3046 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3047 * the host after every successful VM-exit.
3048 *
3049 * @returns VBox status code.
3050 * @param pVM The cross context VM structure.
3051 * @param pVCpu The cross context virtual CPU structure.
3052 *
3053 * @remarks No-long-jump zone!!!
3054 */
3055DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3056{
3057 NOREF(pVM);
3058
3059 AssertPtr(pVCpu);
3060 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3061
3062 int rc = VINF_SUCCESS;
3063#if HC_ARCH_BITS == 64
3064 if (pVM->hm.s.fAllow64BitGuests)
3065 hmR0VmxLazySaveHostMsrs(pVCpu);
3066#endif
3067
3068 /*
3069 * Host Sysenter MSRs.
3070 */
3071 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3072 AssertRCReturn(rc, rc);
3073#if HC_ARCH_BITS == 32
3074 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3075 AssertRCReturn(rc, rc);
3076 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3077#else
3078 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3079 AssertRCReturn(rc, rc);
3080 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3081#endif
3082 AssertRCReturn(rc, rc);
3083
3084 /*
3085 * Host EFER MSR.
3086 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3087 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3088 */
3089 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3090 {
3091 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3092 AssertRCReturn(rc, rc);
3093 }
3094
3095 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3096 * hmR0VmxLoadGuestExitCtls() !! */
3097
3098 return rc;
3099}
3100
3101
3102/**
3103 * Figures out if we need to swap the EFER MSR which is
3104 * particularly expensive.
3105 *
3106 * We check all relevant bits. For now, that's everything
3107 * besides LMA/LME, as these two bits are handled by VM-entry,
3108 * see hmR0VmxLoadGuestExitCtls() and
3109 * hmR0VMxLoadGuestEntryCtls().
3110 *
3111 * @returns true if we need to load guest EFER, false otherwise.
3112 * @param pVCpu The cross context virtual CPU structure.
3113 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3114 * out-of-sync. Make sure to update the required fields
3115 * before using them.
3116 *
3117 * @remarks Requires EFER, CR4.
3118 * @remarks No-long-jump zone!!!
3119 */
3120static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3121{
3122#ifdef HMVMX_ALWAYS_SWAP_EFER
3123 return true;
3124#endif
3125
3126#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3127 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3128 if (CPUMIsGuestInLongMode(pVCpu))
3129 return false;
3130#endif
3131
3132 PVM pVM = pVCpu->CTX_SUFF(pVM);
3133 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3134 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3135
3136 /*
3137 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3138 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3139 */
3140 if ( CPUMIsGuestInLongMode(pVCpu)
3141 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3142 {
3143 return true;
3144 }
3145
3146 /*
3147 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3148 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3149 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3150 */
3151 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3152 && (pMixedCtx->cr0 & X86_CR0_PG)
3153 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3154 {
3155 /* Assert that host is PAE capable. */
3156 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3157 return true;
3158 }
3159
3160 /** @todo Check the latest Intel spec. for any other bits,
3161 * like SMEP/SMAP? */
3162 return false;
3163}
3164
3165
3166/**
3167 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3168 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3169 * controls".
3170 *
3171 * @returns VBox status code.
3172 * @param pVCpu The cross context virtual CPU structure.
3173 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3174 * out-of-sync. Make sure to update the required fields
3175 * before using them.
3176 *
3177 * @remarks Requires EFER.
3178 * @remarks No-long-jump zone!!!
3179 */
3180DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3181{
3182 int rc = VINF_SUCCESS;
3183 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3184 {
3185 PVM pVM = pVCpu->CTX_SUFF(pVM);
3186 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3187 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3188
3189 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3190 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3191
3192 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3193 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3194 {
3195 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3196 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3197 }
3198 else
3199 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3200
3201 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3202 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3203 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3204 {
3205 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3206 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3207 }
3208
3209 /*
3210 * The following should -not- be set (since we're not in SMM mode):
3211 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3212 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3213 */
3214
3215 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3216 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3217
3218 if ((val & zap) != val)
3219 {
3220 LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3221 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3222 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3223 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3224 }
3225
3226 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3227 AssertRCReturn(rc, rc);
3228
3229 pVCpu->hm.s.vmx.u32EntryCtls = val;
3230 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3231 }
3232 return rc;
3233}
3234
3235
3236/**
3237 * Sets up the VM-exit controls in the VMCS.
3238 *
3239 * @returns VBox status code.
3240 * @param pVCpu The cross context virtual CPU structure.
3241 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3242 * out-of-sync. Make sure to update the required fields
3243 * before using them.
3244 *
3245 * @remarks Requires EFER.
3246 */
3247DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3248{
3249 NOREF(pMixedCtx);
3250
3251 int rc = VINF_SUCCESS;
3252 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3253 {
3254 PVM pVM = pVCpu->CTX_SUFF(pVM);
3255 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3256 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3257
3258 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3259 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3260
3261 /*
3262 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3263 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3264 */
3265#if HC_ARCH_BITS == 64
3266 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3267 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3268#else
3269 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3270 {
3271 /* The switcher returns to long mode, EFER is managed by the switcher. */
3272 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3273 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3274 }
3275 else
3276 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3277#endif
3278
3279 /* If the newer VMCS fields for managing EFER exists, use it. */
3280 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3281 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3282 {
3283 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3284 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3285 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3286 }
3287
3288 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3289 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3290
3291 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3292 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3293 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3294
3295 if ( pVM->hm.s.vmx.fUsePreemptTimer
3296 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3297 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3298
3299 if ((val & zap) != val)
3300 {
3301 LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3302 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3303 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3304 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3305 }
3306
3307 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3308 AssertRCReturn(rc, rc);
3309
3310 pVCpu->hm.s.vmx.u32ExitCtls = val;
3311 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3312 }
3313 return rc;
3314}
3315
3316
3317/**
3318 * Loads the guest APIC and related state.
3319 *
3320 * @returns VBox status code.
3321 * @param pVCpu The cross context virtual CPU structure.
3322 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3323 * out-of-sync. Make sure to update the required fields
3324 * before using them.
3325 */
3326DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3327{
3328 NOREF(pMixedCtx);
3329
3330 int rc = VINF_SUCCESS;
3331 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3332 {
3333 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3334 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3335 {
3336 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3337
3338 bool fPendingIntr = false;
3339 uint8_t u8Tpr = 0;
3340 uint8_t u8PendingIntr = 0;
3341 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3342 AssertRCReturn(rc, rc);
3343
3344 /*
3345 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3346 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3347 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3348 * the interrupt when we VM-exit for other reasons.
3349 */
3350 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3351 uint32_t u32TprThreshold = 0;
3352 if (fPendingIntr)
3353 {
3354 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3355 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3356 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3357 if (u8PendingPriority <= u8TprPriority)
3358 u32TprThreshold = u8PendingPriority;
3359 else
3360 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3361 }
3362 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3363
3364 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3365 AssertRCReturn(rc, rc);
3366 }
3367
3368 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3369 }
3370 return rc;
3371}
3372
3373
3374/**
3375 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3376 *
3377 * @returns Guest's interruptibility-state.
3378 * @param pVCpu The cross context virtual CPU structure.
3379 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3380 * out-of-sync. Make sure to update the required fields
3381 * before using them.
3382 *
3383 * @remarks No-long-jump zone!!!
3384 */
3385DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3386{
3387 /*
3388 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3389 */
3390 uint32_t uIntrState = 0;
3391 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3392 {
3393 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3394 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3395 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3396 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3397 {
3398 if (pMixedCtx->eflags.Bits.u1IF)
3399 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3400 else
3401 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3402 }
3403 /* else: Although we can clear the force-flag here, let's keep this side-effects free. */
3404 }
3405
3406 /*
3407 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3408 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3409 * setting this would block host-NMIs and IRET will not clear the blocking.
3410 *
3411 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3412 */
3413 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3414 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3415 {
3416 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3417 }
3418
3419 return uIntrState;
3420}
3421
3422
3423/**
3424 * Loads the guest's interruptibility-state into the guest-state area in the
3425 * VMCS.
3426 *
3427 * @returns VBox status code.
3428 * @param pVCpu The cross context virtual CPU structure.
3429 * @param uIntrState The interruptibility-state to set.
3430 */
3431static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3432{
3433 NOREF(pVCpu);
3434 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3435 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3436 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3437 AssertRC(rc);
3438 return rc;
3439}
3440
3441
3442/**
3443 * Loads the exception intercepts required for guest execution in the VMCS.
3444 *
3445 * @returns VBox status code.
3446 * @param pVCpu The cross context virtual CPU structure.
3447 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3448 * out-of-sync. Make sure to update the required fields
3449 * before using them.
3450 */
3451static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3452{
3453 NOREF(pMixedCtx);
3454 int rc = VINF_SUCCESS;
3455 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3456 {
3457 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3458 if (pVCpu->hm.s.fGIMTrapXcptUD)
3459 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3460#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3461 else
3462 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3463#endif
3464
3465 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3466 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3467
3468 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3469 AssertRCReturn(rc, rc);
3470
3471 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3472 Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3473 pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3474 }
3475 return rc;
3476}
3477
3478
3479/**
3480 * Loads the guest's RIP into the guest-state area in the VMCS.
3481 *
3482 * @returns VBox status code.
3483 * @param pVCpu The cross context virtual CPU structure.
3484 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3485 * out-of-sync. Make sure to update the required fields
3486 * before using them.
3487 *
3488 * @remarks No-long-jump zone!!!
3489 */
3490static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3491{
3492 int rc = VINF_SUCCESS;
3493 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3494 {
3495 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3496 AssertRCReturn(rc, rc);
3497
3498 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3499 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3500 HMCPU_CF_VALUE(pVCpu)));
3501 }
3502 return rc;
3503}
3504
3505
3506/**
3507 * Loads the guest's RSP into the guest-state area in the VMCS.
3508 *
3509 * @returns VBox status code.
3510 * @param pVCpu The cross context virtual CPU structure.
3511 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3512 * out-of-sync. Make sure to update the required fields
3513 * before using them.
3514 *
3515 * @remarks No-long-jump zone!!!
3516 */
3517static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3518{
3519 int rc = VINF_SUCCESS;
3520 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3521 {
3522 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3523 AssertRCReturn(rc, rc);
3524
3525 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3526 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3527 }
3528 return rc;
3529}
3530
3531
3532/**
3533 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3534 *
3535 * @returns VBox status code.
3536 * @param pVCpu The cross context virtual CPU structure.
3537 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3538 * out-of-sync. Make sure to update the required fields
3539 * before using them.
3540 *
3541 * @remarks No-long-jump zone!!!
3542 */
3543static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3544{
3545 int rc = VINF_SUCCESS;
3546 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3547 {
3548 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3549 Let us assert it as such and use 32-bit VMWRITE. */
3550 Assert(!(pMixedCtx->rflags.u64 >> 32));
3551 X86EFLAGS Eflags = pMixedCtx->eflags;
3552 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3553 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3554 * These will never be cleared/set, unless some other part of the VMM
3555 * code is buggy - in which case we're better of finding and fixing
3556 * those bugs than hiding them. */
3557 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3558 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3559 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3560 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3561
3562 /*
3563 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3564 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3565 */
3566 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3567 {
3568 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3569 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3570 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3571 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3572 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3573 }
3574
3575 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3576 AssertRCReturn(rc, rc);
3577
3578 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3579 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3580 }
3581 return rc;
3582}
3583
3584
3585/**
3586 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3587 *
3588 * @returns VBox status code.
3589 * @param pVCpu The cross context virtual CPU structure.
3590 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3591 * out-of-sync. Make sure to update the required fields
3592 * before using them.
3593 *
3594 * @remarks No-long-jump zone!!!
3595 */
3596DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3597{
3598 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3599 AssertRCReturn(rc, rc);
3600 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3601 AssertRCReturn(rc, rc);
3602 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3603 AssertRCReturn(rc, rc);
3604 return rc;
3605}
3606
3607
3608/**
3609 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3610 * CR0 is partially shared with the host and we have to consider the FPU bits.
3611 *
3612 * @returns VBox status code.
3613 * @param pVCpu The cross context virtual CPU structure.
3614 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3615 * out-of-sync. Make sure to update the required fields
3616 * before using them.
3617 *
3618 * @remarks No-long-jump zone!!!
3619 */
3620static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3621{
3622 /*
3623 * Guest CR0.
3624 * Guest FPU.
3625 */
3626 int rc = VINF_SUCCESS;
3627 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3628 {
3629 Assert(!(pMixedCtx->cr0 >> 32));
3630 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3631 PVM pVM = pVCpu->CTX_SUFF(pVM);
3632
3633 /* The guest's view (read access) of its CR0 is unblemished. */
3634 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3635 AssertRCReturn(rc, rc);
3636 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3637
3638 /* Setup VT-x's view of the guest CR0. */
3639 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3640 if (pVM->hm.s.fNestedPaging)
3641 {
3642 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3643 {
3644 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3645 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3646 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3647 }
3648 else
3649 {
3650 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3651 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3652 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3653 }
3654
3655 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3656 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3657 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3658
3659 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3660 AssertRCReturn(rc, rc);
3661 }
3662 else
3663 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3664
3665 /*
3666 * Guest FPU bits.
3667 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3668 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3669 */
3670 u32GuestCR0 |= X86_CR0_NE;
3671 bool fInterceptNM = false;
3672 if (CPUMIsGuestFPUStateActive(pVCpu))
3673 {
3674 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3675 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3676 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3677 }
3678 else
3679 {
3680 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3681 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3682 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3683 }
3684
3685 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3686 bool fInterceptMF = false;
3687 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3688 fInterceptMF = true;
3689
3690 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3691 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3692 {
3693 Assert(PDMVmmDevHeapIsEnabled(pVM));
3694 Assert(pVM->hm.s.vmx.pRealModeTSS);
3695 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3696 fInterceptNM = true;
3697 fInterceptMF = true;
3698 }
3699 else
3700 {
3701 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3702 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3703 }
3704 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3705
3706 if (fInterceptNM)
3707 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3708 else
3709 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3710
3711 if (fInterceptMF)
3712 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3713 else
3714 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3715
3716 /* Additional intercepts for debugging, define these yourself explicitly. */
3717#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3718 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3719 | RT_BIT(X86_XCPT_BP)
3720 | RT_BIT(X86_XCPT_DE)
3721 | RT_BIT(X86_XCPT_NM)
3722 | RT_BIT(X86_XCPT_TS)
3723 | RT_BIT(X86_XCPT_UD)
3724 | RT_BIT(X86_XCPT_NP)
3725 | RT_BIT(X86_XCPT_SS)
3726 | RT_BIT(X86_XCPT_GP)
3727 | RT_BIT(X86_XCPT_PF)
3728 | RT_BIT(X86_XCPT_MF)
3729 ;
3730#elif defined(HMVMX_ALWAYS_TRAP_PF)
3731 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3732#endif
3733
3734 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3735
3736 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3737 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3738 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3739 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3740 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3741 else
3742 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3743
3744 u32GuestCR0 |= uSetCR0;
3745 u32GuestCR0 &= uZapCR0;
3746 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3747
3748 /* Write VT-x's view of the guest CR0 into the VMCS. */
3749 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3750 AssertRCReturn(rc, rc);
3751 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3752 uZapCR0));
3753
3754 /*
3755 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3756 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3757 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3758 */
3759 uint32_t u32CR0Mask = 0;
3760 u32CR0Mask = X86_CR0_PE
3761 | X86_CR0_NE
3762 | X86_CR0_WP
3763 | X86_CR0_PG
3764 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3765 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3766 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3767
3768 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3769 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3770 * and @bugref{6944}. */
3771#if 0
3772 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3773 u32CR0Mask &= ~X86_CR0_PE;
3774#endif
3775 if (pVM->hm.s.fNestedPaging)
3776 u32CR0Mask &= ~X86_CR0_WP;
3777
3778 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3779 if (fInterceptNM)
3780 {
3781 u32CR0Mask |= X86_CR0_TS
3782 | X86_CR0_MP;
3783 }
3784
3785 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3786 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3787 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3788 AssertRCReturn(rc, rc);
3789 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3790
3791 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3792 }
3793 return rc;
3794}
3795
3796
3797/**
3798 * Loads the guest control registers (CR3, CR4) into the guest-state area
3799 * in the VMCS.
3800 *
3801 * @returns VBox status code.
3802 * @param pVCpu The cross context virtual CPU structure.
3803 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3804 * out-of-sync. Make sure to update the required fields
3805 * before using them.
3806 *
3807 * @remarks No-long-jump zone!!!
3808 */
3809static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3810{
3811 int rc = VINF_SUCCESS;
3812 PVM pVM = pVCpu->CTX_SUFF(pVM);
3813
3814 /*
3815 * Guest CR2.
3816 * It's always loaded in the assembler code. Nothing to do here.
3817 */
3818
3819 /*
3820 * Guest CR3.
3821 */
3822 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3823 {
3824 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3825 if (pVM->hm.s.fNestedPaging)
3826 {
3827 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3828
3829 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3830 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3831 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3832 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3833
3834 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3835 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3836 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3837
3838 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3839 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3840 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3841 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3842 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3843 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3844 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3845
3846 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3847 AssertRCReturn(rc, rc);
3848 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3849
3850 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3851 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3852 {
3853 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3854 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3855 {
3856 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3857 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3858 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3859 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3860 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3861 }
3862
3863 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3864 have Unrestricted Execution to handle the guest when it's not using paging. */
3865 GCPhysGuestCR3 = pMixedCtx->cr3;
3866 }
3867 else
3868 {
3869 /*
3870 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3871 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3872 * EPT takes care of translating it to host-physical addresses.
3873 */
3874 RTGCPHYS GCPhys;
3875 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3876 Assert(PDMVmmDevHeapIsEnabled(pVM));
3877
3878 /* We obtain it here every time as the guest could have relocated this PCI region. */
3879 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3880 AssertRCReturn(rc, rc);
3881
3882 GCPhysGuestCR3 = GCPhys;
3883 }
3884
3885 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3886 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3887 }
3888 else
3889 {
3890 /* Non-nested paging case, just use the hypervisor's CR3. */
3891 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3892
3893 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
3894 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3895 }
3896 AssertRCReturn(rc, rc);
3897
3898 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3899 }
3900
3901 /*
3902 * Guest CR4.
3903 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3904 */
3905 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3906 {
3907 Assert(!(pMixedCtx->cr4 >> 32));
3908 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3909
3910 /* The guest's view of its CR4 is unblemished. */
3911 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3912 AssertRCReturn(rc, rc);
3913 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
3914
3915 /* Setup VT-x's view of the guest CR4. */
3916 /*
3917 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3918 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3919 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3920 */
3921 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3922 {
3923 Assert(pVM->hm.s.vmx.pRealModeTSS);
3924 Assert(PDMVmmDevHeapIsEnabled(pVM));
3925 u32GuestCR4 &= ~X86_CR4_VME;
3926 }
3927
3928 if (pVM->hm.s.fNestedPaging)
3929 {
3930 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3931 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3932 {
3933 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3934 u32GuestCR4 |= X86_CR4_PSE;
3935 /* Our identity mapping is a 32-bit page directory. */
3936 u32GuestCR4 &= ~X86_CR4_PAE;
3937 }
3938 /* else use guest CR4.*/
3939 }
3940 else
3941 {
3942 /*
3943 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3944 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3945 */
3946 switch (pVCpu->hm.s.enmShadowMode)
3947 {
3948 case PGMMODE_REAL: /* Real-mode. */
3949 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3950 case PGMMODE_32_BIT: /* 32-bit paging. */
3951 {
3952 u32GuestCR4 &= ~X86_CR4_PAE;
3953 break;
3954 }
3955
3956 case PGMMODE_PAE: /* PAE paging. */
3957 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3958 {
3959 u32GuestCR4 |= X86_CR4_PAE;
3960 break;
3961 }
3962
3963 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3964 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3965#ifdef VBOX_ENABLE_64_BITS_GUESTS
3966 break;
3967#endif
3968 default:
3969 AssertFailed();
3970 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3971 }
3972 }
3973
3974 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3975 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3976 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3977 u32GuestCR4 |= uSetCR4;
3978 u32GuestCR4 &= uZapCR4;
3979
3980 /* Write VT-x's view of the guest CR4 into the VMCS. */
3981 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
3982 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3983 AssertRCReturn(rc, rc);
3984
3985 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
3986 uint32_t u32CR4Mask = X86_CR4_VME
3987 | X86_CR4_PAE
3988 | X86_CR4_PGE
3989 | X86_CR4_PSE
3990 | X86_CR4_VMXE;
3991 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
3992 u32CR4Mask |= X86_CR4_OSXSAVE;
3993 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3994 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3995 AssertRCReturn(rc, rc);
3996
3997 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
3998 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
3999
4000 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4001 }
4002 return rc;
4003}
4004
4005
4006/**
4007 * Loads the guest debug registers into the guest-state area in the VMCS.
4008 *
4009 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4010 *
4011 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4012 *
4013 * @returns VBox status code.
4014 * @param pVCpu The cross context virtual CPU structure.
4015 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4016 * out-of-sync. Make sure to update the required fields
4017 * before using them.
4018 *
4019 * @remarks No-long-jump zone!!!
4020 */
4021static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4022{
4023 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4024 return VINF_SUCCESS;
4025
4026#ifdef VBOX_STRICT
4027 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4028 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4029 {
4030 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4031 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4032 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4033 }
4034#endif
4035
4036 int rc;
4037 PVM pVM = pVCpu->CTX_SUFF(pVM);
4038 bool fSteppingDB = false;
4039 bool fInterceptMovDRx = false;
4040 if (pVCpu->hm.s.fSingleInstruction)
4041 {
4042 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4043 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4044 {
4045 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4046 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4047 AssertRCReturn(rc, rc);
4048 Assert(fSteppingDB == false);
4049 }
4050 else
4051 {
4052 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4053 pVCpu->hm.s.fClearTrapFlag = true;
4054 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4055 fSteppingDB = true;
4056 }
4057 }
4058
4059 if ( fSteppingDB
4060 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4061 {
4062 /*
4063 * Use the combined guest and host DRx values found in the hypervisor
4064 * register set because the debugger has breakpoints active or someone
4065 * is single stepping on the host side without a monitor trap flag.
4066 *
4067 * Note! DBGF expects a clean DR6 state before executing guest code.
4068 */
4069#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4070 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4071 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4072 {
4073 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4074 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4075 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4076 }
4077 else
4078#endif
4079 if (!CPUMIsHyperDebugStateActive(pVCpu))
4080 {
4081 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4082 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4083 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4084 }
4085
4086 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4087 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4088 AssertRCReturn(rc, rc);
4089
4090 pVCpu->hm.s.fUsingHyperDR7 = true;
4091 fInterceptMovDRx = true;
4092 }
4093 else
4094 {
4095 /*
4096 * If the guest has enabled debug registers, we need to load them prior to
4097 * executing guest code so they'll trigger at the right time.
4098 */
4099 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4100 {
4101#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4102 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4103 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4104 {
4105 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4106 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4107 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4108 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4109 }
4110 else
4111#endif
4112 if (!CPUMIsGuestDebugStateActive(pVCpu))
4113 {
4114 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4115 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4116 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4117 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4118 }
4119 Assert(!fInterceptMovDRx);
4120 }
4121 /*
4122 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4123 * must intercept #DB in order to maintain a correct DR6 guest value, and
4124 * because we need to intercept it to prevent nested #DBs from hanging the
4125 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4126 */
4127#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4128 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4129 && !CPUMIsGuestDebugStateActive(pVCpu))
4130#else
4131 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4132#endif
4133 {
4134 fInterceptMovDRx = true;
4135 }
4136
4137 /* Update guest DR7. */
4138 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4139 AssertRCReturn(rc, rc);
4140
4141 pVCpu->hm.s.fUsingHyperDR7 = false;
4142 }
4143
4144 /*
4145 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4146 */
4147 if (fInterceptMovDRx)
4148 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4149 else
4150 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4151 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4152 AssertRCReturn(rc, rc);
4153
4154 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4155 return VINF_SUCCESS;
4156}
4157
4158
4159#ifdef VBOX_STRICT
4160/**
4161 * Strict function to validate segment registers.
4162 *
4163 * @remarks ASSUMES CR0 is up to date.
4164 */
4165static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4166{
4167 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4168 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4169 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4170 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4171 && ( !CPUMIsGuestInRealModeEx(pCtx)
4172 && !CPUMIsGuestInV86ModeEx(pCtx)))
4173 {
4174 /* Protected mode checks */
4175 /* CS */
4176 Assert(pCtx->cs.Attr.n.u1Present);
4177 Assert(!(pCtx->cs.Attr.u & 0xf00));
4178 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4179 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4180 || !(pCtx->cs.Attr.n.u1Granularity));
4181 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4182 || (pCtx->cs.Attr.n.u1Granularity));
4183 /* CS cannot be loaded with NULL in protected mode. */
4184 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4185 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4186 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4187 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4188 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4189 else
4190 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4191 /* SS */
4192 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4193 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4194 if ( !(pCtx->cr0 & X86_CR0_PE)
4195 || pCtx->cs.Attr.n.u4Type == 3)
4196 {
4197 Assert(!pCtx->ss.Attr.n.u2Dpl);
4198 }
4199 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4200 {
4201 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4202 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4203 Assert(pCtx->ss.Attr.n.u1Present);
4204 Assert(!(pCtx->ss.Attr.u & 0xf00));
4205 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4206 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4207 || !(pCtx->ss.Attr.n.u1Granularity));
4208 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4209 || (pCtx->ss.Attr.n.u1Granularity));
4210 }
4211 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4212 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4213 {
4214 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4215 Assert(pCtx->ds.Attr.n.u1Present);
4216 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4217 Assert(!(pCtx->ds.Attr.u & 0xf00));
4218 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4219 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4220 || !(pCtx->ds.Attr.n.u1Granularity));
4221 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4222 || (pCtx->ds.Attr.n.u1Granularity));
4223 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4224 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4225 }
4226 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4227 {
4228 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4229 Assert(pCtx->es.Attr.n.u1Present);
4230 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4231 Assert(!(pCtx->es.Attr.u & 0xf00));
4232 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4233 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4234 || !(pCtx->es.Attr.n.u1Granularity));
4235 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4236 || (pCtx->es.Attr.n.u1Granularity));
4237 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4238 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4239 }
4240 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4241 {
4242 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4243 Assert(pCtx->fs.Attr.n.u1Present);
4244 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4245 Assert(!(pCtx->fs.Attr.u & 0xf00));
4246 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4247 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4248 || !(pCtx->fs.Attr.n.u1Granularity));
4249 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4250 || (pCtx->fs.Attr.n.u1Granularity));
4251 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4252 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4253 }
4254 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4255 {
4256 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4257 Assert(pCtx->gs.Attr.n.u1Present);
4258 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4259 Assert(!(pCtx->gs.Attr.u & 0xf00));
4260 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4261 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4262 || !(pCtx->gs.Attr.n.u1Granularity));
4263 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4264 || (pCtx->gs.Attr.n.u1Granularity));
4265 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4266 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4267 }
4268 /* 64-bit capable CPUs. */
4269# if HC_ARCH_BITS == 64
4270 Assert(!(pCtx->cs.u64Base >> 32));
4271 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4272 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4273 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4274# endif
4275 }
4276 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4277 || ( CPUMIsGuestInRealModeEx(pCtx)
4278 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4279 {
4280 /* Real and v86 mode checks. */
4281 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4282 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4283 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4284 {
4285 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4286 }
4287 else
4288 {
4289 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4290 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4291 }
4292
4293 /* CS */
4294 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4295 Assert(pCtx->cs.u32Limit == 0xffff);
4296 Assert(u32CSAttr == 0xf3);
4297 /* SS */
4298 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4299 Assert(pCtx->ss.u32Limit == 0xffff);
4300 Assert(u32SSAttr == 0xf3);
4301 /* DS */
4302 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4303 Assert(pCtx->ds.u32Limit == 0xffff);
4304 Assert(u32DSAttr == 0xf3);
4305 /* ES */
4306 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4307 Assert(pCtx->es.u32Limit == 0xffff);
4308 Assert(u32ESAttr == 0xf3);
4309 /* FS */
4310 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4311 Assert(pCtx->fs.u32Limit == 0xffff);
4312 Assert(u32FSAttr == 0xf3);
4313 /* GS */
4314 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4315 Assert(pCtx->gs.u32Limit == 0xffff);
4316 Assert(u32GSAttr == 0xf3);
4317 /* 64-bit capable CPUs. */
4318# if HC_ARCH_BITS == 64
4319 Assert(!(pCtx->cs.u64Base >> 32));
4320 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4321 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4322 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4323# endif
4324 }
4325}
4326#endif /* VBOX_STRICT */
4327
4328
4329/**
4330 * Writes a guest segment register into the guest-state area in the VMCS.
4331 *
4332 * @returns VBox status code.
4333 * @param pVCpu The cross context virtual CPU structure.
4334 * @param idxSel Index of the selector in the VMCS.
4335 * @param idxLimit Index of the segment limit in the VMCS.
4336 * @param idxBase Index of the segment base in the VMCS.
4337 * @param idxAccess Index of the access rights of the segment in the VMCS.
4338 * @param pSelReg Pointer to the segment selector.
4339 *
4340 * @remarks No-long-jump zone!!!
4341 */
4342static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4343 uint32_t idxAccess, PCPUMSELREG pSelReg)
4344{
4345 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4346 AssertRCReturn(rc, rc);
4347 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4348 AssertRCReturn(rc, rc);
4349 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4350 AssertRCReturn(rc, rc);
4351
4352 uint32_t u32Access = pSelReg->Attr.u;
4353 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4354 {
4355 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4356 u32Access = 0xf3;
4357 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4358 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4359 }
4360 else
4361 {
4362 /*
4363 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4364 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4365 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4366 * loaded in protected-mode have their attribute as 0.
4367 */
4368 if (!u32Access)
4369 u32Access = X86DESCATTR_UNUSABLE;
4370 }
4371
4372 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4373 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4374 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4375
4376 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4377 AssertRCReturn(rc, rc);
4378 return rc;
4379}
4380
4381
4382/**
4383 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4384 * into the guest-state area in the VMCS.
4385 *
4386 * @returns VBox status code.
4387 * @param pVCpu The cross context virtual CPU structure.
4388 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4389 * out-of-sync. Make sure to update the required fields
4390 * before using them.
4391 *
4392 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4393 * @remarks No-long-jump zone!!!
4394 */
4395static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4396{
4397 int rc = VERR_INTERNAL_ERROR_5;
4398 PVM pVM = pVCpu->CTX_SUFF(pVM);
4399
4400 /*
4401 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4402 */
4403 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4404 {
4405 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4406 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4407 {
4408 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4409 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4410 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4411 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4412 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4413 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4414 }
4415
4416#ifdef VBOX_WITH_REM
4417 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4418 {
4419 Assert(pVM->hm.s.vmx.pRealModeTSS);
4420 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4421 if ( pVCpu->hm.s.vmx.fWasInRealMode
4422 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4423 {
4424 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4425 in real-mode (e.g. OpenBSD 4.0) */
4426 REMFlushTBs(pVM);
4427 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4428 pVCpu->hm.s.vmx.fWasInRealMode = false;
4429 }
4430 }
4431#endif
4432 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4433 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4434 AssertRCReturn(rc, rc);
4435 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4436 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4437 AssertRCReturn(rc, rc);
4438 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4439 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4440 AssertRCReturn(rc, rc);
4441 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4442 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4443 AssertRCReturn(rc, rc);
4444 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4445 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4446 AssertRCReturn(rc, rc);
4447 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4448 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4449 AssertRCReturn(rc, rc);
4450
4451#ifdef VBOX_STRICT
4452 /* Validate. */
4453 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4454#endif
4455
4456 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4457 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4458 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4459 }
4460
4461 /*
4462 * Guest TR.
4463 */
4464 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4465 {
4466 /*
4467 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4468 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4469 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4470 */
4471 uint16_t u16Sel = 0;
4472 uint32_t u32Limit = 0;
4473 uint64_t u64Base = 0;
4474 uint32_t u32AccessRights = 0;
4475
4476 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4477 {
4478 u16Sel = pMixedCtx->tr.Sel;
4479 u32Limit = pMixedCtx->tr.u32Limit;
4480 u64Base = pMixedCtx->tr.u64Base;
4481 u32AccessRights = pMixedCtx->tr.Attr.u;
4482 }
4483 else
4484 {
4485 Assert(pVM->hm.s.vmx.pRealModeTSS);
4486 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4487
4488 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4489 RTGCPHYS GCPhys;
4490 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4491 AssertRCReturn(rc, rc);
4492
4493 X86DESCATTR DescAttr;
4494 DescAttr.u = 0;
4495 DescAttr.n.u1Present = 1;
4496 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4497
4498 u16Sel = 0;
4499 u32Limit = HM_VTX_TSS_SIZE;
4500 u64Base = GCPhys; /* in real-mode phys = virt. */
4501 u32AccessRights = DescAttr.u;
4502 }
4503
4504 /* Validate. */
4505 Assert(!(u16Sel & RT_BIT(2)));
4506 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4507 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4508 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4509 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4510 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4511 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4512 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4513 Assert( (u32Limit & 0xfff) == 0xfff
4514 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4515 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4516 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4517
4518 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4519 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4520 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4521 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4522
4523 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4524 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4525 }
4526
4527 /*
4528 * Guest GDTR.
4529 */
4530 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4531 {
4532 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4533 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4534
4535 /* Validate. */
4536 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4537
4538 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4539 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4540 }
4541
4542 /*
4543 * Guest LDTR.
4544 */
4545 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4546 {
4547 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4548 uint32_t u32Access = 0;
4549 if (!pMixedCtx->ldtr.Attr.u)
4550 u32Access = X86DESCATTR_UNUSABLE;
4551 else
4552 u32Access = pMixedCtx->ldtr.Attr.u;
4553
4554 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4555 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4556 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4557 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4558
4559 /* Validate. */
4560 if (!(u32Access & X86DESCATTR_UNUSABLE))
4561 {
4562 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4563 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4564 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4565 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4566 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4567 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4568 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4569 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4570 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4571 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4572 }
4573
4574 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4575 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4576 }
4577
4578 /*
4579 * Guest IDTR.
4580 */
4581 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4582 {
4583 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4584 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4585
4586 /* Validate. */
4587 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4588
4589 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4590 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4591 }
4592
4593 return VINF_SUCCESS;
4594}
4595
4596
4597/**
4598 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4599 * areas.
4600 *
4601 * These MSRs will automatically be loaded to the host CPU on every successful
4602 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4603 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4604 * -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4605 *
4606 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4607 *
4608 * @returns VBox status code.
4609 * @param pVCpu The cross context virtual CPU structure.
4610 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4611 * out-of-sync. Make sure to update the required fields
4612 * before using them.
4613 *
4614 * @remarks No-long-jump zone!!!
4615 */
4616static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4617{
4618 AssertPtr(pVCpu);
4619 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4620
4621 /*
4622 * MSRs that we use the auto-load/store MSR area in the VMCS.
4623 */
4624 PVM pVM = pVCpu->CTX_SUFF(pVM);
4625 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4626 {
4627 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4628#if HC_ARCH_BITS == 32
4629 if (pVM->hm.s.fAllow64BitGuests)
4630 {
4631 int rc = VINF_SUCCESS;
4632 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4633 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4634 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4635 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4636 AssertRCReturn(rc, rc);
4637# ifdef LOG_ENABLED
4638 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4639 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4640 {
4641 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4642 pMsr->u64Value));
4643 }
4644# endif
4645 }
4646#endif
4647 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4648 }
4649
4650 /*
4651 * Guest Sysenter MSRs.
4652 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4653 * VM-exits on WRMSRs for these MSRs.
4654 */
4655 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4656 {
4657 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4658 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4659 }
4660
4661 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4662 {
4663 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4664 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4665 }
4666
4667 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4668 {
4669 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4670 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4671 }
4672
4673 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4674 {
4675 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4676 {
4677 /*
4678 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4679 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4680 */
4681 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4682 {
4683 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4684 AssertRCReturn(rc,rc);
4685 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4686 }
4687 else
4688 {
4689 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4690 NULL /* pfAddedAndUpdated */);
4691 AssertRCReturn(rc, rc);
4692
4693 /* We need to intercept reads too, see @bugref{7386#c16}. */
4694 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4695 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4696 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4697 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4698 }
4699 }
4700 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4701 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4702 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4703 }
4704
4705 return VINF_SUCCESS;
4706}
4707
4708
4709/**
4710 * Loads the guest activity state into the guest-state area in the VMCS.
4711 *
4712 * @returns VBox status code.
4713 * @param pVCpu The cross context virtual CPU structure.
4714 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4715 * out-of-sync. Make sure to update the required fields
4716 * before using them.
4717 *
4718 * @remarks No-long-jump zone!!!
4719 */
4720static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4721{
4722 NOREF(pMixedCtx);
4723 /** @todo See if we can make use of other states, e.g.
4724 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4725 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4726 {
4727 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4728 AssertRCReturn(rc, rc);
4729
4730 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4731 }
4732 return VINF_SUCCESS;
4733}
4734
4735
4736/**
4737 * Sets up the appropriate function to run guest code.
4738 *
4739 * @returns VBox status code.
4740 * @param pVCpu The cross context virtual CPU structure.
4741 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4742 * out-of-sync. Make sure to update the required fields
4743 * before using them.
4744 *
4745 * @remarks No-long-jump zone!!!
4746 */
4747static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4748{
4749 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4750 {
4751#ifndef VBOX_ENABLE_64_BITS_GUESTS
4752 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4753#endif
4754 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4755#if HC_ARCH_BITS == 32
4756 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4757 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4758 {
4759 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4760 {
4761 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4762 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4763 | HM_CHANGED_VMX_ENTRY_CTLS
4764 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4765 }
4766 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4767 }
4768#else
4769 /* 64-bit host. */
4770 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4771#endif
4772 }
4773 else
4774 {
4775 /* Guest is not in long mode, use the 32-bit handler. */
4776#if HC_ARCH_BITS == 32
4777 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4778 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4779 {
4780 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4781 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4782 | HM_CHANGED_VMX_ENTRY_CTLS
4783 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4784 }
4785#endif
4786 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4787 }
4788 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4789 return VINF_SUCCESS;
4790}
4791
4792
4793/**
4794 * Wrapper for running the guest code in VT-x.
4795 *
4796 * @returns VBox status code, no informational status codes.
4797 * @param pVM The cross context VM structure.
4798 * @param pVCpu The cross context virtual CPU structure.
4799 * @param pCtx Pointer to the guest-CPU context.
4800 *
4801 * @remarks No-long-jump zone!!!
4802 */
4803DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4804{
4805 /*
4806 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4807 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4808 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4809 */
4810 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4811 /** @todo Add stats for resume vs launch. */
4812#ifdef VBOX_WITH_KERNEL_USING_XMM
4813 int rc = HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4814#else
4815 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4816#endif
4817 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4818 return rc;
4819}
4820
4821
4822/**
4823 * Reports world-switch error and dumps some useful debug info.
4824 *
4825 * @param pVM The cross context VM structure.
4826 * @param pVCpu The cross context virtual CPU structure.
4827 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4828 * @param pCtx Pointer to the guest-CPU context.
4829 * @param pVmxTransient Pointer to the VMX transient structure (only
4830 * exitReason updated).
4831 */
4832static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4833{
4834 Assert(pVM);
4835 Assert(pVCpu);
4836 Assert(pCtx);
4837 Assert(pVmxTransient);
4838 HMVMX_ASSERT_PREEMPT_SAFE();
4839
4840 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4841 switch (rcVMRun)
4842 {
4843 case VERR_VMX_INVALID_VMXON_PTR:
4844 AssertFailed();
4845 break;
4846 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4847 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4848 {
4849 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4850 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4851 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4852 AssertRC(rc);
4853
4854 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4855 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4856 Cannot do it here as we may have been long preempted. */
4857
4858#ifdef VBOX_STRICT
4859 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4860 pVmxTransient->uExitReason));
4861 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4862 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4863 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4864 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4865 else
4866 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4867 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4868 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4869
4870 /* VMX control bits. */
4871 uint32_t u32Val;
4872 uint64_t u64Val;
4873 RTHCUINTREG uHCReg;
4874 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4875 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4876 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4877 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4878 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4879 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4880 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4881 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4882 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4883 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4884 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4885 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4886 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4887 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4888 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4889 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4890 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4891 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4892 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4893 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4894 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4895 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4896 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4897 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4898 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4899 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4900 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4901 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4902 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4903 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4904 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4905 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4906 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4907 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4908 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4909 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4910 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4911 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4912 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4913 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4914 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4915 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4916
4917 /* Guest bits. */
4918 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4919 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4920 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4921 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4922 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4923 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4924 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4925 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4926
4927 /* Host bits. */
4928 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4929 Log4(("Host CR0 %#RHr\n", uHCReg));
4930 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4931 Log4(("Host CR3 %#RHr\n", uHCReg));
4932 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4933 Log4(("Host CR4 %#RHr\n", uHCReg));
4934
4935 RTGDTR HostGdtr;
4936 PCX86DESCHC pDesc;
4937 ASMGetGDTR(&HostGdtr);
4938 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4939 Log4(("Host CS %#08x\n", u32Val));
4940 if (u32Val < HostGdtr.cbGdt)
4941 {
4942 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4943 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4944 }
4945
4946 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4947 Log4(("Host DS %#08x\n", u32Val));
4948 if (u32Val < HostGdtr.cbGdt)
4949 {
4950 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4951 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4952 }
4953
4954 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4955 Log4(("Host ES %#08x\n", u32Val));
4956 if (u32Val < HostGdtr.cbGdt)
4957 {
4958 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4959 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4960 }
4961
4962 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4963 Log4(("Host FS %#08x\n", u32Val));
4964 if (u32Val < HostGdtr.cbGdt)
4965 {
4966 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4967 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4968 }
4969
4970 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4971 Log4(("Host GS %#08x\n", u32Val));
4972 if (u32Val < HostGdtr.cbGdt)
4973 {
4974 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4975 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4976 }
4977
4978 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4979 Log4(("Host SS %#08x\n", u32Val));
4980 if (u32Val < HostGdtr.cbGdt)
4981 {
4982 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4983 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4984 }
4985
4986 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4987 Log4(("Host TR %#08x\n", u32Val));
4988 if (u32Val < HostGdtr.cbGdt)
4989 {
4990 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4991 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4992 }
4993
4994 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4995 Log4(("Host TR Base %#RHv\n", uHCReg));
4996 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4997 Log4(("Host GDTR Base %#RHv\n", uHCReg));
4998 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4999 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5000 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5001 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5002 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5003 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5004 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5005 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5006 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5007 Log4(("Host RSP %#RHv\n", uHCReg));
5008 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5009 Log4(("Host RIP %#RHv\n", uHCReg));
5010# if HC_ARCH_BITS == 64
5011 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5012 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5013 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5014 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5015 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5016 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5017# endif
5018#endif /* VBOX_STRICT */
5019 break;
5020 }
5021
5022 default:
5023 /* Impossible */
5024 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5025 break;
5026 }
5027 NOREF(pVM); NOREF(pCtx);
5028}
5029
5030
5031#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5032#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5033# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5034#endif
5035#ifdef VBOX_STRICT
5036static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5037{
5038 switch (idxField)
5039 {
5040 case VMX_VMCS_GUEST_RIP:
5041 case VMX_VMCS_GUEST_RSP:
5042 case VMX_VMCS_GUEST_SYSENTER_EIP:
5043 case VMX_VMCS_GUEST_SYSENTER_ESP:
5044 case VMX_VMCS_GUEST_GDTR_BASE:
5045 case VMX_VMCS_GUEST_IDTR_BASE:
5046 case VMX_VMCS_GUEST_CS_BASE:
5047 case VMX_VMCS_GUEST_DS_BASE:
5048 case VMX_VMCS_GUEST_ES_BASE:
5049 case VMX_VMCS_GUEST_FS_BASE:
5050 case VMX_VMCS_GUEST_GS_BASE:
5051 case VMX_VMCS_GUEST_SS_BASE:
5052 case VMX_VMCS_GUEST_LDTR_BASE:
5053 case VMX_VMCS_GUEST_TR_BASE:
5054 case VMX_VMCS_GUEST_CR3:
5055 return true;
5056 }
5057 return false;
5058}
5059
5060static bool hmR0VmxIsValidReadField(uint32_t idxField)
5061{
5062 switch (idxField)
5063 {
5064 /* Read-only fields. */
5065 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5066 return true;
5067 }
5068 /* Remaining readable fields should also be writable. */
5069 return hmR0VmxIsValidWriteField(idxField);
5070}
5071#endif /* VBOX_STRICT */
5072
5073
5074/**
5075 * Executes the specified handler in 64-bit mode.
5076 *
5077 * @returns VBox status code (no informational status codes).
5078 * @param pVM The cross context VM structure.
5079 * @param pVCpu The cross context virtual CPU structure.
5080 * @param pCtx Pointer to the guest CPU context.
5081 * @param enmOp The operation to perform.
5082 * @param cParams Number of parameters.
5083 * @param paParam Array of 32-bit parameters.
5084 */
5085VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5086 uint32_t cParams, uint32_t *paParam)
5087{
5088 NOREF(pCtx);
5089
5090 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5091 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5092 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5093 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5094
5095#ifdef VBOX_STRICT
5096 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5097 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5098
5099 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5100 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5101#endif
5102
5103 /* Disable interrupts. */
5104 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5105
5106#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5107 RTCPUID idHostCpu = RTMpCpuId();
5108 CPUMR0SetLApic(pVCpu, idHostCpu);
5109#endif
5110
5111 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5112 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5113
5114 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5115 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5116
5117 /* Leave VMX Root Mode. */
5118 VMXDisable();
5119
5120 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5121
5122 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5123 CPUMSetHyperEIP(pVCpu, enmOp);
5124 for (int i = (int)cParams - 1; i >= 0; i--)
5125 CPUMPushHyper(pVCpu, paParam[i]);
5126
5127 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5128
5129 /* Call the switcher. */
5130 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5131 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5132
5133 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5134 /* Make sure the VMX instructions don't cause #UD faults. */
5135 SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
5136
5137 /* Re-enter VMX Root Mode */
5138 int rc2 = VMXEnable(HCPhysCpuPage);
5139 if (RT_FAILURE(rc2))
5140 {
5141 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5142 ASMSetFlags(fOldEFlags);
5143 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5144 return rc2;
5145 }
5146
5147 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5148 AssertRC(rc2);
5149 Assert(!(ASMGetFlags() & X86_EFL_IF));
5150 ASMSetFlags(fOldEFlags);
5151 return rc;
5152}
5153
5154
5155/**
5156 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5157 * supporting 64-bit guests.
5158 *
5159 * @returns VBox status code.
5160 * @param fResume Whether to VMLAUNCH or VMRESUME.
5161 * @param pCtx Pointer to the guest-CPU context.
5162 * @param pCache Pointer to the VMCS cache.
5163 * @param pVM The cross context VM structure.
5164 * @param pVCpu The cross context virtual CPU structure.
5165 */
5166DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5167{
5168 NOREF(fResume);
5169
5170 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5171 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5172
5173#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5174 pCache->uPos = 1;
5175 pCache->interPD = PGMGetInterPaeCR3(pVM);
5176 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5177#endif
5178
5179#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5180 pCache->TestIn.HCPhysCpuPage = 0;
5181 pCache->TestIn.HCPhysVmcs = 0;
5182 pCache->TestIn.pCache = 0;
5183 pCache->TestOut.HCPhysVmcs = 0;
5184 pCache->TestOut.pCache = 0;
5185 pCache->TestOut.pCtx = 0;
5186 pCache->TestOut.eflags = 0;
5187#else
5188 NOREF(pCache);
5189#endif
5190
5191 uint32_t aParam[10];
5192 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5193 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5194 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5195 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5196 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5197 aParam[5] = 0;
5198 aParam[6] = VM_RC_ADDR(pVM, pVM);
5199 aParam[7] = 0;
5200 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5201 aParam[9] = 0;
5202
5203#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5204 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5205 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5206#endif
5207 int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5208
5209#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5210 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5211 Assert(pCtx->dr[4] == 10);
5212 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5213#endif
5214
5215#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5216 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5217 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5218 pVCpu->hm.s.vmx.HCPhysVmcs));
5219 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5220 pCache->TestOut.HCPhysVmcs));
5221 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5222 pCache->TestOut.pCache));
5223 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5224 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5225 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5226 pCache->TestOut.pCtx));
5227 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5228#endif
5229 return rc;
5230}
5231
5232
5233/**
5234 * Initialize the VMCS-Read cache.
5235 *
5236 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5237 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5238 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5239 * (those that have a 32-bit FULL & HIGH part).
5240 *
5241 * @returns VBox status code.
5242 * @param pVM The cross context VM structure.
5243 * @param pVCpu The cross context virtual CPU structure.
5244 */
5245static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5246{
5247#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5248{ \
5249 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5250 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5251 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5252 ++cReadFields; \
5253}
5254
5255 AssertPtr(pVM);
5256 AssertPtr(pVCpu);
5257 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5258 uint32_t cReadFields = 0;
5259
5260 /*
5261 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5262 * and serve to indicate exceptions to the rules.
5263 */
5264
5265 /* Guest-natural selector base fields. */
5266#if 0
5267 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5268 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5269 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5270#endif
5271 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5272 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5273 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5274 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5275 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5276 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5277 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5278 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5279 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5280 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5281 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5282 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5283#if 0
5284 /* Unused natural width guest-state fields. */
5285 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5286 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5287#endif
5288 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5289 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5290
5291 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5292#if 0
5293 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5294 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5295 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5296 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5297 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5298 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5299 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5300 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5301 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5302#endif
5303
5304 /* Natural width guest-state fields. */
5305 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5306#if 0
5307 /* Currently unused field. */
5308 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5309#endif
5310
5311 if (pVM->hm.s.fNestedPaging)
5312 {
5313 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5314 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5315 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5316 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5317 }
5318 else
5319 {
5320 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5321 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5322 }
5323
5324#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5325 return VINF_SUCCESS;
5326}
5327
5328
5329/**
5330 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5331 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5332 * darwin, running 64-bit guests).
5333 *
5334 * @returns VBox status code.
5335 * @param pVCpu The cross context virtual CPU structure.
5336 * @param idxField The VMCS field encoding.
5337 * @param u64Val 16, 32 or 64-bit value.
5338 */
5339VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5340{
5341 int rc;
5342 switch (idxField)
5343 {
5344 /*
5345 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5346 */
5347 /* 64-bit Control fields. */
5348 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5349 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5350 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5351 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5352 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5353 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5354 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5355 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5356 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5357 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5358 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5359 case VMX_VMCS64_CTRL_EPTP_FULL:
5360 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5361 /* 64-bit Guest-state fields. */
5362 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5363 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5364 case VMX_VMCS64_GUEST_PAT_FULL:
5365 case VMX_VMCS64_GUEST_EFER_FULL:
5366 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5367 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5368 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5369 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5370 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5371 /* 64-bit Host-state fields. */
5372 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5373 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5374 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5375 {
5376 rc = VMXWriteVmcs32(idxField, u64Val);
5377 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5378 break;
5379 }
5380
5381 /*
5382 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5383 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5384 */
5385 /* Natural-width Guest-state fields. */
5386 case VMX_VMCS_GUEST_CR3:
5387 case VMX_VMCS_GUEST_ES_BASE:
5388 case VMX_VMCS_GUEST_CS_BASE:
5389 case VMX_VMCS_GUEST_SS_BASE:
5390 case VMX_VMCS_GUEST_DS_BASE:
5391 case VMX_VMCS_GUEST_FS_BASE:
5392 case VMX_VMCS_GUEST_GS_BASE:
5393 case VMX_VMCS_GUEST_LDTR_BASE:
5394 case VMX_VMCS_GUEST_TR_BASE:
5395 case VMX_VMCS_GUEST_GDTR_BASE:
5396 case VMX_VMCS_GUEST_IDTR_BASE:
5397 case VMX_VMCS_GUEST_RSP:
5398 case VMX_VMCS_GUEST_RIP:
5399 case VMX_VMCS_GUEST_SYSENTER_ESP:
5400 case VMX_VMCS_GUEST_SYSENTER_EIP:
5401 {
5402 if (!(u64Val >> 32))
5403 {
5404 /* If this field is 64-bit, VT-x will zero out the top bits. */
5405 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5406 }
5407 else
5408 {
5409 /* Assert that only the 32->64 switcher case should ever come here. */
5410 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5411 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5412 }
5413 break;
5414 }
5415
5416 default:
5417 {
5418 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5419 rc = VERR_INVALID_PARAMETER;
5420 break;
5421 }
5422 }
5423 AssertRCReturn(rc, rc);
5424 return rc;
5425}
5426
5427
5428/**
5429 * Queue up a VMWRITE by using the VMCS write cache.
5430 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5431 *
5432 * @param pVCpu The cross context virtual CPU structure.
5433 * @param idxField The VMCS field encoding.
5434 * @param u64Val 16, 32 or 64-bit value.
5435 */
5436VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5437{
5438 AssertPtr(pVCpu);
5439 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5440
5441 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5442 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5443
5444 /* Make sure there are no duplicates. */
5445 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5446 {
5447 if (pCache->Write.aField[i] == idxField)
5448 {
5449 pCache->Write.aFieldVal[i] = u64Val;
5450 return VINF_SUCCESS;
5451 }
5452 }
5453
5454 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5455 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5456 pCache->Write.cValidEntries++;
5457 return VINF_SUCCESS;
5458}
5459#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5460
5461
5462/**
5463 * Sets up the usage of TSC-offsetting and updates the VMCS.
5464 *
5465 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5466 * VMX preemption timer.
5467 *
5468 * @returns VBox status code.
5469 * @param pVM The cross context VM structure.
5470 * @param pVCpu The cross context virtual CPU structure.
5471 *
5472 * @remarks No-long-jump zone!!!
5473 */
5474static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5475{
5476 int rc;
5477 bool fOffsettedTsc;
5478 bool fParavirtTsc;
5479 if (pVM->hm.s.vmx.fUsePreemptTimer)
5480 {
5481 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5482 &fOffsettedTsc, &fParavirtTsc);
5483
5484 /* Make sure the returned values have sane upper and lower boundaries. */
5485 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5486 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5487 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5488 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5489
5490 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5491 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5492 }
5493 else
5494 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5495
5496 /** @todo later optimize this to be done elsewhere and not before every
5497 * VM-entry. */
5498 if (fParavirtTsc)
5499 {
5500 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5501 information before every VM-entry, hence disable it for performance sake. */
5502#if 0
5503 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5504 AssertRC(rc);
5505#endif
5506 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5507 }
5508
5509 if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5510 {
5511 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5512 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5513
5514 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5515 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5516 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5517 }
5518 else
5519 {
5520 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5521 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5522 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5523 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5524 }
5525}
5526
5527
5528/**
5529 * Determines if an exception is a contributory exception.
5530 *
5531 * Contributory exceptions are ones which can cause double-faults unless the
5532 * original exception was a benign exception. Page-fault is intentionally not
5533 * included here as it's a conditional contributory exception.
5534 *
5535 * @returns true if the exception is contributory, false otherwise.
5536 * @param uVector The exception vector.
5537 */
5538DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5539{
5540 switch (uVector)
5541 {
5542 case X86_XCPT_GP:
5543 case X86_XCPT_SS:
5544 case X86_XCPT_NP:
5545 case X86_XCPT_TS:
5546 case X86_XCPT_DE:
5547 return true;
5548 default:
5549 break;
5550 }
5551 return false;
5552}
5553
5554
5555/**
5556 * Sets an event as a pending event to be injected into the guest.
5557 *
5558 * @param pVCpu The cross context virtual CPU structure.
5559 * @param u32IntInfo The VM-entry interruption-information field.
5560 * @param cbInstr The VM-entry instruction length in bytes (for software
5561 * interrupts, exceptions and privileged software
5562 * exceptions).
5563 * @param u32ErrCode The VM-entry exception error code.
5564 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5565 * page-fault.
5566 *
5567 * @remarks Statistics counter assumes this is a guest event being injected or
5568 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5569 * always incremented.
5570 */
5571DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5572 RTGCUINTPTR GCPtrFaultAddress)
5573{
5574 Assert(!pVCpu->hm.s.Event.fPending);
5575 pVCpu->hm.s.Event.fPending = true;
5576 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5577 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5578 pVCpu->hm.s.Event.cbInstr = cbInstr;
5579 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5580
5581 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5582}
5583
5584
5585/**
5586 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5587 *
5588 * @param pVCpu The cross context virtual CPU structure.
5589 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5590 * out-of-sync. Make sure to update the required fields
5591 * before using them.
5592 */
5593DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5594{
5595 NOREF(pMixedCtx);
5596 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5597 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5598 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5599 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5600}
5601
5602
5603/**
5604 * Handle a condition that occurred while delivering an event through the guest
5605 * IDT.
5606 *
5607 * @returns Strict VBox status code (i.e. informational status codes too).
5608 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5609 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5610 * to continue execution of the guest which will delivery the \#DF.
5611 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5612 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5613 *
5614 * @param pVCpu The cross context virtual CPU structure.
5615 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5616 * out-of-sync. Make sure to update the required fields
5617 * before using them.
5618 * @param pVmxTransient Pointer to the VMX transient structure.
5619 *
5620 * @remarks No-long-jump zone!!!
5621 */
5622static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5623{
5624 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5625
5626 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5627 AssertRCReturn(rc2, rc2);
5628 rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5629 AssertRCReturn(rc2, rc2);
5630
5631 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5632 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5633 {
5634 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5635 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5636
5637 typedef enum
5638 {
5639 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5640 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5641 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5642 VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
5643 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5644 } VMXREFLECTXCPT;
5645
5646 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5647 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5648 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5649 {
5650 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5651 {
5652 enmReflect = VMXREFLECTXCPT_XCPT;
5653#ifdef VBOX_STRICT
5654 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5655 && uExitVector == X86_XCPT_PF)
5656 {
5657 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5658 }
5659#endif
5660 if ( uExitVector == X86_XCPT_PF
5661 && uIdtVector == X86_XCPT_PF)
5662 {
5663 pVmxTransient->fVectoringDoublePF = true;
5664 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5665 }
5666 else if ( uExitVector == X86_XCPT_AC
5667 && uIdtVector == X86_XCPT_AC)
5668 {
5669 enmReflect = VMXREFLECTXCPT_HANG;
5670 Log4(("IDT: Nested #AC - Bad guest\n"));
5671 }
5672 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5673 && hmR0VmxIsContributoryXcpt(uExitVector)
5674 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5675 || uIdtVector == X86_XCPT_PF))
5676 {
5677 enmReflect = VMXREFLECTXCPT_DF;
5678 }
5679 else if (uIdtVector == X86_XCPT_DF)
5680 enmReflect = VMXREFLECTXCPT_TF;
5681 }
5682 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5683 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5684 {
5685 /*
5686 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5687 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5688 */
5689 enmReflect = VMXREFLECTXCPT_XCPT;
5690
5691 if (uExitVector == X86_XCPT_PF)
5692 {
5693 pVmxTransient->fVectoringPF = true;
5694 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5695 }
5696 }
5697 }
5698 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5699 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5700 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5701 {
5702 /*
5703 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5704 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5705 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5706 */
5707 enmReflect = VMXREFLECTXCPT_XCPT;
5708 }
5709
5710 /*
5711 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5712 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5713 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5714 *
5715 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5716 */
5717 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5718 && enmReflect == VMXREFLECTXCPT_XCPT
5719 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5720 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5721 {
5722 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5723 }
5724
5725 switch (enmReflect)
5726 {
5727 case VMXREFLECTXCPT_XCPT:
5728 {
5729 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5730 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5731 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5732
5733 uint32_t u32ErrCode = 0;
5734 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5735 {
5736 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5737 AssertRCReturn(rc2, rc2);
5738 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5739 }
5740
5741 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5742 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5743 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5744 rcStrict = VINF_SUCCESS;
5745 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5746 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5747
5748 break;
5749 }
5750
5751 case VMXREFLECTXCPT_DF:
5752 {
5753 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5754 rcStrict = VINF_HM_DOUBLE_FAULT;
5755 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5756 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5757
5758 break;
5759 }
5760
5761 case VMXREFLECTXCPT_TF:
5762 {
5763 rcStrict = VINF_EM_RESET;
5764 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5765 uExitVector));
5766 break;
5767 }
5768
5769 case VMXREFLECTXCPT_HANG:
5770 {
5771 rcStrict = VERR_EM_GUEST_CPU_HANG;
5772 break;
5773 }
5774
5775 default:
5776 Assert(rcStrict == VINF_SUCCESS);
5777 break;
5778 }
5779 }
5780 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5781 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5782 && uExitVector != X86_XCPT_DF
5783 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5784 {
5785 /*
5786 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5787 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5788 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5789 */
5790 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5791 {
5792 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
5793 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5794 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5795 }
5796 }
5797
5798 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
5799 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
5800 return rcStrict;
5801}
5802
5803
5804/**
5805 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5806 *
5807 * @returns VBox status code.
5808 * @param pVCpu The cross context virtual CPU structure.
5809 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5810 * out-of-sync. Make sure to update the required fields
5811 * before using them.
5812 *
5813 * @remarks No-long-jump zone!!!
5814 */
5815static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5816{
5817 NOREF(pMixedCtx);
5818
5819 /*
5820 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5821 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5822 */
5823 VMMRZCallRing3Disable(pVCpu);
5824 HM_DISABLE_PREEMPT();
5825
5826 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5827 {
5828 uint32_t uVal = 0;
5829 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5830 AssertRCReturn(rc, rc);
5831
5832 uint32_t uShadow = 0;
5833 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5834 AssertRCReturn(rc, rc);
5835
5836 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5837 CPUMSetGuestCR0(pVCpu, uVal);
5838 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5839 }
5840
5841 HM_RESTORE_PREEMPT();
5842 VMMRZCallRing3Enable(pVCpu);
5843 return VINF_SUCCESS;
5844}
5845
5846
5847/**
5848 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5849 *
5850 * @returns VBox status code.
5851 * @param pVCpu The cross context virtual CPU structure.
5852 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5853 * out-of-sync. Make sure to update the required fields
5854 * before using them.
5855 *
5856 * @remarks No-long-jump zone!!!
5857 */
5858static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5859{
5860 NOREF(pMixedCtx);
5861
5862 int rc = VINF_SUCCESS;
5863 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5864 {
5865 uint32_t uVal = 0;
5866 uint32_t uShadow = 0;
5867 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5868 AssertRCReturn(rc, rc);
5869 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5870 AssertRCReturn(rc, rc);
5871
5872 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5873 CPUMSetGuestCR4(pVCpu, uVal);
5874 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5875 }
5876 return rc;
5877}
5878
5879
5880/**
5881 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5882 *
5883 * @returns VBox status code.
5884 * @param pVCpu The cross context virtual CPU structure.
5885 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5886 * out-of-sync. Make sure to update the required fields
5887 * before using them.
5888 *
5889 * @remarks No-long-jump zone!!!
5890 */
5891static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5892{
5893 int rc = VINF_SUCCESS;
5894 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
5895 {
5896 uint64_t u64Val = 0;
5897 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5898 AssertRCReturn(rc, rc);
5899
5900 pMixedCtx->rip = u64Val;
5901 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
5902 }
5903 return rc;
5904}
5905
5906
5907/**
5908 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
5909 *
5910 * @returns VBox status code.
5911 * @param pVCpu The cross context virtual CPU structure.
5912 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5913 * out-of-sync. Make sure to update the required fields
5914 * before using them.
5915 *
5916 * @remarks No-long-jump zone!!!
5917 */
5918static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5919{
5920 int rc = VINF_SUCCESS;
5921 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
5922 {
5923 uint64_t u64Val = 0;
5924 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5925 AssertRCReturn(rc, rc);
5926
5927 pMixedCtx->rsp = u64Val;
5928 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
5929 }
5930 return rc;
5931}
5932
5933
5934/**
5935 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5936 *
5937 * @returns VBox status code.
5938 * @param pVCpu The cross context virtual CPU structure.
5939 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5940 * out-of-sync. Make sure to update the required fields
5941 * before using them.
5942 *
5943 * @remarks No-long-jump zone!!!
5944 */
5945static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5946{
5947 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
5948 {
5949 uint32_t uVal = 0;
5950 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5951 AssertRCReturn(rc, rc);
5952
5953 pMixedCtx->eflags.u32 = uVal;
5954 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5955 {
5956 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5957 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5958
5959 pMixedCtx->eflags.Bits.u1VM = 0;
5960 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5961 }
5962
5963 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
5964 }
5965 return VINF_SUCCESS;
5966}
5967
5968
5969/**
5970 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5971 * guest-CPU context.
5972 */
5973DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5974{
5975 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5976 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5977 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5978 return rc;
5979}
5980
5981
5982/**
5983 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5984 * from the guest-state area in the VMCS.
5985 *
5986 * @param pVCpu The cross context virtual CPU structure.
5987 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5988 * out-of-sync. Make sure to update the required fields
5989 * before using them.
5990 *
5991 * @remarks No-long-jump zone!!!
5992 */
5993static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5994{
5995 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
5996 {
5997 uint32_t uIntrState = 0;
5998 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5999 AssertRC(rc);
6000
6001 if (!uIntrState)
6002 {
6003 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6004 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6005
6006 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6007 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6008 }
6009 else
6010 {
6011 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6012 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6013 {
6014 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6015 AssertRC(rc);
6016 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6017 AssertRC(rc);
6018
6019 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6020 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6021 }
6022 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6023 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6024
6025 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6026 {
6027 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6028 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6029 }
6030 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6031 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6032 }
6033
6034 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6035 }
6036}
6037
6038
6039/**
6040 * Saves the guest's activity state.
6041 *
6042 * @returns VBox status code.
6043 * @param pVCpu The cross context virtual CPU structure.
6044 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6045 * out-of-sync. Make sure to update the required fields
6046 * before using them.
6047 *
6048 * @remarks No-long-jump zone!!!
6049 */
6050static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6051{
6052 NOREF(pMixedCtx);
6053 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6054 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6055 return VINF_SUCCESS;
6056}
6057
6058
6059/**
6060 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6061 * the current VMCS into the guest-CPU context.
6062 *
6063 * @returns VBox status code.
6064 * @param pVCpu The cross context virtual CPU structure.
6065 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6066 * out-of-sync. Make sure to update the required fields
6067 * before using them.
6068 *
6069 * @remarks No-long-jump zone!!!
6070 */
6071static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6072{
6073 int rc = VINF_SUCCESS;
6074 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6075 {
6076 uint32_t u32Val = 0;
6077 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6078 pMixedCtx->SysEnter.cs = u32Val;
6079 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6080 }
6081
6082 uint64_t u64Val = 0;
6083 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6084 {
6085 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6086 pMixedCtx->SysEnter.eip = u64Val;
6087 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6088 }
6089 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6090 {
6091 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6092 pMixedCtx->SysEnter.esp = u64Val;
6093 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6094 }
6095 return rc;
6096}
6097
6098
6099/**
6100 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6101 * the CPU back into the guest-CPU context.
6102 *
6103 * @returns VBox status code.
6104 * @param pVCpu The cross context virtual CPU structure.
6105 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6106 * out-of-sync. Make sure to update the required fields
6107 * before using them.
6108 *
6109 * @remarks No-long-jump zone!!!
6110 */
6111static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6112{
6113#if HC_ARCH_BITS == 64
6114 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
6115 {
6116 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6117 VMMRZCallRing3Disable(pVCpu);
6118 HM_DISABLE_PREEMPT();
6119
6120 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6121 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6122 {
6123 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6124 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6125 }
6126
6127 HM_RESTORE_PREEMPT();
6128 VMMRZCallRing3Enable(pVCpu);
6129 }
6130 else
6131 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6132#else
6133 NOREF(pMixedCtx);
6134 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6135#endif
6136
6137 return VINF_SUCCESS;
6138}
6139
6140
6141/**
6142 * Saves the auto load/store'd guest MSRs from the current VMCS into
6143 * the guest-CPU context.
6144 *
6145 * @returns VBox status code.
6146 * @param pVCpu The cross context virtual CPU structure.
6147 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6148 * out-of-sync. Make sure to update the required fields
6149 * before using them.
6150 *
6151 * @remarks No-long-jump zone!!!
6152 */
6153static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6154{
6155 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6156 return VINF_SUCCESS;
6157
6158 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6159 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6160 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6161 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6162 {
6163 switch (pMsr->u32Msr)
6164 {
6165 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6166 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6167 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6168 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6169 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6170 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6171 break;
6172
6173 default:
6174 {
6175 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6176 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6177 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6178 }
6179 }
6180 }
6181
6182 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6183 return VINF_SUCCESS;
6184}
6185
6186
6187/**
6188 * Saves the guest control registers from the current VMCS into the guest-CPU
6189 * context.
6190 *
6191 * @returns VBox status code.
6192 * @param pVCpu The cross context virtual CPU structure.
6193 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6194 * out-of-sync. Make sure to update the required fields
6195 * before using them.
6196 *
6197 * @remarks No-long-jump zone!!!
6198 */
6199static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6200{
6201 /* Guest CR0. Guest FPU. */
6202 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6203 AssertRCReturn(rc, rc);
6204
6205 /* Guest CR4. */
6206 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6207 AssertRCReturn(rc, rc);
6208
6209 /* Guest CR2 - updated always during the world-switch or in #PF. */
6210 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6211 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6212 {
6213 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6214 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6215
6216 PVM pVM = pVCpu->CTX_SUFF(pVM);
6217 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6218 || ( pVM->hm.s.fNestedPaging
6219 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6220 {
6221 uint64_t u64Val = 0;
6222 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6223 if (pMixedCtx->cr3 != u64Val)
6224 {
6225 CPUMSetGuestCR3(pVCpu, u64Val);
6226 if (VMMRZCallRing3IsEnabled(pVCpu))
6227 {
6228 PGMUpdateCR3(pVCpu, u64Val);
6229 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6230 }
6231 else
6232 {
6233 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6234 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6235 }
6236 }
6237
6238 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6239 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6240 {
6241 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
6242 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
6243 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
6244 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
6245
6246 if (VMMRZCallRing3IsEnabled(pVCpu))
6247 {
6248 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6249 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6250 }
6251 else
6252 {
6253 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6254 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6255 }
6256 }
6257 }
6258
6259 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6260 }
6261
6262 /*
6263 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6264 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6265 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6266 *
6267 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6268 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6269 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6270 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6271 *
6272 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6273 */
6274 if (VMMRZCallRing3IsEnabled(pVCpu))
6275 {
6276 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6277 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6278
6279 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6280 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6281
6282 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6283 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6284 }
6285
6286 return rc;
6287}
6288
6289
6290/**
6291 * Reads a guest segment register from the current VMCS into the guest-CPU
6292 * context.
6293 *
6294 * @returns VBox status code.
6295 * @param pVCpu The cross context virtual CPU structure.
6296 * @param idxSel Index of the selector in the VMCS.
6297 * @param idxLimit Index of the segment limit in the VMCS.
6298 * @param idxBase Index of the segment base in the VMCS.
6299 * @param idxAccess Index of the access rights of the segment in the VMCS.
6300 * @param pSelReg Pointer to the segment selector.
6301 *
6302 * @remarks No-long-jump zone!!!
6303 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6304 * macro as that takes care of whether to read from the VMCS cache or
6305 * not.
6306 */
6307DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6308 PCPUMSELREG pSelReg)
6309{
6310 NOREF(pVCpu);
6311
6312 uint32_t u32Val = 0;
6313 int rc = VMXReadVmcs32(idxSel, &u32Val);
6314 AssertRCReturn(rc, rc);
6315 pSelReg->Sel = (uint16_t)u32Val;
6316 pSelReg->ValidSel = (uint16_t)u32Val;
6317 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6318
6319 rc = VMXReadVmcs32(idxLimit, &u32Val);
6320 AssertRCReturn(rc, rc);
6321 pSelReg->u32Limit = u32Val;
6322
6323 uint64_t u64Val = 0;
6324 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6325 AssertRCReturn(rc, rc);
6326 pSelReg->u64Base = u64Val;
6327
6328 rc = VMXReadVmcs32(idxAccess, &u32Val);
6329 AssertRCReturn(rc, rc);
6330 pSelReg->Attr.u = u32Val;
6331
6332 /*
6333 * If VT-x marks the segment as unusable, most other bits remain undefined:
6334 * - For CS the L, D and G bits have meaning.
6335 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6336 * - For the remaining data segments no bits are defined.
6337 *
6338 * The present bit and the unusable bit has been observed to be set at the
6339 * same time (the selector was supposed to be invalid as we started executing
6340 * a V8086 interrupt in ring-0).
6341 *
6342 * What should be important for the rest of the VBox code, is that the P bit is
6343 * cleared. Some of the other VBox code recognizes the unusable bit, but
6344 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6345 * safe side here, we'll strip off P and other bits we don't care about. If
6346 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6347 *
6348 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6349 */
6350 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6351 {
6352 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6353
6354 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6355 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6356 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6357
6358 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6359#ifdef DEBUG_bird
6360 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6361 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6362 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6363#endif
6364 }
6365 return VINF_SUCCESS;
6366}
6367
6368
6369#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6370# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6371 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6372 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6373#else
6374# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6375 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6376 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6377#endif
6378
6379
6380/**
6381 * Saves the guest segment registers from the current VMCS into the guest-CPU
6382 * context.
6383 *
6384 * @returns VBox status code.
6385 * @param pVCpu The cross context virtual CPU structure.
6386 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6387 * out-of-sync. Make sure to update the required fields
6388 * before using them.
6389 *
6390 * @remarks No-long-jump zone!!!
6391 */
6392static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6393{
6394 /* Guest segment registers. */
6395 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6396 {
6397 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6398 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6399 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6400 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6401 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6402 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6403 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6404
6405 /* Restore segment attributes for real-on-v86 mode hack. */
6406 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6407 {
6408 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6409 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6410 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6411 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6412 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6413 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6414 }
6415 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6416 }
6417
6418 return VINF_SUCCESS;
6419}
6420
6421
6422/**
6423 * Saves the guest descriptor table registers and task register from the current
6424 * VMCS into the guest-CPU context.
6425 *
6426 * @returns VBox status code.
6427 * @param pVCpu The cross context virtual CPU structure.
6428 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6429 * out-of-sync. Make sure to update the required fields
6430 * before using them.
6431 *
6432 * @remarks No-long-jump zone!!!
6433 */
6434static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6435{
6436 int rc = VINF_SUCCESS;
6437
6438 /* Guest LDTR. */
6439 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6440 {
6441 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6442 AssertRCReturn(rc, rc);
6443 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6444 }
6445
6446 /* Guest GDTR. */
6447 uint64_t u64Val = 0;
6448 uint32_t u32Val = 0;
6449 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6450 {
6451 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6452 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6453 pMixedCtx->gdtr.pGdt = u64Val;
6454 pMixedCtx->gdtr.cbGdt = u32Val;
6455 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6456 }
6457
6458 /* Guest IDTR. */
6459 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6460 {
6461 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6462 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6463 pMixedCtx->idtr.pIdt = u64Val;
6464 pMixedCtx->idtr.cbIdt = u32Val;
6465 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6466 }
6467
6468 /* Guest TR. */
6469 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6470 {
6471 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6472 AssertRCReturn(rc, rc);
6473
6474 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6475 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6476 {
6477 rc = VMXLOCAL_READ_SEG(TR, tr);
6478 AssertRCReturn(rc, rc);
6479 }
6480 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6481 }
6482 return rc;
6483}
6484
6485#undef VMXLOCAL_READ_SEG
6486
6487
6488/**
6489 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6490 * context.
6491 *
6492 * @returns VBox status code.
6493 * @param pVCpu The cross context virtual CPU structure.
6494 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6495 * out-of-sync. Make sure to update the required fields
6496 * before using them.
6497 *
6498 * @remarks No-long-jump zone!!!
6499 */
6500static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6501{
6502 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6503 {
6504 if (!pVCpu->hm.s.fUsingHyperDR7)
6505 {
6506 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6507 uint32_t u32Val;
6508 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6509 pMixedCtx->dr[7] = u32Val;
6510 }
6511
6512 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6513 }
6514 return VINF_SUCCESS;
6515}
6516
6517
6518/**
6519 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6520 *
6521 * @returns VBox status code.
6522 * @param pVCpu The cross context virtual CPU structure.
6523 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6524 * out-of-sync. Make sure to update the required fields
6525 * before using them.
6526 *
6527 * @remarks No-long-jump zone!!!
6528 */
6529static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6530{
6531 NOREF(pMixedCtx);
6532
6533 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6534 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6535 return VINF_SUCCESS;
6536}
6537
6538
6539/**
6540 * Saves the entire guest state from the currently active VMCS into the
6541 * guest-CPU context.
6542 *
6543 * This essentially VMREADs all guest-data.
6544 *
6545 * @returns VBox status code.
6546 * @param pVCpu The cross context virtual CPU structure.
6547 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6548 * out-of-sync. Make sure to update the required fields
6549 * before using them.
6550 */
6551static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6552{
6553 Assert(pVCpu);
6554 Assert(pMixedCtx);
6555
6556 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6557 return VINF_SUCCESS;
6558
6559 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6560 again on the ring-3 callback path, there is no real need to. */
6561 if (VMMRZCallRing3IsEnabled(pVCpu))
6562 VMMR0LogFlushDisable(pVCpu);
6563 else
6564 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6565 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6566
6567 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6568 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6569
6570 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6571 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6572
6573 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6574 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6575
6576 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6577 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6578
6579 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6580 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6581
6582 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6583 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6584
6585 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6586 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6587
6588 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6589 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6590
6591 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6592 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6593
6594 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6595 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6596
6597 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6598 ("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
6599 HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
6600
6601 if (VMMRZCallRing3IsEnabled(pVCpu))
6602 VMMR0LogFlushEnable(pVCpu);
6603
6604 return VINF_SUCCESS;
6605}
6606
6607
6608/**
6609 * Saves basic guest registers needed for IEM instruction execution.
6610 *
6611 * @returns VBox status code (OR-able).
6612 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6613 * @param pMixedCtx Pointer to the CPU context of the guest.
6614 * @param fMemory Whether the instruction being executed operates on
6615 * memory or not. Only CR0 is synced up if clear.
6616 * @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
6617 */
6618static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
6619{
6620 /*
6621 * We assume all general purpose registers other than RSP are available.
6622 *
6623 * RIP is a must, as it will be incremented or otherwise changed.
6624 *
6625 * RFLAGS are always required to figure the CPL.
6626 *
6627 * RSP isn't always required, however it's a GPR, so frequently required.
6628 *
6629 * SS and CS are the only segment register needed if IEM doesn't do memory
6630 * access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
6631 *
6632 * CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
6633 * be required for memory accesses.
6634 *
6635 * Note! Before IEM dispatches an exception, it will call us to sync in everything.
6636 */
6637 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6638 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6639 if (fNeedRsp)
6640 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6641 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6642 if (!fMemory)
6643 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6644 else
6645 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6646 AssertRCReturn(rc, rc);
6647 return rc;
6648}
6649
6650
6651/**
6652 * Ensures that we've got a complete basic guest-context.
6653 *
6654 * This excludes the FPU, SSE, AVX, and similar extended state. The interface
6655 * is for the interpreter.
6656 *
6657 * @returns VBox status code.
6658 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6659 * @param pMixedCtx Pointer to the guest-CPU context which may have data
6660 * needing to be synced in.
6661 * @thread EMT(pVCpu)
6662 */
6663VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6664{
6665 /* Note! Since this is only applicable to VT-x, the implementation is placed
6666 in the VT-x part of the sources instead of the generic stuff. */
6667 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
6668 return hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6669 return VINF_SUCCESS;
6670}
6671
6672
6673/**
6674 * Check per-VM and per-VCPU force flag actions that require us to go back to
6675 * ring-3 for one reason or another.
6676 *
6677 * @returns Strict VBox status code (i.e. informational status codes too)
6678 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6679 * ring-3.
6680 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6681 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6682 * interrupts)
6683 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6684 * all EMTs to be in ring-3.
6685 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6686 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6687 * to the EM loop.
6688 *
6689 * @param pVM The cross context VM structure.
6690 * @param pVCpu The cross context virtual CPU structure.
6691 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6692 * out-of-sync. Make sure to update the required fields
6693 * before using them.
6694 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6695 */
6696static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6697{
6698 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6699
6700 /*
6701 * Anything pending? Should be more likely than not if we're doing a good job.
6702 */
6703 if ( !fStepping
6704 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6705 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6706 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6707 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6708 return VINF_SUCCESS;
6709
6710 /* We need the control registers now, make sure the guest-CPU context is updated. */
6711 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6712 AssertRCReturn(rc3, rc3);
6713
6714 /* Pending HM CR3 sync. */
6715 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6716 {
6717 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6718 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6719 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6720 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6721 }
6722
6723 /* Pending HM PAE PDPEs. */
6724 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6725 {
6726 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6727 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6728 }
6729
6730 /* Pending PGM C3 sync. */
6731 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6732 {
6733 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6734 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6735 if (rcStrict2 != VINF_SUCCESS)
6736 {
6737 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6738 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6739 return rcStrict2;
6740 }
6741 }
6742
6743 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6744 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6745 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6746 {
6747 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6748 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6749 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6750 return rc2;
6751 }
6752
6753 /* Pending VM request packets, such as hardware interrupts. */
6754 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6755 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6756 {
6757 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6758 return VINF_EM_PENDING_REQUEST;
6759 }
6760
6761 /* Pending PGM pool flushes. */
6762 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6763 {
6764 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6765 return VINF_PGM_POOL_FLUSH_PENDING;
6766 }
6767
6768 /* Pending DMA requests. */
6769 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6770 {
6771 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6772 return VINF_EM_RAW_TO_R3;
6773 }
6774
6775 return VINF_SUCCESS;
6776}
6777
6778
6779/**
6780 * Converts any TRPM trap into a pending HM event. This is typically used when
6781 * entering from ring-3 (not longjmp returns).
6782 *
6783 * @param pVCpu The cross context virtual CPU structure.
6784 */
6785static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6786{
6787 Assert(TRPMHasTrap(pVCpu));
6788 Assert(!pVCpu->hm.s.Event.fPending);
6789
6790 uint8_t uVector;
6791 TRPMEVENT enmTrpmEvent;
6792 RTGCUINT uErrCode;
6793 RTGCUINTPTR GCPtrFaultAddress;
6794 uint8_t cbInstr;
6795
6796 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6797 AssertRC(rc);
6798
6799 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6800 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6801 if (enmTrpmEvent == TRPM_TRAP)
6802 {
6803 switch (uVector)
6804 {
6805 case X86_XCPT_NMI:
6806 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6807 break;
6808
6809 case X86_XCPT_BP:
6810 case X86_XCPT_OF:
6811 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6812 break;
6813
6814 case X86_XCPT_PF:
6815 case X86_XCPT_DF:
6816 case X86_XCPT_TS:
6817 case X86_XCPT_NP:
6818 case X86_XCPT_SS:
6819 case X86_XCPT_GP:
6820 case X86_XCPT_AC:
6821 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6822 /* no break! */
6823 default:
6824 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6825 break;
6826 }
6827 }
6828 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6829 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6830 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6831 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6832 else
6833 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6834
6835 rc = TRPMResetTrap(pVCpu);
6836 AssertRC(rc);
6837 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6838 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6839
6840 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6841 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6842}
6843
6844
6845/**
6846 * Converts the pending HM event into a TRPM trap.
6847 *
6848 * @param pVCpu The cross context virtual CPU structure.
6849 */
6850static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6851{
6852 Assert(pVCpu->hm.s.Event.fPending);
6853
6854 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6855 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6856 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6857 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6858
6859 /* If a trap was already pending, we did something wrong! */
6860 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6861
6862 TRPMEVENT enmTrapType;
6863 switch (uVectorType)
6864 {
6865 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6866 enmTrapType = TRPM_HARDWARE_INT;
6867 break;
6868
6869 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6870 enmTrapType = TRPM_SOFTWARE_INT;
6871 break;
6872
6873 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6874 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6875 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6876 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6877 enmTrapType = TRPM_TRAP;
6878 break;
6879
6880 default:
6881 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6882 enmTrapType = TRPM_32BIT_HACK;
6883 break;
6884 }
6885
6886 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6887
6888 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6889 AssertRC(rc);
6890
6891 if (fErrorCodeValid)
6892 TRPMSetErrorCode(pVCpu, uErrorCode);
6893
6894 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6895 && uVector == X86_XCPT_PF)
6896 {
6897 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6898 }
6899 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6900 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6901 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6902 {
6903 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6904 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6905 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6906 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6907 }
6908
6909 /* Clear any pending events from the VMCS. */
6910 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
6911 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
6912
6913 /* We're now done converting the pending event. */
6914 pVCpu->hm.s.Event.fPending = false;
6915}
6916
6917
6918/**
6919 * Does the necessary state syncing before returning to ring-3 for any reason
6920 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6921 *
6922 * @returns VBox status code.
6923 * @param pVM The cross context VM structure.
6924 * @param pVCpu The cross context virtual CPU structure.
6925 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6926 * be out-of-sync. Make sure to update the required
6927 * fields before using them.
6928 * @param fSaveGuestState Whether to save the guest state or not.
6929 *
6930 * @remarks No-long-jmp zone!!!
6931 */
6932static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6933{
6934 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6935 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6936
6937 RTCPUID idCpu = RTMpCpuId();
6938 Log4Func(("HostCpuId=%u\n", idCpu));
6939
6940 /*
6941 * !!! IMPORTANT !!!
6942 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
6943 */
6944
6945 /* Save the guest state if necessary. */
6946 if ( fSaveGuestState
6947 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6948 {
6949 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6950 AssertRCReturn(rc, rc);
6951 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6952 }
6953
6954 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6955 if (CPUMIsGuestFPUStateActive(pVCpu))
6956 {
6957 /* We shouldn't reload CR0 without saving it first. */
6958 if (!fSaveGuestState)
6959 {
6960 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6961 AssertRCReturn(rc, rc);
6962 }
6963 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6964 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6965 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6966 }
6967
6968 /* Restore host debug registers if necessary and resync on next R0 reentry. */
6969#ifdef VBOX_STRICT
6970 if (CPUMIsHyperDebugStateActive(pVCpu))
6971 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6972#endif
6973 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6974 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6975 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6976 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6977
6978#if HC_ARCH_BITS == 64
6979 /* Restore host-state bits that VT-x only restores partially. */
6980 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6981 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6982 {
6983 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6984 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6985 }
6986 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6987#endif
6988
6989#if HC_ARCH_BITS == 64
6990 /* Restore the lazy host MSRs as we're leaving VT-x context. */
6991 if ( pVM->hm.s.fAllow64BitGuests
6992 && pVCpu->hm.s.vmx.fLazyMsrs)
6993 {
6994 /* We shouldn't reload the guest MSRs without saving it first. */
6995 if (!fSaveGuestState)
6996 {
6997 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6998 AssertRCReturn(rc, rc);
6999 }
7000 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7001 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7002 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7003 }
7004#endif
7005
7006 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7007 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7008
7009 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7010 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7011 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7012 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7013 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7014 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7015 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7016 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7017
7018 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7019
7020 /** @todo This partially defeats the purpose of having preemption hooks.
7021 * The problem is, deregistering the hooks should be moved to a place that
7022 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7023 * context.
7024 */
7025 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7026 {
7027 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7028 AssertRCReturn(rc, rc);
7029
7030 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7031 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7032 }
7033 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7034 NOREF(idCpu);
7035
7036 return VINF_SUCCESS;
7037}
7038
7039
7040/**
7041 * Leaves the VT-x session.
7042 *
7043 * @returns VBox status code.
7044 * @param pVM The cross context VM structure.
7045 * @param pVCpu The cross context virtual CPU structure.
7046 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7047 * out-of-sync. Make sure to update the required fields
7048 * before using them.
7049 *
7050 * @remarks No-long-jmp zone!!!
7051 */
7052DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7053{
7054 HM_DISABLE_PREEMPT();
7055 HMVMX_ASSERT_CPU_SAFE();
7056 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7057 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7058
7059 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7060 and done this from the VMXR0ThreadCtxCallback(). */
7061 if (!pVCpu->hm.s.fLeaveDone)
7062 {
7063 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7064 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7065 pVCpu->hm.s.fLeaveDone = true;
7066 }
7067 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7068
7069 /*
7070 * !!! IMPORTANT !!!
7071 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7072 */
7073
7074 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7075 /** @todo Deregistering here means we need to VMCLEAR always
7076 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7077 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7078 VMMR0ThreadCtxHookDisable(pVCpu);
7079
7080 /* Leave HM context. This takes care of local init (term). */
7081 int rc = HMR0LeaveCpu(pVCpu);
7082
7083 HM_RESTORE_PREEMPT();
7084 return rc;
7085}
7086
7087
7088/**
7089 * Does the necessary state syncing before doing a longjmp to ring-3.
7090 *
7091 * @returns VBox status code.
7092 * @param pVM The cross context VM structure.
7093 * @param pVCpu The cross context virtual CPU structure.
7094 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7095 * out-of-sync. Make sure to update the required fields
7096 * before using them.
7097 *
7098 * @remarks No-long-jmp zone!!!
7099 */
7100DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7101{
7102 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7103}
7104
7105
7106/**
7107 * Take necessary actions before going back to ring-3.
7108 *
7109 * An action requires us to go back to ring-3. This function does the necessary
7110 * steps before we can safely return to ring-3. This is not the same as longjmps
7111 * to ring-3, this is voluntary and prepares the guest so it may continue
7112 * executing outside HM (recompiler/IEM).
7113 *
7114 * @returns VBox status code.
7115 * @param pVM The cross context VM structure.
7116 * @param pVCpu The cross context virtual CPU structure.
7117 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7118 * out-of-sync. Make sure to update the required fields
7119 * before using them.
7120 * @param rcExit The reason for exiting to ring-3. Can be
7121 * VINF_VMM_UNKNOWN_RING3_CALL.
7122 */
7123static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7124{
7125 Assert(pVM);
7126 Assert(pVCpu);
7127 Assert(pMixedCtx);
7128 HMVMX_ASSERT_PREEMPT_SAFE();
7129
7130 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7131 {
7132 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7133 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7134 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7135 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7136 }
7137
7138 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7139 VMMRZCallRing3Disable(pVCpu);
7140 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7141
7142 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7143 if (pVCpu->hm.s.Event.fPending)
7144 {
7145 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7146 Assert(!pVCpu->hm.s.Event.fPending);
7147 }
7148
7149 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7150 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7151
7152 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7153 and if we're injecting an event we should have a TRPM trap pending. */
7154 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7155#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7156 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7157#endif
7158
7159 /* Save guest state and restore host state bits. */
7160 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7161 AssertRCReturn(rc, rc);
7162 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7163 /* Thread-context hooks are unregistered at this point!!! */
7164
7165 /* Sync recompiler state. */
7166 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7167 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7168 | CPUM_CHANGED_LDTR
7169 | CPUM_CHANGED_GDTR
7170 | CPUM_CHANGED_IDTR
7171 | CPUM_CHANGED_TR
7172 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7173 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7174 if ( pVM->hm.s.fNestedPaging
7175 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7176 {
7177 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7178 }
7179
7180 Assert(!pVCpu->hm.s.fClearTrapFlag);
7181
7182 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7183 if (rcExit != VINF_EM_RAW_INTERRUPT)
7184 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7185
7186 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7187
7188 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7189 VMMRZCallRing3RemoveNotification(pVCpu);
7190 VMMRZCallRing3Enable(pVCpu);
7191
7192 return rc;
7193}
7194
7195
7196/**
7197 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7198 * longjump to ring-3 and possibly get preempted.
7199 *
7200 * @returns VBox status code.
7201 * @param pVCpu The cross context virtual CPU structure.
7202 * @param enmOperation The operation causing the ring-3 longjump.
7203 * @param pvUser Opaque pointer to the guest-CPU context. The data
7204 * may be out-of-sync. Make sure to update the required
7205 * fields before using them.
7206 */
7207static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7208{
7209 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7210 {
7211 /*
7212 * !!! IMPORTANT !!!
7213 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7214 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7215 */
7216 VMMRZCallRing3RemoveNotification(pVCpu);
7217 VMMRZCallRing3Disable(pVCpu);
7218 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7219 RTThreadPreemptDisable(&PreemptState);
7220
7221 PVM pVM = pVCpu->CTX_SUFF(pVM);
7222 if (CPUMIsGuestFPUStateActive(pVCpu))
7223 CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
7224
7225 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7226
7227#if HC_ARCH_BITS == 64
7228 /* Restore host-state bits that VT-x only restores partially. */
7229 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7230 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7231 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7232 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7233
7234 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7235 if ( pVM->hm.s.fAllow64BitGuests
7236 && pVCpu->hm.s.vmx.fLazyMsrs)
7237 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7238#endif
7239 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7240 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7241 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7242 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7243 {
7244 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7245 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7246 }
7247
7248 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7249 VMMR0ThreadCtxHookDisable(pVCpu);
7250 HMR0LeaveCpu(pVCpu);
7251 RTThreadPreemptRestore(&PreemptState);
7252 return VINF_SUCCESS;
7253 }
7254
7255 Assert(pVCpu);
7256 Assert(pvUser);
7257 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7258 HMVMX_ASSERT_PREEMPT_SAFE();
7259
7260 VMMRZCallRing3Disable(pVCpu);
7261 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7262
7263 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7264 enmOperation));
7265
7266 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7267 AssertRCReturn(rc, rc);
7268
7269 VMMRZCallRing3Enable(pVCpu);
7270 return VINF_SUCCESS;
7271}
7272
7273
7274/**
7275 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7276 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7277 *
7278 * @param pVCpu The cross context virtual CPU structure.
7279 */
7280DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7281{
7282 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7283 {
7284 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7285 {
7286 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7287 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7288 AssertRC(rc);
7289 Log4(("Setup interrupt-window exiting\n"));
7290 }
7291 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7292}
7293
7294
7295/**
7296 * Clears the interrupt-window exiting control in the VMCS.
7297 *
7298 * @param pVCpu The cross context virtual CPU structure.
7299 */
7300DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7301{
7302 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7303 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7304 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7305 AssertRC(rc);
7306 Log4(("Cleared interrupt-window exiting\n"));
7307}
7308
7309
7310/**
7311 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7312 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7313 *
7314 * @param pVCpu The cross context virtual CPU structure.
7315 */
7316DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7317{
7318 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7319 {
7320 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7321 {
7322 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7323 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7324 AssertRC(rc);
7325 Log4(("Setup NMI-window exiting\n"));
7326 }
7327 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7328}
7329
7330
7331/**
7332 * Clears the NMI-window exiting control in the VMCS.
7333 *
7334 * @param pVCpu The cross context virtual CPU structure.
7335 */
7336DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7337{
7338 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7339 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7340 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7341 AssertRC(rc);
7342 Log4(("Cleared NMI-window exiting\n"));
7343}
7344
7345
7346/**
7347 * Evaluates the event to be delivered to the guest and sets it as the pending
7348 * event.
7349 *
7350 * @param pVCpu The cross context virtual CPU structure.
7351 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7352 * out-of-sync. Make sure to update the required fields
7353 * before using them.
7354 */
7355static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7356{
7357 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7358 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7359 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7360 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7361 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7362
7363 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7364 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7365 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7366 Assert(!TRPMHasTrap(pVCpu));
7367
7368 /*
7369 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7370 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7371 */
7372 /** @todo SMI. SMIs take priority over NMIs. */
7373 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7374 {
7375 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7376 if ( !pVCpu->hm.s.Event.fPending
7377 && !fBlockNmi
7378 && !fBlockSti
7379 && !fBlockMovSS)
7380 {
7381 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7382 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7383 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7384
7385 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7386 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7387 }
7388 else
7389 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7390 }
7391 /*
7392 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7393 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt.
7394 */
7395 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7396 && !pVCpu->hm.s.fSingleInstruction)
7397 {
7398 Assert(!DBGFIsStepping(pVCpu));
7399 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7400 AssertRC(rc);
7401 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7402 if ( !pVCpu->hm.s.Event.fPending
7403 && !fBlockInt
7404 && !fBlockSti
7405 && !fBlockMovSS)
7406 {
7407 uint8_t u8Interrupt;
7408 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7409 if (RT_SUCCESS(rc))
7410 {
7411 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7412 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7413 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7414
7415 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7416 }
7417 else
7418 {
7419 /** @todo Does this actually happen? If not turn it into an assertion. */
7420 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
7421 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7422 }
7423 }
7424 else
7425 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7426 }
7427}
7428
7429
7430/**
7431 * Sets a pending-debug exception to be delivered to the guest if the guest is
7432 * single-stepping in the VMCS.
7433 *
7434 * @param pVCpu The cross context virtual CPU structure.
7435 */
7436DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7437{
7438 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7439 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7440 AssertRC(rc);
7441}
7442
7443
7444/**
7445 * Injects any pending events into the guest if the guest is in a state to
7446 * receive them.
7447 *
7448 * @returns Strict VBox status code (i.e. informational status codes too).
7449 * @param pVCpu The cross context virtual CPU structure.
7450 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7451 * out-of-sync. Make sure to update the required fields
7452 * before using them.
7453 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7454 * return VINF_EM_DBG_STEPPED if the event was
7455 * dispatched directly.
7456 */
7457static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
7458{
7459 HMVMX_ASSERT_PREEMPT_SAFE();
7460 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7461
7462 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7463 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7464 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7465 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7466
7467 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7468 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7469 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7470 Assert(!TRPMHasTrap(pVCpu));
7471
7472 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7473 if (pVCpu->hm.s.Event.fPending)
7474 {
7475 /*
7476 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7477 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7478 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7479 *
7480 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7481 */
7482 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7483#ifdef VBOX_STRICT
7484 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7485 {
7486 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7487 Assert(!fBlockInt);
7488 Assert(!fBlockSti);
7489 Assert(!fBlockMovSS);
7490 }
7491 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7492 {
7493 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7494 Assert(!fBlockSti);
7495 Assert(!fBlockMovSS);
7496 Assert(!fBlockNmi);
7497 }
7498#endif
7499 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7500 (uint8_t)uIntType));
7501 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7502 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7503 fStepping, &uIntrState);
7504 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7505
7506 /* Update the interruptibility-state as it could have been changed by
7507 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7508 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7509 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7510
7511 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7512 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7513 else
7514 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7515 }
7516
7517 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7518 if ( fBlockSti
7519 || fBlockMovSS)
7520 {
7521 if (!pVCpu->hm.s.fSingleInstruction)
7522 {
7523 /*
7524 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7525 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7526 * See Intel spec. 27.3.4 "Saving Non-Register State".
7527 */
7528 Assert(!DBGFIsStepping(pVCpu));
7529 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7530 AssertRCReturn(rc2, rc2);
7531 if (pMixedCtx->eflags.Bits.u1TF)
7532 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7533 }
7534 else if (pMixedCtx->eflags.Bits.u1TF)
7535 {
7536 /*
7537 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7538 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7539 */
7540 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7541 uIntrState = 0;
7542 }
7543 }
7544
7545 /*
7546 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7547 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7548 */
7549 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7550 AssertRC(rc2);
7551
7552 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7553 NOREF(fBlockMovSS); NOREF(fBlockSti);
7554 return rcStrict;
7555}
7556
7557
7558/**
7559 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7560 *
7561 * @param pVCpu The cross context virtual CPU structure.
7562 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7563 * out-of-sync. Make sure to update the required fields
7564 * before using them.
7565 */
7566DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7567{
7568 NOREF(pMixedCtx);
7569 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7570 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7571}
7572
7573
7574/**
7575 * Injects a double-fault (\#DF) exception into the VM.
7576 *
7577 * @returns Strict VBox status code (i.e. informational status codes too).
7578 * @param pVCpu The cross context virtual CPU structure.
7579 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7580 * out-of-sync. Make sure to update the required fields
7581 * before using them.
7582 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7583 * and should return VINF_EM_DBG_STEPPED if the event
7584 * is injected directly (register modified by us, not
7585 * by hardware on VM-entry).
7586 * @param puIntrState Pointer to the current guest interruptibility-state.
7587 * This interruptibility-state will be updated if
7588 * necessary. This cannot not be NULL.
7589 */
7590DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7591{
7592 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7593 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7594 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7595 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7596 fStepping, puIntrState);
7597}
7598
7599
7600/**
7601 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7602 *
7603 * @param pVCpu The cross context virtual CPU structure.
7604 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7605 * out-of-sync. Make sure to update the required fields
7606 * before using them.
7607 */
7608DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7609{
7610 NOREF(pMixedCtx);
7611 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7612 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7613 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7614}
7615
7616
7617/**
7618 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7619 *
7620 * @param pVCpu The cross context virtual CPU structure.
7621 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7622 * out-of-sync. Make sure to update the required fields
7623 * before using them.
7624 * @param cbInstr The value of RIP that is to be pushed on the guest
7625 * stack.
7626 */
7627DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7628{
7629 NOREF(pMixedCtx);
7630 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7631 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7632 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7633}
7634
7635
7636/**
7637 * Injects a general-protection (\#GP) fault into the VM.
7638 *
7639 * @returns Strict VBox status code (i.e. informational status codes too).
7640 * @param pVCpu The cross context virtual CPU structure.
7641 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7642 * out-of-sync. Make sure to update the required fields
7643 * before using them.
7644 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7645 * mode, i.e. in real-mode it's not valid).
7646 * @param u32ErrorCode The error code associated with the \#GP.
7647 * @param fStepping Whether we're running in
7648 * hmR0VmxRunGuestCodeStep() and should return
7649 * VINF_EM_DBG_STEPPED if the event is injected
7650 * directly (register modified by us, not by
7651 * hardware on VM-entry).
7652 * @param puIntrState Pointer to the current guest interruptibility-state.
7653 * This interruptibility-state will be updated if
7654 * necessary. This cannot not be NULL.
7655 */
7656DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7657 bool fStepping, uint32_t *puIntrState)
7658{
7659 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7660 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7661 if (fErrorCodeValid)
7662 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7663 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7664 fStepping, puIntrState);
7665}
7666
7667
7668/**
7669 * Sets a general-protection (\#GP) exception as pending-for-injection into the
7670 * VM.
7671 *
7672 * @param pVCpu The cross context virtual CPU structure.
7673 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7674 * out-of-sync. Make sure to update the required fields
7675 * before using them.
7676 * @param u32ErrorCode The error code associated with the \#GP.
7677 */
7678DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7679{
7680 NOREF(pMixedCtx);
7681 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7682 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7683 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7684 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7685}
7686
7687
7688/**
7689 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7690 *
7691 * @param pVCpu The cross context virtual CPU structure.
7692 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7693 * out-of-sync. Make sure to update the required fields
7694 * before using them.
7695 * @param uVector The software interrupt vector number.
7696 * @param cbInstr The value of RIP that is to be pushed on the guest
7697 * stack.
7698 */
7699DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7700{
7701 NOREF(pMixedCtx);
7702 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7703 if ( uVector == X86_XCPT_BP
7704 || uVector == X86_XCPT_OF)
7705 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7706 else
7707 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7708 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7709}
7710
7711
7712/**
7713 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7714 * stack.
7715 *
7716 * @returns Strict VBox status code (i.e. informational status codes too).
7717 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7718 * @param pVM The cross context VM structure.
7719 * @param pMixedCtx Pointer to the guest-CPU context.
7720 * @param uValue The value to push to the guest stack.
7721 */
7722DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7723{
7724 /*
7725 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7726 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7727 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7728 */
7729 if (pMixedCtx->sp == 1)
7730 return VINF_EM_RESET;
7731 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7732 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7733 AssertRC(rc);
7734 return rc;
7735}
7736
7737
7738/**
7739 * Injects an event into the guest upon VM-entry by updating the relevant fields
7740 * in the VM-entry area in the VMCS.
7741 *
7742 * @returns Strict VBox status code (i.e. informational status codes too).
7743 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7744 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7745 *
7746 * @param pVCpu The cross context virtual CPU structure.
7747 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7748 * be out-of-sync. Make sure to update the required
7749 * fields before using them.
7750 * @param u64IntInfo The VM-entry interruption-information field.
7751 * @param cbInstr The VM-entry instruction length in bytes (for
7752 * software interrupts, exceptions and privileged
7753 * software exceptions).
7754 * @param u32ErrCode The VM-entry exception error code.
7755 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7756 * @param puIntrState Pointer to the current guest interruptibility-state.
7757 * This interruptibility-state will be updated if
7758 * necessary. This cannot not be NULL.
7759 * @param fStepping Whether we're running in
7760 * hmR0VmxRunGuestCodeStep() and should return
7761 * VINF_EM_DBG_STEPPED if the event is injected
7762 * directly (register modified by us, not by
7763 * hardware on VM-entry).
7764 *
7765 * @remarks Requires CR0!
7766 * @remarks No-long-jump zone!!!
7767 */
7768static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7769 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
7770 uint32_t *puIntrState)
7771{
7772 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7773 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7774 Assert(puIntrState);
7775 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7776
7777 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7778 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7779
7780#ifdef VBOX_STRICT
7781 /* Validate the error-code-valid bit for hardware exceptions. */
7782 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7783 {
7784 switch (uVector)
7785 {
7786 case X86_XCPT_PF:
7787 case X86_XCPT_DF:
7788 case X86_XCPT_TS:
7789 case X86_XCPT_NP:
7790 case X86_XCPT_SS:
7791 case X86_XCPT_GP:
7792 case X86_XCPT_AC:
7793 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7794 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7795 /* fallthru */
7796 default:
7797 break;
7798 }
7799 }
7800#endif
7801
7802 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7803 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7804 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7805
7806 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7807
7808 /* We require CR0 to check if the guest is in real-mode. */
7809 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7810 AssertRCReturn(rc, rc);
7811
7812 /*
7813 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7814 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7815 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7816 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7817 */
7818 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7819 {
7820 PVM pVM = pVCpu->CTX_SUFF(pVM);
7821 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7822 {
7823 Assert(PDMVmmDevHeapIsEnabled(pVM));
7824 Assert(pVM->hm.s.vmx.pRealModeTSS);
7825
7826 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7827 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7828 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7829 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7830 AssertRCReturn(rc, rc);
7831 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7832
7833 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7834 size_t const cbIdtEntry = sizeof(X86IDTR16);
7835 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7836 {
7837 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7838 if (uVector == X86_XCPT_DF)
7839 return VINF_EM_RESET;
7840
7841 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7842 if (uVector == X86_XCPT_GP)
7843 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
7844
7845 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7846 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7847 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
7848 fStepping, puIntrState);
7849 }
7850
7851 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7852 uint16_t uGuestIp = pMixedCtx->ip;
7853 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7854 {
7855 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7856 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7857 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7858 }
7859 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7860 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7861
7862 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7863 X86IDTR16 IdtEntry;
7864 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7865 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7866 AssertRCReturn(rc, rc);
7867
7868 /* Construct the stack frame for the interrupt/exception handler. */
7869 VBOXSTRICTRC rcStrict;
7870 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7871 if (rcStrict == VINF_SUCCESS)
7872 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7873 if (rcStrict == VINF_SUCCESS)
7874 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7875
7876 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7877 if (rcStrict == VINF_SUCCESS)
7878 {
7879 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7880 pMixedCtx->rip = IdtEntry.offSel;
7881 pMixedCtx->cs.Sel = IdtEntry.uSel;
7882 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7883 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7884 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7885 && uVector == X86_XCPT_PF)
7886 pMixedCtx->cr2 = GCPtrFaultAddress;
7887
7888 /* If any other guest-state bits are changed here, make sure to update
7889 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7890 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7891 | HM_CHANGED_GUEST_RIP
7892 | HM_CHANGED_GUEST_RFLAGS
7893 | HM_CHANGED_GUEST_RSP);
7894
7895 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7896 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7897 {
7898 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7899 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7900 Log4(("Clearing inhibition due to STI.\n"));
7901 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7902 }
7903 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7904 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
7905
7906 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7907 it, if we are returning to ring-3 before executing guest code. */
7908 pVCpu->hm.s.Event.fPending = false;
7909
7910 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
7911 if (fStepping)
7912 rcStrict = VINF_EM_DBG_STEPPED;
7913 }
7914 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7915 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7916 return rcStrict;
7917 }
7918
7919 /*
7920 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7921 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7922 */
7923 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7924 }
7925
7926 /* Validate. */
7927 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7928 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7929 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7930
7931 /* Inject. */
7932 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7933 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7934 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7935 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7936
7937 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7938 && uVector == X86_XCPT_PF)
7939 pMixedCtx->cr2 = GCPtrFaultAddress;
7940
7941 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7942 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7943
7944 AssertRCReturn(rc, rc);
7945 return VINF_SUCCESS;
7946}
7947
7948
7949/**
7950 * Clears the interrupt-window exiting control in the VMCS and if necessary
7951 * clears the current event in the VMCS as well.
7952 *
7953 * @returns VBox status code.
7954 * @param pVCpu The cross context virtual CPU structure.
7955 *
7956 * @remarks Use this function only to clear events that have not yet been
7957 * delivered to the guest but are injected in the VMCS!
7958 * @remarks No-long-jump zone!!!
7959 */
7960static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
7961{
7962 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7963
7964 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7965 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7966
7967 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
7968 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
7969}
7970
7971
7972/**
7973 * Enters the VT-x session.
7974 *
7975 * @returns VBox status code.
7976 * @param pVM The cross context VM structure.
7977 * @param pVCpu The cross context virtual CPU structure.
7978 * @param pCpu Pointer to the CPU info struct.
7979 */
7980VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7981{
7982 AssertPtr(pVM);
7983 AssertPtr(pVCpu);
7984 Assert(pVM->hm.s.vmx.fSupported);
7985 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7986 NOREF(pCpu); NOREF(pVM);
7987
7988 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7989 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7990
7991#ifdef VBOX_STRICT
7992 /* Make sure we're in VMX root mode. */
7993 RTCCUINTREG u32HostCR4 = ASMGetCR4();
7994 if (!(u32HostCR4 & X86_CR4_VMXE))
7995 {
7996 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7997 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7998 }
7999#endif
8000
8001 /*
8002 * Load the VCPU's VMCS as the current (and active) one.
8003 */
8004 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8005 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8006 if (RT_FAILURE(rc))
8007 return rc;
8008
8009 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8010 pVCpu->hm.s.fLeaveDone = false;
8011 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8012
8013 return VINF_SUCCESS;
8014}
8015
8016
8017/**
8018 * The thread-context callback (only on platforms which support it).
8019 *
8020 * @param enmEvent The thread-context event.
8021 * @param pVCpu The cross context virtual CPU structure.
8022 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8023 * @thread EMT(pVCpu)
8024 */
8025VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8026{
8027 NOREF(fGlobalInit);
8028
8029 switch (enmEvent)
8030 {
8031 case RTTHREADCTXEVENT_OUT:
8032 {
8033 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8034 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8035 VMCPU_ASSERT_EMT(pVCpu);
8036
8037 PVM pVM = pVCpu->CTX_SUFF(pVM);
8038 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8039
8040 /* No longjmps (logger flushes, locks) in this fragile context. */
8041 VMMRZCallRing3Disable(pVCpu);
8042 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8043
8044 /*
8045 * Restore host-state (FPU, debug etc.)
8046 */
8047 if (!pVCpu->hm.s.fLeaveDone)
8048 {
8049 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8050 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8051 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
8052 pVCpu->hm.s.fLeaveDone = true;
8053 }
8054
8055 /* Leave HM context, takes care of local init (term). */
8056 int rc = HMR0LeaveCpu(pVCpu);
8057 AssertRC(rc); NOREF(rc);
8058
8059 /* Restore longjmp state. */
8060 VMMRZCallRing3Enable(pVCpu);
8061 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8062 break;
8063 }
8064
8065 case RTTHREADCTXEVENT_IN:
8066 {
8067 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8068 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8069 VMCPU_ASSERT_EMT(pVCpu);
8070
8071 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8072 VMMRZCallRing3Disable(pVCpu);
8073 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8074
8075 /* Initialize the bare minimum state required for HM. This takes care of
8076 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8077 int rc = HMR0EnterCpu(pVCpu);
8078 AssertRC(rc);
8079 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8080
8081 /* Load the active VMCS as the current one. */
8082 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8083 {
8084 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8085 AssertRC(rc); NOREF(rc);
8086 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8087 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8088 }
8089 pVCpu->hm.s.fLeaveDone = false;
8090
8091 /* Restore longjmp state. */
8092 VMMRZCallRing3Enable(pVCpu);
8093 break;
8094 }
8095
8096 default:
8097 break;
8098 }
8099}
8100
8101
8102/**
8103 * Saves the host state in the VMCS host-state.
8104 * Sets up the VM-exit MSR-load area.
8105 *
8106 * The CPU state will be loaded from these fields on every successful VM-exit.
8107 *
8108 * @returns VBox status code.
8109 * @param pVM The cross context VM structure.
8110 * @param pVCpu The cross context virtual CPU structure.
8111 *
8112 * @remarks No-long-jump zone!!!
8113 */
8114static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8115{
8116 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8117
8118 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8119 return VINF_SUCCESS;
8120
8121 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8122 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8123
8124 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8125 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8126
8127 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8128 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8129
8130 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8131 return rc;
8132}
8133
8134
8135/**
8136 * Saves the host state in the VMCS host-state.
8137 *
8138 * @returns VBox status code.
8139 * @param pVM The cross context VM structure.
8140 * @param pVCpu The cross context virtual CPU structure.
8141 *
8142 * @remarks No-long-jump zone!!!
8143 */
8144VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8145{
8146 AssertPtr(pVM);
8147 AssertPtr(pVCpu);
8148
8149 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8150
8151 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8152 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8153 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8154 return hmR0VmxSaveHostState(pVM, pVCpu);
8155}
8156
8157
8158/**
8159 * Loads the guest state into the VMCS guest-state area.
8160 *
8161 * The will typically be done before VM-entry when the guest-CPU state and the
8162 * VMCS state may potentially be out of sync.
8163 *
8164 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8165 * VM-entry controls.
8166 * Sets up the appropriate VMX non-root function to execute guest code based on
8167 * the guest CPU mode.
8168 *
8169 * @returns VBox status code.
8170 * @param pVM The cross context VM structure.
8171 * @param pVCpu The cross context virtual CPU structure.
8172 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8173 * out-of-sync. Make sure to update the required fields
8174 * before using them.
8175 *
8176 * @remarks No-long-jump zone!!!
8177 */
8178static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8179{
8180 AssertPtr(pVM);
8181 AssertPtr(pVCpu);
8182 AssertPtr(pMixedCtx);
8183 HMVMX_ASSERT_PREEMPT_SAFE();
8184
8185 VMMRZCallRing3Disable(pVCpu);
8186 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8187
8188 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8189
8190 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8191
8192 /* Determine real-on-v86 mode. */
8193 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8194 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8195 && CPUMIsGuestInRealModeEx(pMixedCtx))
8196 {
8197 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8198 }
8199
8200 /*
8201 * Load the guest-state into the VMCS.
8202 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8203 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8204 */
8205 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8206 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8207
8208 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8209 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8210 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8211
8212 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8213 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8214 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8215
8216 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8217 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8218
8219 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8220 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8221
8222 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8223 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8224 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8225
8226 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8227 determine we don't have to swap EFER after all. */
8228 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8229 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8230
8231 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8232 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8233
8234 rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8235 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8236
8237 /*
8238 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8239 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8240 */
8241 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8242 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8243
8244 /* Clear any unused and reserved bits. */
8245 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8246
8247 VMMRZCallRing3Enable(pVCpu);
8248
8249 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8250 return rc;
8251}
8252
8253
8254/**
8255 * Loads the state shared between the host and guest into the VMCS.
8256 *
8257 * @param pVM The cross context VM structure.
8258 * @param pVCpu The cross context virtual CPU structure.
8259 * @param pCtx Pointer to the guest-CPU context.
8260 *
8261 * @remarks No-long-jump zone!!!
8262 */
8263static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8264{
8265 NOREF(pVM);
8266
8267 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8268 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8269
8270 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8271 {
8272 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8273 AssertRC(rc);
8274 }
8275
8276 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8277 {
8278 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8279 AssertRC(rc);
8280
8281 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8282 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8283 {
8284 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8285 AssertRC(rc);
8286 }
8287 }
8288
8289 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8290 {
8291#if HC_ARCH_BITS == 64
8292 if (pVM->hm.s.fAllow64BitGuests)
8293 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8294#endif
8295 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8296 }
8297
8298 /* Loading CR0, debug state might have changed intercepts, update VMCS. */
8299 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8300 {
8301 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8302 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8303 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8304 AssertRC(rc);
8305 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8306 }
8307
8308 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8309 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8310}
8311
8312
8313/**
8314 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8315 *
8316 * @param pVM The cross context VM structure.
8317 * @param pVCpu The cross context virtual CPU structure.
8318 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8319 * out-of-sync. Make sure to update the required fields
8320 * before using them.
8321 */
8322DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8323{
8324 HMVMX_ASSERT_PREEMPT_SAFE();
8325
8326 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8327#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8328 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8329#endif
8330
8331 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8332 {
8333 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8334 AssertRC(rc);
8335 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8336 }
8337 else if (HMCPU_CF_VALUE(pVCpu))
8338 {
8339 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8340 AssertRC(rc);
8341 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8342 }
8343
8344 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8345 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8346 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8347 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8348}
8349
8350
8351/**
8352 * Does the preparations before executing guest code in VT-x.
8353 *
8354 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8355 * recompiler/IEM. We must be cautious what we do here regarding committing
8356 * guest-state information into the VMCS assuming we assuredly execute the
8357 * guest in VT-x mode.
8358 *
8359 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8360 * the common-state (TRPM/forceflags), we must undo those changes so that the
8361 * recompiler/IEM can (and should) use them when it resumes guest execution.
8362 * Otherwise such operations must be done when we can no longer exit to ring-3.
8363 *
8364 * @returns Strict VBox status code (i.e. informational status codes too).
8365 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8366 * have been disabled.
8367 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8368 * double-fault into the guest.
8369 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8370 * dispatched directly.
8371 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8372 *
8373 * @param pVM The cross context VM structure.
8374 * @param pVCpu The cross context virtual CPU structure.
8375 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8376 * out-of-sync. Make sure to update the required fields
8377 * before using them.
8378 * @param pVmxTransient Pointer to the VMX transient structure.
8379 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8380 * us ignore some of the reasons for returning to
8381 * ring-3, and return VINF_EM_DBG_STEPPED if event
8382 * dispatching took place.
8383 */
8384static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8385{
8386 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8387
8388#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8389 PGMRZDynMapFlushAutoSet(pVCpu);
8390#endif
8391
8392 /* Check force flag actions that might require us to go back to ring-3. */
8393 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8394 if (rcStrict == VINF_SUCCESS)
8395 { /* FFs doesn't get set all the time. */ }
8396 else
8397 return rcStrict;
8398
8399#ifndef IEM_VERIFICATION_MODE_FULL
8400 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8401 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8402 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8403 {
8404 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8405 RTGCPHYS GCPhysApicBase;
8406 GCPhysApicBase = pMixedCtx->msrApicBase;
8407 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8408
8409 /* Unalias any existing mapping. */
8410 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8411 AssertRCReturn(rc, rc);
8412
8413 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8414 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
8415 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8416 AssertRCReturn(rc, rc);
8417
8418 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8419 }
8420#endif /* !IEM_VERIFICATION_MODE_FULL */
8421
8422 if (TRPMHasTrap(pVCpu))
8423 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8424 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8425
8426 /*
8427 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8428 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8429 */
8430 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, fStepping);
8431 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8432 { /* likely */ }
8433 else
8434 {
8435 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8436 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8437 return rcStrict;
8438 }
8439
8440 /*
8441 * Load the guest state bits, we can handle longjmps/getting preempted here.
8442 *
8443 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8444 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8445 * Hence, this needs to be done -after- injection of events.
8446 */
8447 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8448
8449 /*
8450 * No longjmps to ring-3 from this point on!!!
8451 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8452 * This also disables flushing of the R0-logger instance (if any).
8453 */
8454 VMMRZCallRing3Disable(pVCpu);
8455
8456 /*
8457 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8458 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8459 *
8460 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8461 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8462 *
8463 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8464 * executing guest code.
8465 */
8466 pVmxTransient->fEFlags = ASMIntDisableFlags();
8467
8468 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8469 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8470 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8471 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8472 {
8473 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8474 {
8475 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8476 pVCpu->hm.s.Event.fPending = false;
8477
8478 return VINF_SUCCESS;
8479 }
8480
8481 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8482 rcStrict = VINF_EM_RAW_INTERRUPT;
8483 }
8484 else
8485 {
8486 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8487 rcStrict = VINF_EM_RAW_TO_R3;
8488 }
8489
8490 ASMSetFlags(pVmxTransient->fEFlags);
8491 VMMRZCallRing3Enable(pVCpu);
8492
8493 return rcStrict;
8494}
8495
8496
8497/**
8498 * Prepares to run guest code in VT-x and we've committed to doing so. This
8499 * means there is no backing out to ring-3 or anywhere else at this
8500 * point.
8501 *
8502 * @param pVM The cross context VM structure.
8503 * @param pVCpu The cross context virtual CPU structure.
8504 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8505 * out-of-sync. Make sure to update the required fields
8506 * before using them.
8507 * @param pVmxTransient Pointer to the VMX transient structure.
8508 *
8509 * @remarks Called with preemption disabled.
8510 * @remarks No-long-jump zone!!!
8511 */
8512static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8513{
8514 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8515 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8516 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8517
8518 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8519 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8520
8521#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8522 if (!CPUMIsGuestFPUStateActive(pVCpu))
8523 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8524 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8525#endif
8526
8527 if ( pVCpu->hm.s.fPreloadGuestFpu
8528 && !CPUMIsGuestFPUStateActive(pVCpu))
8529 {
8530 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8531 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8532 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8533 }
8534
8535 /*
8536 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8537 */
8538 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8539 && pVCpu->hm.s.vmx.cMsrs > 0)
8540 {
8541 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8542 }
8543
8544 /*
8545 * Load the host state bits as we may've been preempted (only happens when
8546 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8547 */
8548 /** @todo Why should hmR0VmxSetupVMRunHandler() changing pfnStartVM have
8549 * any effect to the host state needing to be saved? */
8550 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8551 {
8552 /* This ASSUMES that pfnStartVM has been set up already. */
8553 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8554 AssertRC(rc);
8555 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
8556 }
8557 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8558
8559 /*
8560 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8561 */
8562 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8563 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8564 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8565
8566 /* Store status of the shared guest-host state at the time of VM-entry. */
8567#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8568 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8569 {
8570 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8571 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8572 }
8573 else
8574#endif
8575 {
8576 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8577 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8578 }
8579 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8580
8581 /*
8582 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8583 */
8584 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8585 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8586
8587 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8588 RTCPUID idCurrentCpu = pCpu->idCpu;
8589 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8590 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8591 {
8592 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8593 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8594 }
8595
8596 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8597 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8598 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8599 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8600
8601 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8602
8603 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8604 to start executing. */
8605
8606 /*
8607 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8608 */
8609 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8610 {
8611 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8612 {
8613 bool fMsrUpdated;
8614 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8615 AssertRC(rc2);
8616 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8617
8618 rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8619 &fMsrUpdated);
8620 AssertRC(rc2);
8621 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8622
8623 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8624 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8625 }
8626 else
8627 {
8628 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8629 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8630 }
8631 }
8632
8633#ifdef VBOX_STRICT
8634 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8635 hmR0VmxCheckHostEferMsr(pVCpu);
8636 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8637#endif
8638#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8639 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8640 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8641 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8642#endif
8643}
8644
8645
8646/**
8647 * Performs some essential restoration of state after running guest code in
8648 * VT-x.
8649 *
8650 * @param pVM The cross context VM structure.
8651 * @param pVCpu The cross context virtual CPU structure.
8652 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8653 * out-of-sync. Make sure to update the required fields
8654 * before using them.
8655 * @param pVmxTransient Pointer to the VMX transient structure.
8656 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8657 *
8658 * @remarks Called with interrupts disabled, and returns with interrups enabled!
8659 *
8660 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8661 * unconditionally when it is safe to do so.
8662 */
8663static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8664{
8665 NOREF(pVM);
8666
8667 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8668
8669 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8670 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8671 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8672 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8673 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8674 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8675
8676 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8677 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
8678
8679 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8680 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8681 Assert(!ASMIntAreEnabled());
8682 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8683
8684#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8685 if (CPUMIsGuestFPUStateActive(pVCpu))
8686 {
8687 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8688 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8689 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8690 }
8691#endif
8692
8693#if HC_ARCH_BITS == 64
8694 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8695#endif
8696 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8697#ifdef VBOX_STRICT
8698 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8699#endif
8700 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8701 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8702
8703 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8704 uint32_t uExitReason;
8705 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8706 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8707 AssertRC(rc);
8708 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8709 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8710
8711 /* Update the VM-exit history array. */
8712 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8713
8714 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8715 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8716 {
8717 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8718 pVmxTransient->fVMEntryFailed));
8719 return;
8720 }
8721
8722 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8723 {
8724 /** @todo We can optimize this by only syncing with our force-flags when
8725 * really needed and keeping the VMCS state as it is for most
8726 * VM-exits. */
8727 /* Update the guest interruptibility-state from the VMCS. */
8728 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8729
8730#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8731 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8732 AssertRC(rc);
8733#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8734 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8735 AssertRC(rc);
8736#endif
8737
8738 /*
8739 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8740 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8741 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8742 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8743 */
8744 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8745 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8746 {
8747 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8748 AssertRC(rc);
8749 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8750 }
8751 }
8752}
8753
8754
8755/**
8756 * Runs the guest code using VT-x the normal way.
8757 *
8758 * @returns VBox status code.
8759 * @param pVM The cross context VM structure.
8760 * @param pVCpu The cross context virtual CPU structure.
8761 * @param pCtx Pointer to the guest-CPU context.
8762 *
8763 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8764 */
8765static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8766{
8767 VMXTRANSIENT VmxTransient;
8768 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8769 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8770 uint32_t cLoops = 0;
8771
8772 for (;; cLoops++)
8773 {
8774 Assert(!HMR0SuspendPending());
8775 HMVMX_ASSERT_CPU_SAFE();
8776
8777 /* Preparatory work for running guest code, this may force us to return
8778 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8779 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8780 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8781 if (rcStrict != VINF_SUCCESS)
8782 break;
8783
8784 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8785 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8786 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8787
8788 /* Restore any residual host-state and save any bits shared between host
8789 and guest into the guest-CPU state. Re-enables interrupts! */
8790 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
8791
8792 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8793 if (RT_SUCCESS(rcRun))
8794 { /* very likely */ }
8795 else
8796 {
8797 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8798 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
8799 return rcRun;
8800 }
8801
8802 /* Profile the VM-exit. */
8803 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8804 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8805 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8806 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8807 HMVMX_START_EXIT_DISPATCH_PROF();
8808
8809 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8810
8811 /* Handle the VM-exit. */
8812#ifdef HMVMX_USE_FUNCTION_TABLE
8813 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8814#else
8815 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8816#endif
8817 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8818 if (rcStrict == VINF_SUCCESS)
8819 {
8820 if (cLoops <= pVM->hm.s.cMaxResumeLoops)
8821 continue; /* likely */
8822 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8823 rcStrict = VINF_EM_RAW_INTERRUPT;
8824 }
8825 break;
8826 }
8827
8828 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8829 return rcStrict;
8830}
8831
8832
8833
8834/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8835 *
8836 * The following few functions and associated structure contains the bloat
8837 * necessary for providing detailed debug events and dtrace probes as well as
8838 * reliable host side single stepping. This works on the principle of
8839 * "subclassing" the normal execution loop and workers. We replace the loop
8840 * method completely and override selected helpers to add necessary adjustments
8841 * to their core operation.
8842 *
8843 * The goal is to keep the "parent" code lean and mean, so as not to acrifice
8844 * any performance for debug and analysis features.
8845 *
8846 * @{
8847 */
8848
8849typedef struct VMXRUNDBGSTATE
8850{
8851 /** The RIP we started executing at. This is for detecting that we stepped. */
8852 uint64_t uRipStart;
8853 /** The CS we started exectuing with. */
8854 uint16_t uCsStart;
8855
8856 /** Whether we've actually modified the 1st execution control field. */
8857 bool fModifiedProcCtls : 1;
8858 /** Whether we've actually modified the 2nd execution control field. */
8859 bool fModifiedProcCtls2 : 1;
8860 /** Whether we've actually modified the exception bitmap. */
8861 bool fModifiedXcptBitmap : 1;
8862
8863 /** We desire the modified the CR0 mask to be cleared. */
8864 bool fClearCr0Mask : 1;
8865 /** We desire the modified the CR4 mask to be cleared. */
8866 bool fClearCr4Mask : 1;
8867 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
8868 uint32_t fCpe1Extra;
8869 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
8870 uint32_t fCpe1Unwanted;
8871 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
8872 uint32_t fCpe2Extra;
8873 /** Extra stuff we need in */
8874 uint32_t bmXcptExtra;
8875 /** The sequence number of the Dtrace provider settings the state was
8876 * configured against. */
8877 uint32_t uDtraceSettingsSeqNo;
8878 /** Exits to check (one bit per exit). */
8879 uint32_t bmExitsToCheck[3];
8880
8881 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
8882 uint32_t fProcCtlsInitial;
8883 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
8884 uint32_t fProcCtls2Initial;
8885 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
8886 uint32_t bmXcptInitial;
8887
8888} VMXRUNDBGSTATE;
8889AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
8890typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
8891
8892
8893/**
8894 * Initializes the VMXRUNDBGSTATE structure.
8895 *
8896 * @param pVCpu The cross context virtual CPU structure of the
8897 * calling EMT.
8898 * @param pCtx The CPU register context to go with @a pVCpu.
8899 * @param pDbgState The structure to initialize.
8900 */
8901DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
8902{
8903 pDbgState->uRipStart = pCtx->rip;
8904 pDbgState->uCsStart = pCtx->cs.Sel;
8905
8906 pDbgState->fModifiedProcCtls = false;
8907 pDbgState->fModifiedProcCtls2 = false;
8908 pDbgState->fModifiedXcptBitmap = false;
8909 pDbgState->fClearCr0Mask = false;
8910 pDbgState->fClearCr4Mask = false;
8911 pDbgState->fCpe1Extra = 0;
8912 pDbgState->fCpe1Unwanted = 0;
8913 pDbgState->fCpe2Extra = 0;
8914 pDbgState->bmXcptExtra = 0;
8915 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
8916 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
8917 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
8918}
8919
8920
8921/**
8922 * Updates the VMSC fields with changes requested by @a pDbgState.
8923 *
8924 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
8925 * immediately before executing guest code, i.e. when interrupts are disabled.
8926 * We don't check status codes here as we cannot easily assert or return in the
8927 * latter case.
8928 *
8929 * @param pVCpu The cross context virtual CPU structure.
8930 * @param pDbgState The debug state.
8931 */
8932DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
8933{
8934 /*
8935 * Ensure desired flags in VMCS control fields are set.
8936 * (Ignoring write failure here, as we're committed and it's just debug extras.)
8937 *
8938 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
8939 * there should be no stale data in pCtx at this point.
8940 */
8941 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
8942 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
8943 {
8944 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
8945 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
8946 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8947 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
8948 pDbgState->fModifiedProcCtls = true;
8949 }
8950
8951 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
8952 {
8953 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
8954 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
8955 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
8956 pDbgState->fModifiedProcCtls2 = true;
8957 }
8958
8959 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
8960 {
8961 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
8962 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8963 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
8964 pDbgState->fModifiedXcptBitmap = true;
8965 }
8966
8967 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
8968 {
8969 pVCpu->hm.s.vmx.u32CR0Mask = 0;
8970 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
8971 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
8972 }
8973
8974 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
8975 {
8976 pVCpu->hm.s.vmx.u32CR4Mask = 0;
8977 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
8978 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
8979 }
8980}
8981
8982
8983DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
8984{
8985 /*
8986 * Restore exit control settings as we may not reenter this function the
8987 * next time around.
8988 */
8989 /* We reload the initial value, trigger what we can of recalculations the
8990 next time around. From the looks of things, that's all that's required atm. */
8991 if (pDbgState->fModifiedProcCtls)
8992 {
8993 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
8994 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
8995 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
8996 AssertRCReturn(rc2, rc2);
8997 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
8998 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0 | HM_CHANGED_GUEST_DEBUG);
8999 }
9000
9001 /* We're currently the only ones messing with this one, so just restore the
9002 cached value and reload the field. */
9003 if ( pDbgState->fModifiedProcCtls2
9004 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9005 {
9006 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9007 AssertRCReturn(rc2, rc2);
9008 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9009 }
9010
9011 /* If we've modified the exception bitmap, we restore it and trigger
9012 reloading and partial recalculation the next time around. */
9013 if (pDbgState->fModifiedXcptBitmap)
9014 {
9015 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9016 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS | HM_CHANGED_GUEST_CR0);
9017 }
9018
9019 /* We assume hmR0VmxLoadSharedCR0 will recalculate and load the CR0 mask. */
9020 if (pDbgState->fClearCr0Mask)
9021 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9022
9023 /* We assume hmR0VmxLoadGuestCR3AndCR4 will recalculate and load the CR4 mask. */
9024 if (pDbgState->fClearCr4Mask)
9025 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9026
9027 return rcStrict;
9028}
9029
9030
9031/**
9032 * Configures exit controls for current DBGF and DTrace settings.
9033 *
9034 * This updates @a pDbgState and the VMCS execution control fields to reflect
9035 * the necessary exits demanded by DBGF and DTrace.
9036 *
9037 * @param pVM The cross context VM structure.
9038 * @param pVCpu The cross context virtual CPU structure.
9039 * @param pCtx Pointer to the guest-CPU context.
9040 * @param pDbgState The debug state.
9041 * @param pVmxTransient Pointer to the VMX transient structure. May update
9042 * fUpdateTscOffsettingAndPreemptTimer.
9043 */
9044static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9045 PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9046{
9047 /*
9048 * Take down the dtrace serial number so we can spot changes.
9049 */
9050 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9051 ASMCompilerBarrier();
9052
9053 /*
9054 * We'll rebuild most of the middle block of data members (holding the
9055 * current settings) as we go along here, so start by clearing it all.
9056 */
9057 pDbgState->bmXcptExtra = 0;
9058 pDbgState->fCpe1Extra = 0;
9059 pDbgState->fCpe1Unwanted = 0;
9060 pDbgState->fCpe2Extra = 0;
9061 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9062 pDbgState->bmExitsToCheck[i] = 0;
9063
9064 /*
9065 * Software interrupts (INT XXh) - no idea how to trigger these...
9066 */
9067 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9068 || VBOXVMM_INT_SOFTWARE_ENABLED())
9069 {
9070 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9071 }
9072
9073 /*
9074 * Exception bitmap and XCPT events+probes.
9075 */
9076 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9077 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9078 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9079
9080 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9081 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9082 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9083 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9084 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9085 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9086 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9087 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9088 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9089 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9090 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9091 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9092 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9093 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9094 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9095 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9096 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9097 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9098
9099 if (pDbgState->bmXcptExtra)
9100 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9101
9102 /*
9103 * Process events and probes for VM exits, making sure we get the wanted exits.
9104 *
9105 * Note! This is the reverse of waht hmR0VmxHandleExitDtraceEvents does.
9106 * So, when adding/changing/removing please don't forget to update it.
9107 *
9108 * Some of the macros are picking up local variables to save horizontal space,
9109 * (being able to see it in a table is the lesser evil here).
9110 */
9111#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9112 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9113 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9114#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9115 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9116 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9117 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9118 } else do { } while (0)
9119#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9120 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9121 { \
9122 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9123 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9124 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9125 } else do { } while (0)
9126#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9127 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9128 { \
9129 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9130 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9131 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9132 } else do { } while (0)
9133#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9134 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9135 { \
9136 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9137 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9138 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9139 } else do { } while (0)
9140
9141 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9142 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9143 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9144 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9145 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9146
9147 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9148 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9149 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9150 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9151 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9152 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9153 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9154 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9155 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9156 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9157 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9158 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9159 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9160 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9161 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9162 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9163 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9164 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9165 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9166 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9167 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9168 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9169 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9170 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9171 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9172 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9173 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9174 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9175 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9176 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9177 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9178 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9179 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9180 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9181 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9182 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9183
9184 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9185 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9186 {
9187 int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9188 rc2 |= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9189 rc2 |= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9190 AssertRC(rc2);
9191
9192#if 0 /** @todo fix me */
9193 pDbgState->fClearCr0Mask = true;
9194 pDbgState->fClearCr4Mask = true;
9195#endif
9196 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9197 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9198 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9199 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9200 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9201 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9202 require clearing here and in the loop if we start using it. */
9203 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9204 }
9205 else
9206 {
9207 if (pDbgState->fClearCr0Mask)
9208 {
9209 pDbgState->fClearCr0Mask = false;
9210 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9211 }
9212 if (pDbgState->fClearCr4Mask)
9213 {
9214 pDbgState->fClearCr4Mask = false;
9215 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9216 }
9217 }
9218 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9219 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9220
9221 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9222 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9223 {
9224 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9225 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9226 }
9227 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9228 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9229
9230 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9231 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9232 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9233 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9234 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* parnoia */
9235 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9236 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* parnoia */
9237 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9238#if 0 /** @todo too slow, fix handler. */
9239 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9240#endif
9241 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9242
9243 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9244 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9245 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9246 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9247 {
9248 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9249 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9250 }
9251 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9252 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9253 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9254 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9255
9256 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9257 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9258 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9259 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9260 {
9261 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9262 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9263 }
9264 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9265 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9266 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9267 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9268
9269 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9270 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9271 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9272 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9273 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9274 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9275 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9276 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9277 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9278 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9279 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9280 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9281 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9282 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9283 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9284 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9285 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9286 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9287 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9288 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9289 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9290 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9291
9292#undef IS_EITHER_ENABLED
9293#undef SET_ONLY_XBM_IF_EITHER_EN
9294#undef SET_CPE1_XBM_IF_EITHER_EN
9295#undef SET_CPEU_XBM_IF_EITHER_EN
9296#undef SET_CPE2_XBM_IF_EITHER_EN
9297
9298 /*
9299 * Sanitize the control stuff.
9300 */
9301 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9302 if (pDbgState->fCpe2Extra)
9303 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9304 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9305 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9306 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9307 {
9308 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9309 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9310 }
9311
9312 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9313 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9314 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9315 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9316}
9317
9318
9319/**
9320 * Fires off DBGF events and dtrace probes for an exit, when it's appropriate.
9321 *
9322 * The caller has checked exit against the VMXRUNDBGSTATE::bmExitsToCheck
9323 * bitmap. The caller has checked for NMIs already, so we don't have to do that
9324 * either.
9325 *
9326 * @returns Strict VBox status code (i.e. informational status codes too).
9327 * @param pVM The cross context VM structure.
9328 * @param pVCpu The cross context virtual CPU structure.
9329 * @param pMixedCtx Pointer to the guest-CPU context.
9330 * @param pVmxTransient Pointer to the VMX-transient structure.
9331 * @param uExitReason The VM-exit reason.
9332 *
9333 * @remarks The name of this function is displayed by dtrace, so keep it short
9334 * and to the point. No longer than 33 chars long, please.
9335 */
9336static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9337 PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9338{
9339 /*
9340 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9341 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9342 *
9343 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9344 * does. Must add/change/remove both places. Same ordering, please.
9345 *
9346 * Added/removed events must also be reflected in the next section
9347 * where we dispatch dtrace events.
9348 */
9349 bool fDtrace1 = false;
9350 bool fDtrace2 = false;
9351 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9352 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9353 uint32_t uEventArg = 0;
9354#define SET_EXIT(a_EventSubName) \
9355 do { \
9356 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9357 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9358 } while (0)
9359#define SET_BOTH(a_EventSubName) \
9360 do { \
9361 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9362 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9363 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9364 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9365 } while (0)
9366 switch (uExitReason)
9367 {
9368 case VMX_EXIT_MTF:
9369 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9370
9371 case VMX_EXIT_XCPT_OR_NMI:
9372 {
9373 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9374 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9375 {
9376 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9377 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9378 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9379 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9380 {
9381 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9382 {
9383 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9384 uEventArg = pVmxTransient->uExitIntErrorCode;
9385 }
9386 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9387 switch (enmEvent1)
9388 {
9389 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9390 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9391 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9392 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9393 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9394 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9395 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9396 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9397 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9398 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9399 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9400 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9401 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9402 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9403 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9404 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9405 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9406 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9407 default: break;
9408 }
9409 }
9410 else
9411 AssertFailed();
9412 break;
9413
9414 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9415 uEventArg = idxVector;
9416 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9417 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9418 break;
9419 }
9420 break;
9421 }
9422
9423 case VMX_EXIT_TRIPLE_FAULT:
9424 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9425 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9426 break;
9427 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9428 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9429 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9430 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9431 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9432
9433 /* Instruction specific VM-exits: */
9434 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9435 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9436 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9437 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9438 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9439 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9440 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9441 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9442 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9443 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9444 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9445 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9446 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9447 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9448 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9449 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9450 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9451 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9452 case VMX_EXIT_MOV_CRX:
9453 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9454/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
9455* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
9456 if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
9457 == VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
9458 SET_BOTH(CRX_READ);
9459 else
9460 SET_BOTH(CRX_WRITE);
9461 uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
9462 break;
9463 case VMX_EXIT_MOV_DRX:
9464 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9465 if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
9466 == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
9467 SET_BOTH(DRX_READ);
9468 else
9469 SET_BOTH(DRX_WRITE);
9470 uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
9471 break;
9472 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9473 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9474 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9475 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9476 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9477 case VMX_EXIT_XDTR_ACCESS:
9478 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9479 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9480 {
9481 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9482 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9483 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9484 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9485 }
9486 break;
9487
9488 case VMX_EXIT_TR_ACCESS:
9489 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9490 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9491 {
9492 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9493 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9494 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9495 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9496 }
9497 break;
9498
9499 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9500 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9501 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9502 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9503 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9504 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9505 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9506 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9507 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9508 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9509 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9510
9511 /* Events that aren't relevant at this point. */
9512 case VMX_EXIT_EXT_INT:
9513 case VMX_EXIT_INT_WINDOW:
9514 case VMX_EXIT_NMI_WINDOW:
9515 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9516 case VMX_EXIT_PREEMPT_TIMER:
9517 case VMX_EXIT_IO_INSTR:
9518 break;
9519
9520 /* Errors and unexpected events. */
9521 case VMX_EXIT_INIT_SIGNAL:
9522 case VMX_EXIT_SIPI:
9523 case VMX_EXIT_IO_SMI:
9524 case VMX_EXIT_SMI:
9525 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9526 case VMX_EXIT_ERR_MSR_LOAD:
9527 case VMX_EXIT_ERR_MACHINE_CHECK:
9528 break;
9529
9530 default:
9531 AssertMsgFailed(("Unexpected exit=%#x\n", uExitReason));
9532 break;
9533 }
9534#undef SET_BOTH
9535#undef SET_EXIT
9536
9537 /*
9538 * Dtrace tracepoints go first. We do them here at once so we don't
9539 * have to copy the guest state saving and stuff a few dozen times.
9540 * Down side is that we've got to repeat the switch, though this time
9541 * we use enmEvent since the probes are a subset of what DBGF does.
9542 */
9543 if (fDtrace1 || fDtrace2)
9544 {
9545 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9546 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9547 switch (enmEvent1)
9548 {
9549 /** @todo consider which extra parameters would be helpful for each probe. */
9550 case DBGFEVENT_END: break;
9551 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9552 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9553 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9554 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9555 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9556 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9557 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9558 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9559 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9560 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9561 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9562 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9563 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9564 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9565 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9566 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9567 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9568 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9569 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9570 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9571 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9572 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9573 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9574 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9575 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9576 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9577 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9578 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9579 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9580 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9581 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9582 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9583 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9584 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9585 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9586 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9587 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9588 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9589 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9590 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9591 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9592 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9593 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9594 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9595 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9596 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9597 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9598 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9599 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9600 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9601 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9602 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9603 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9604 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9605 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9606 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9607 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9608 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9609 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9610 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9611 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9612 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9613 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9614 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9615 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9616 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9617 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9618 }
9619 switch (enmEvent2)
9620 {
9621 /** @todo consider which extra parameters would be helpful for each probe. */
9622 case DBGFEVENT_END: break;
9623 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9624 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9625 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9626 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9627 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9628 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9629 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9630 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9631 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9632 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9633 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9634 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9635 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9636 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9637 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9638 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9639 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9640 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9641 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9642 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9643 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9644 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9645 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9646 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9647 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9648 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9649 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9650 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9651 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9652 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9653 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9654 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9655 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9656 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9657 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9658 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9659 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9660 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9661 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9662 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9663 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9664 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9665 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9666 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9667 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9668 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9669 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9670 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9671 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9672 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9673 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9674 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9675 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9676 }
9677 }
9678
9679 /*
9680 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9681 * the DBGF call will do a full check).
9682 *
9683 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9684 * Note! If we have to events, we prioritize the first, i.e. the instruction
9685 * one, in order to avoid event nesting.
9686 */
9687 if ( enmEvent1 != DBGFEVENT_END
9688 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9689 {
9690 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9691 if (rcStrict != VINF_SUCCESS)
9692 return rcStrict;
9693 }
9694 else if ( enmEvent2 != DBGFEVENT_END
9695 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9696 {
9697 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9698 if (rcStrict != VINF_SUCCESS)
9699 return rcStrict;
9700 }
9701
9702 return VINF_SUCCESS;
9703}
9704
9705
9706/**
9707 * Single-stepping VM-exit filtering.
9708 *
9709 * This is preprocessing the exits and deciding whether we've gotten far enough
9710 * to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit handling is
9711 * performed.
9712 *
9713 * @returns Strict VBox status code (i.e. informational status codes too).
9714 * @param pVM The cross context VM structure.
9715 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9716 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9717 * out-of-sync. Make sure to update the required
9718 * fields before using them.
9719 * @param pVmxTransient Pointer to the VMX-transient structure.
9720 * @param uExitReason The VM-exit reason.
9721 * @param pDbgState The debug state.
9722 */
9723DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9724 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9725{
9726 /*
9727 * Expensive (saves context) generic dtrace exit probe.
9728 */
9729 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9730 { /* more likely */ }
9731 else
9732 {
9733 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9734 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9735 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9736 }
9737
9738 /*
9739 * Check for host NMI, just to get that out of the way.
9740 */
9741 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9742 { /* normally likely */ }
9743 else
9744 {
9745 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9746 AssertRCReturn(rc2, rc2);
9747 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9748 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9749 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9750 }
9751
9752 /*
9753 * Check for single stepping event if we're stepping.
9754 */
9755 if (pVCpu->hm.s.fSingleInstruction)
9756 {
9757 switch (uExitReason)
9758 {
9759 case VMX_EXIT_MTF:
9760 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9761
9762 /* Various events: */
9763 case VMX_EXIT_XCPT_OR_NMI:
9764 case VMX_EXIT_EXT_INT:
9765 case VMX_EXIT_TRIPLE_FAULT:
9766 case VMX_EXIT_INT_WINDOW:
9767 case VMX_EXIT_NMI_WINDOW:
9768 case VMX_EXIT_TASK_SWITCH:
9769 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9770 case VMX_EXIT_APIC_ACCESS:
9771 case VMX_EXIT_EPT_VIOLATION:
9772 case VMX_EXIT_EPT_MISCONFIG:
9773 case VMX_EXIT_PREEMPT_TIMER:
9774
9775 /* Instruction specific VM-exits: */
9776 case VMX_EXIT_CPUID:
9777 case VMX_EXIT_GETSEC:
9778 case VMX_EXIT_HLT:
9779 case VMX_EXIT_INVD:
9780 case VMX_EXIT_INVLPG:
9781 case VMX_EXIT_RDPMC:
9782 case VMX_EXIT_RDTSC:
9783 case VMX_EXIT_RSM:
9784 case VMX_EXIT_VMCALL:
9785 case VMX_EXIT_VMCLEAR:
9786 case VMX_EXIT_VMLAUNCH:
9787 case VMX_EXIT_VMPTRLD:
9788 case VMX_EXIT_VMPTRST:
9789 case VMX_EXIT_VMREAD:
9790 case VMX_EXIT_VMRESUME:
9791 case VMX_EXIT_VMWRITE:
9792 case VMX_EXIT_VMXOFF:
9793 case VMX_EXIT_VMXON:
9794 case VMX_EXIT_MOV_CRX:
9795 case VMX_EXIT_MOV_DRX:
9796 case VMX_EXIT_IO_INSTR:
9797 case VMX_EXIT_RDMSR:
9798 case VMX_EXIT_WRMSR:
9799 case VMX_EXIT_MWAIT:
9800 case VMX_EXIT_MONITOR:
9801 case VMX_EXIT_PAUSE:
9802 case VMX_EXIT_XDTR_ACCESS:
9803 case VMX_EXIT_TR_ACCESS:
9804 case VMX_EXIT_INVEPT:
9805 case VMX_EXIT_RDTSCP:
9806 case VMX_EXIT_INVVPID:
9807 case VMX_EXIT_WBINVD:
9808 case VMX_EXIT_XSETBV:
9809 case VMX_EXIT_RDRAND:
9810 case VMX_EXIT_INVPCID:
9811 case VMX_EXIT_VMFUNC:
9812 case VMX_EXIT_RDSEED:
9813 case VMX_EXIT_XSAVES:
9814 case VMX_EXIT_XRSTORS:
9815 {
9816 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9817 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9818 AssertRCReturn(rc2, rc2);
9819 if ( pMixedCtx->rip != pDbgState->uRipStart
9820 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
9821 return VINF_EM_DBG_STEPPED;
9822 break;
9823 }
9824
9825 /* Errors and unexpected events: */
9826 case VMX_EXIT_INIT_SIGNAL:
9827 case VMX_EXIT_SIPI:
9828 case VMX_EXIT_IO_SMI:
9829 case VMX_EXIT_SMI:
9830 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9831 case VMX_EXIT_ERR_MSR_LOAD:
9832 case VMX_EXIT_ERR_MACHINE_CHECK:
9833 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9834 break;
9835
9836 default:
9837 AssertMsgFailed(("Unexpected exit=%#x\n", uExitReason));
9838 break;
9839 }
9840 }
9841
9842 /*
9843 * Check for debugger event breakpoints and dtrace probes.
9844 */
9845 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9846 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9847 {
9848 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9849 if (rcStrict != VINF_SUCCESS)
9850 return rcStrict;
9851 }
9852
9853 /*
9854 * Normal processing.
9855 */
9856#ifdef HMVMX_USE_FUNCTION_TABLE
9857 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9858#else
9859 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9860#endif
9861}
9862
9863
9864/**
9865 * Single steps guest code using VT-x.
9866 *
9867 * @returns Strict VBox status code (i.e. informational status codes too).
9868 * @param pVM The cross context VM structure.
9869 * @param pVCpu The cross context virtual CPU structure.
9870 * @param pCtx Pointer to the guest-CPU context.
9871 *
9872 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
9873 */
9874static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9875{
9876 VMXTRANSIENT VmxTransient;
9877 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9878
9879 /* Set HMCPU indicators. */
9880 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
9881 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
9882 pVCpu->hm.s.fDebugWantRdTscExit = false;
9883 pVCpu->hm.s.fUsingDebugLoop = true;
9884
9885 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
9886 VMXRUNDBGSTATE DbgState;
9887 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
9888 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
9889
9890 /*
9891 * The loop.
9892 */
9893 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9894 for (uint32_t cLoops = 0; ; cLoops++)
9895 {
9896 Assert(!HMR0SuspendPending());
9897 HMVMX_ASSERT_CPU_SAFE();
9898 bool fStepping = pVCpu->hm.s.fSingleInstruction;
9899
9900 /*
9901 * Preparatory work for running guest code, this may force us to return
9902 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
9903 */
9904 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9905 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
9906 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
9907 if (rcStrict != VINF_SUCCESS)
9908 break;
9909
9910 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9911 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
9912
9913 /*
9914 * Now we can run the guest code.
9915 */
9916 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9917
9918 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9919
9920 /*
9921 * Restore any residual host-state and save any bits shared between host
9922 * and guest into the guest-CPU state. Re-enables interrupts!
9923 */
9924 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
9925
9926 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9927 if (RT_SUCCESS(rcRun))
9928 { /* very likely */ }
9929 else
9930 {
9931 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9932 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9933 return rcRun;
9934 }
9935
9936 /* Profile the VM-exit. */
9937 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9938 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9939 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9940 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9941 HMVMX_START_EXIT_DISPATCH_PROF();
9942
9943 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9944
9945 /*
9946 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
9947 */
9948 rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
9949 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9950 if (rcStrict != VINF_SUCCESS)
9951 break;
9952 if (cLoops > pVM->hm.s.cMaxResumeLoops)
9953 {
9954 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9955 rcStrict = VINF_EM_RAW_INTERRUPT;
9956 break;
9957 }
9958
9959 /*
9960 * Stepping: Did the RIP change, if so, consider it a single step.
9961 * Otherwise, make sure one of the TFs gets set.
9962 */
9963 if (fStepping)
9964 {
9965 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
9966 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
9967 AssertRCReturn(rc2, rc2);
9968 if ( pCtx->rip != DbgState.uRipStart
9969 || pCtx->cs.Sel != DbgState.uCsStart)
9970 {
9971 rcStrict = VINF_EM_DBG_STEPPED;
9972 break;
9973 }
9974 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
9975 }
9976
9977 /*
9978 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
9979 */
9980 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
9981 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
9982 }
9983
9984 /*
9985 * Clear the X86_EFL_TF if necessary.
9986 */
9987 if (pVCpu->hm.s.fClearTrapFlag)
9988 {
9989 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
9990 AssertRCReturn(rc2, rc2);
9991 pVCpu->hm.s.fClearTrapFlag = false;
9992 pCtx->eflags.Bits.u1TF = 0;
9993 }
9994 /** @todo there seems to be issues with the resume flag when the monitor trap
9995 * flag is pending without being used. Seen early in bios init when
9996 * accessing APIC page in protected mode. */
9997
9998 /*
9999 * Restore exit control settings as we may not reenter this function the
10000 * next time around.
10001 */
10002 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10003
10004 /* Restore HMCPU indicators. */
10005 pVCpu->hm.s.fUsingDebugLoop = false;
10006 pVCpu->hm.s.fDebugWantRdTscExit = false;
10007 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10008
10009 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10010 return rcStrict;
10011}
10012
10013
10014/** @} */
10015
10016
10017/**
10018 * Checks if any expensive dtrace probes are enabled and we should go to the
10019 * debug loop.
10020 *
10021 * @returns true if we should use debug loop, false if not.
10022 */
10023static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10024{
10025 /* It's probably faster to OR the raw 32-bit counter variables together.
10026 Since the variables are in an array and the probes are next to one
10027 another (more or less), we have good locality. So, better read
10028 eight-nine cache lines ever time and only have one conditional, than
10029 128+ conditionals, right? */
10030 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10031 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10032 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10033 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10034 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10035 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10036 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10037 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10038 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10039 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10040 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10041 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10042 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10043 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10044 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10045 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10046 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10047 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10048 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10049 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10050 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10051 ) != 0
10052 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10053 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10054 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10055 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10056 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10057 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10058 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10059 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10060 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10061 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10062 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10063 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10064 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10065 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10066 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10067 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10068 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10069 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10070 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10071 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10072 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10073 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10074 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10075 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10076 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10077 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10078 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10079 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10080 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10081 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10082 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10083 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10084 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10085 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10086 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10087 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10088 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10089 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10090 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10091 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10092 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10093 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10094 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10095 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10096 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10097 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10098 ) != 0
10099 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10100 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10101 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10102 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10103 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10104 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10105 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10106 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10107 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10108 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10109 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10110 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10111 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10112 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10113 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10114 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10115 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10116 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10117 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10118 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10119 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10120 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10121 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10122 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10123 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10124 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10125 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10126 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10127 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10128 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10129 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10130 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10131 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10132 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10133 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10134 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10135 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10136 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10137 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10138 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10139 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10140 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10141 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10142 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10143 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10144 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10145 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10146 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10147 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10148 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10149 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10150 ) != 0;
10151}
10152
10153
10154/**
10155 * Runs the guest code using VT-x.
10156 *
10157 * @returns Strict VBox status code (i.e. informational status codes too).
10158 * @param pVM The cross context VM structure.
10159 * @param pVCpu The cross context virtual CPU structure.
10160 * @param pCtx Pointer to the guest-CPU context.
10161 */
10162VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10163{
10164 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10165 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10166 HMVMX_ASSERT_PREEMPT_SAFE();
10167
10168 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10169
10170 VBOXSTRICTRC rcStrict;
10171 if ( !pVCpu->hm.s.fUseDebugLoop
10172 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10173 && !DBGFIsStepping(pVCpu) )
10174 rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10175 else
10176 rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10177
10178 if (rcStrict == VERR_EM_INTERPRETER)
10179 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10180 else if (rcStrict == VINF_EM_RESET)
10181 rcStrict = VINF_EM_TRIPLE_FAULT;
10182
10183 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10184 if (RT_FAILURE(rc2))
10185 {
10186 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10187 rcStrict = rc2;
10188 }
10189 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10190 return rcStrict;
10191}
10192
10193
10194#ifndef HMVMX_USE_FUNCTION_TABLE
10195DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10196{
10197# ifdef DEBUG_ramshankar
10198# define RETURN_EXIT_CALL(a_CallExpr) \
10199 do { \
10200 /* int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); */ \
10201 VBOXSTRICTRC rcStrict = a_CallExpr; \
10202 /* HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); */ \
10203 return rcStrict; \
10204 } while (0)
10205# else
10206# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10207# endif
10208 switch (rcReason)
10209 {
10210 case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10211 case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10212 case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10213 case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10214 case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10215 case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10216 case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10217 case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10218 case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10219 case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10220 case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10221 case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10222 case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10223 case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10224 case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10225 case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10226 case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10227 case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10228 case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10229 case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10230 case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10231 case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10232 case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10233 case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10234 case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10235 case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10236 case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10237 case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10238 case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10239 case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10240 case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10241 case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10242 case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10243 case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10244
10245 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10246 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10247 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10248 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10249 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10250 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10251 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10252 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10253 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10254
10255 case VMX_EXIT_VMCLEAR:
10256 case VMX_EXIT_VMLAUNCH:
10257 case VMX_EXIT_VMPTRLD:
10258 case VMX_EXIT_VMPTRST:
10259 case VMX_EXIT_VMREAD:
10260 case VMX_EXIT_VMRESUME:
10261 case VMX_EXIT_VMWRITE:
10262 case VMX_EXIT_VMXOFF:
10263 case VMX_EXIT_VMXON:
10264 case VMX_EXIT_INVEPT:
10265 case VMX_EXIT_INVVPID:
10266 case VMX_EXIT_VMFUNC:
10267 case VMX_EXIT_XSAVES:
10268 case VMX_EXIT_XRSTORS:
10269 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10270 case VMX_EXIT_RESERVED_60:
10271 case VMX_EXIT_RDSEED: /* only spurious exits, so undefined */
10272 case VMX_EXIT_RESERVED_62:
10273 default:
10274 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10275 }
10276}
10277#endif /* !HMVMX_USE_FUNCTION_TABLE */
10278
10279
10280#ifdef VBOX_STRICT
10281/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10282# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10283 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10284
10285# define HMVMX_ASSERT_PREEMPT_CPUID() \
10286 do { \
10287 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10288 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10289 } while (0)
10290
10291# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10292 do { \
10293 AssertPtr(pVCpu); \
10294 AssertPtr(pMixedCtx); \
10295 AssertPtr(pVmxTransient); \
10296 Assert(pVmxTransient->fVMEntryFailed == false); \
10297 Assert(ASMIntAreEnabled()); \
10298 HMVMX_ASSERT_PREEMPT_SAFE(); \
10299 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10300 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10301 HMVMX_ASSERT_PREEMPT_SAFE(); \
10302 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10303 HMVMX_ASSERT_PREEMPT_CPUID(); \
10304 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10305 } while (0)
10306
10307# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10308 do { \
10309 Log4Func(("\n")); \
10310 } while (0)
10311#else /* nonstrict builds: */
10312# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10313 do { \
10314 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10315 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10316 } while (0)
10317# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10318#endif
10319
10320
10321/**
10322 * Advances the guest RIP after reading it from the VMCS.
10323 *
10324 * @returns VBox status code, no informational status codes.
10325 * @param pVCpu The cross context virtual CPU structure.
10326 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10327 * out-of-sync. Make sure to update the required fields
10328 * before using them.
10329 * @param pVmxTransient Pointer to the VMX transient structure.
10330 *
10331 * @remarks No-long-jump zone!!!
10332 */
10333DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10334{
10335 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10336 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10337 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10338 AssertRCReturn(rc, rc);
10339
10340 pMixedCtx->rip += pVmxTransient->cbInstr;
10341 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10342
10343 /*
10344 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10345 * pending debug exception field as it takes care of priority of events.
10346 *
10347 * See Intel spec. 32.2.1 "Debug Exceptions".
10348 */
10349 if ( !pVCpu->hm.s.fSingleInstruction
10350 && pMixedCtx->eflags.Bits.u1TF)
10351 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10352
10353 return VINF_SUCCESS;
10354}
10355
10356
10357/**
10358 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10359 * and update error record fields accordingly.
10360 *
10361 * @return VMX_IGS_* return codes.
10362 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10363 * wrong with the guest state.
10364 *
10365 * @param pVM The cross context VM structure.
10366 * @param pVCpu The cross context virtual CPU structure.
10367 * @param pCtx Pointer to the guest-CPU state.
10368 *
10369 * @remarks This function assumes our cache of the VMCS controls
10370 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10371 */
10372static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10373{
10374#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10375#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10376 uError = (err); \
10377 break; \
10378 } else do { } while (0)
10379
10380 int rc;
10381 uint32_t uError = VMX_IGS_ERROR;
10382 uint32_t u32Val;
10383 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10384
10385 do
10386 {
10387 /*
10388 * CR0.
10389 */
10390 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10391 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10392 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10393 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10394 if (fUnrestrictedGuest)
10395 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
10396
10397 uint32_t u32GuestCR0;
10398 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
10399 AssertRCBreak(rc);
10400 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
10401 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
10402 if ( !fUnrestrictedGuest
10403 && (u32GuestCR0 & X86_CR0_PG)
10404 && !(u32GuestCR0 & X86_CR0_PE))
10405 {
10406 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10407 }
10408
10409 /*
10410 * CR4.
10411 */
10412 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10413 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10414
10415 uint32_t u32GuestCR4;
10416 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
10417 AssertRCBreak(rc);
10418 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
10419 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
10420
10421 /*
10422 * IA32_DEBUGCTL MSR.
10423 */
10424 uint64_t u64Val;
10425 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10426 AssertRCBreak(rc);
10427 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10428 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10429 {
10430 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10431 }
10432 uint64_t u64DebugCtlMsr = u64Val;
10433
10434#ifdef VBOX_STRICT
10435 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10436 AssertRCBreak(rc);
10437 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10438#endif
10439 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10440
10441 /*
10442 * RIP and RFLAGS.
10443 */
10444 uint32_t u32Eflags;
10445#if HC_ARCH_BITS == 64
10446 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10447 AssertRCBreak(rc);
10448 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10449 if ( !fLongModeGuest
10450 || !pCtx->cs.Attr.n.u1Long)
10451 {
10452 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10453 }
10454 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10455 * must be identical if the "IA-32e mode guest" VM-entry
10456 * control is 1 and CS.L is 1. No check applies if the
10457 * CPU supports 64 linear-address bits. */
10458
10459 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10460 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10461 AssertRCBreak(rc);
10462 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10463 VMX_IGS_RFLAGS_RESERVED);
10464 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10465 u32Eflags = u64Val;
10466#else
10467 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10468 AssertRCBreak(rc);
10469 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10470 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10471#endif
10472
10473 if ( fLongModeGuest
10474 || ( fUnrestrictedGuest
10475 && !(u32GuestCR0 & X86_CR0_PE)))
10476 {
10477 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10478 }
10479
10480 uint32_t u32EntryInfo;
10481 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10482 AssertRCBreak(rc);
10483 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10484 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10485 {
10486 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10487 }
10488
10489 /*
10490 * 64-bit checks.
10491 */
10492#if HC_ARCH_BITS == 64
10493 if (fLongModeGuest)
10494 {
10495 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10496 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10497 }
10498
10499 if ( !fLongModeGuest
10500 && (u32GuestCR4 & X86_CR4_PCIDE))
10501 {
10502 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10503 }
10504
10505 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10506 * 51:32 beyond the processor's physical-address width are 0. */
10507
10508 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10509 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10510 {
10511 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10512 }
10513
10514 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10515 AssertRCBreak(rc);
10516 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10517
10518 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10519 AssertRCBreak(rc);
10520 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10521#endif
10522
10523 /*
10524 * PERF_GLOBAL MSR.
10525 */
10526 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10527 {
10528 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10529 AssertRCBreak(rc);
10530 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10531 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10532 }
10533
10534 /*
10535 * PAT MSR.
10536 */
10537 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10538 {
10539 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10540 AssertRCBreak(rc);
10541 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10542 for (unsigned i = 0; i < 8; i++)
10543 {
10544 uint8_t u8Val = (u64Val & 0xff);
10545 if ( u8Val != 0 /* UC */
10546 && u8Val != 1 /* WC */
10547 && u8Val != 4 /* WT */
10548 && u8Val != 5 /* WP */
10549 && u8Val != 6 /* WB */
10550 && u8Val != 7 /* UC- */)
10551 {
10552 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10553 }
10554 u64Val >>= 8;
10555 }
10556 }
10557
10558 /*
10559 * EFER MSR.
10560 */
10561 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10562 {
10563 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10564 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10565 AssertRCBreak(rc);
10566 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10567 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10568 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10569 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10570 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10571 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10572 || !(u32GuestCR0 & X86_CR0_PG)
10573 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10574 VMX_IGS_EFER_LMA_LME_MISMATCH);
10575 }
10576
10577 /*
10578 * Segment registers.
10579 */
10580 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10581 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10582 if (!(u32Eflags & X86_EFL_VM))
10583 {
10584 /* CS */
10585 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10586 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10587 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10588 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10589 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10590 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10591 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10592 /* CS cannot be loaded with NULL in protected mode. */
10593 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10594 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10595 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10596 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10597 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10598 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10599 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10600 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10601 else
10602 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10603
10604 /* SS */
10605 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10606 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10607 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10608 if ( !(pCtx->cr0 & X86_CR0_PE)
10609 || pCtx->cs.Attr.n.u4Type == 3)
10610 {
10611 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10612 }
10613 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10614 {
10615 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10616 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10617 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10618 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10619 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10620 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10621 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10622 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10623 }
10624
10625 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
10626 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10627 {
10628 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10629 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10630 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10631 || pCtx->ds.Attr.n.u4Type > 11
10632 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10633 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10634 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10635 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10636 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10637 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10638 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10639 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10640 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10641 }
10642 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10643 {
10644 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10645 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10646 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10647 || pCtx->es.Attr.n.u4Type > 11
10648 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10649 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10650 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10651 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10652 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10653 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10654 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10655 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10656 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10657 }
10658 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10659 {
10660 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10661 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10662 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10663 || pCtx->fs.Attr.n.u4Type > 11
10664 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10665 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10666 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10667 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10668 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10669 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10670 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10671 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10672 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10673 }
10674 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10675 {
10676 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10677 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10678 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10679 || pCtx->gs.Attr.n.u4Type > 11
10680 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10681 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10682 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10683 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10684 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10685 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10686 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10687 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10688 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10689 }
10690 /* 64-bit capable CPUs. */
10691#if HC_ARCH_BITS == 64
10692 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10693 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10694 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10695 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10696 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10697 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10698 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10699 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10700 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10701 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10702 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10703#endif
10704 }
10705 else
10706 {
10707 /* V86 mode checks. */
10708 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10709 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10710 {
10711 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10712 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10713 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10714 }
10715 else
10716 {
10717 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10718 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10719 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10720 }
10721
10722 /* CS */
10723 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10724 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10725 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10726 /* SS */
10727 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10728 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10729 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10730 /* DS */
10731 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10732 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10733 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10734 /* ES */
10735 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10736 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10737 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10738 /* FS */
10739 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10740 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10741 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10742 /* GS */
10743 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10744 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10745 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10746 /* 64-bit capable CPUs. */
10747#if HC_ARCH_BITS == 64
10748 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10749 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10750 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10751 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10752 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10753 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10754 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10755 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10756 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10757 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10758 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10759#endif
10760 }
10761
10762 /*
10763 * TR.
10764 */
10765 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10766 /* 64-bit capable CPUs. */
10767#if HC_ARCH_BITS == 64
10768 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10769#endif
10770 if (fLongModeGuest)
10771 {
10772 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10773 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10774 }
10775 else
10776 {
10777 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10778 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10779 VMX_IGS_TR_ATTR_TYPE_INVALID);
10780 }
10781 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10782 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10783 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10784 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10785 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10786 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10787 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10788 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10789
10790 /*
10791 * GDTR and IDTR.
10792 */
10793#if HC_ARCH_BITS == 64
10794 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10795 AssertRCBreak(rc);
10796 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10797
10798 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10799 AssertRCBreak(rc);
10800 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10801#endif
10802
10803 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10804 AssertRCBreak(rc);
10805 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10806
10807 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10808 AssertRCBreak(rc);
10809 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10810
10811 /*
10812 * Guest Non-Register State.
10813 */
10814 /* Activity State. */
10815 uint32_t u32ActivityState;
10816 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10817 AssertRCBreak(rc);
10818 HMVMX_CHECK_BREAK( !u32ActivityState
10819 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
10820 VMX_IGS_ACTIVITY_STATE_INVALID);
10821 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10822 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10823 uint32_t u32IntrState;
10824 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
10825 AssertRCBreak(rc);
10826 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
10827 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10828 {
10829 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10830 }
10831
10832 /** @todo Activity state and injecting interrupts. Left as a todo since we
10833 * currently don't use activity states but ACTIVE. */
10834
10835 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10836 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10837
10838 /* Guest interruptibility-state. */
10839 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10840 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10841 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
10842 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10843 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10844 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10845 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10846 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10847 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10848 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
10849 {
10850 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10851 {
10852 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10853 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10854 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10855 }
10856 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10857 {
10858 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10859 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10860 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10861 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10862 }
10863 }
10864 /** @todo Assumes the processor is not in SMM. */
10865 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10866 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10867 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10868 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10869 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10870 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
10871 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10872 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10873 {
10874 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
10875 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10876 }
10877
10878 /* Pending debug exceptions. */
10879#if HC_ARCH_BITS == 64
10880 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
10881 AssertRCBreak(rc);
10882 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10883 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10884 u32Val = u64Val; /* For pending debug exceptions checks below. */
10885#else
10886 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
10887 AssertRCBreak(rc);
10888 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10889 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10890#endif
10891
10892 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10893 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
10894 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10895 {
10896 if ( (u32Eflags & X86_EFL_TF)
10897 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10898 {
10899 /* Bit 14 is PendingDebug.BS. */
10900 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10901 }
10902 if ( !(u32Eflags & X86_EFL_TF)
10903 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10904 {
10905 /* Bit 14 is PendingDebug.BS. */
10906 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10907 }
10908 }
10909
10910 /* VMCS link pointer. */
10911 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10912 AssertRCBreak(rc);
10913 if (u64Val != UINT64_C(0xffffffffffffffff))
10914 {
10915 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10916 /** @todo Bits beyond the processor's physical-address width MBZ. */
10917 /** @todo 32-bit located in memory referenced by value of this field (as a
10918 * physical address) must contain the processor's VMCS revision ID. */
10919 /** @todo SMM checks. */
10920 }
10921
10922 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10923 * not using Nested Paging? */
10924 if ( pVM->hm.s.fNestedPaging
10925 && !fLongModeGuest
10926 && CPUMIsGuestInPAEModeEx(pCtx))
10927 {
10928 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10929 AssertRCBreak(rc);
10930 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10931
10932 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10933 AssertRCBreak(rc);
10934 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10935
10936 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10937 AssertRCBreak(rc);
10938 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10939
10940 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10941 AssertRCBreak(rc);
10942 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10943 }
10944
10945 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10946 if (uError == VMX_IGS_ERROR)
10947 uError = VMX_IGS_REASON_NOT_FOUND;
10948 } while (0);
10949
10950 pVCpu->hm.s.u32HMError = uError;
10951 return uError;
10952
10953#undef HMVMX_ERROR_BREAK
10954#undef HMVMX_CHECK_BREAK
10955}
10956
10957/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10958/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
10959/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10960
10961/** @name VM-exit handlers.
10962 * @{
10963 */
10964
10965/**
10966 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
10967 */
10968HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10969{
10970 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10971 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
10972 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
10973 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
10974 return VINF_SUCCESS;
10975 return VINF_EM_RAW_INTERRUPT;
10976}
10977
10978
10979/**
10980 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
10981 */
10982HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10983{
10984 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10985 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
10986
10987 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10988 AssertRCReturn(rc, rc);
10989
10990 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10991 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
10992 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
10993 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
10994
10995 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10996 {
10997 /*
10998 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
10999 * anything we inject is not going to cause a VM-exit directly for the event being injected.
11000 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11001 *
11002 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11003 */
11004 VMXDispatchHostNmi();
11005 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11006 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11007 return VINF_SUCCESS;
11008 }
11009
11010 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11011 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11012 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11013 { /* likely */ }
11014 else
11015 {
11016 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11017 rcStrictRc1 = VINF_SUCCESS;
11018 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11019 return rcStrictRc1;
11020 }
11021
11022 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11023 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11024 switch (uIntType)
11025 {
11026 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11027 Assert(uVector == X86_XCPT_DB);
11028 /* no break */
11029 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11030 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11031 /* no break */
11032 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11033 {
11034 switch (uVector)
11035 {
11036 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11037 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11038 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11039 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11040 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11041 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11042 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11043
11044 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11045 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11046 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11047 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11048 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11049 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11050 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11051 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11052 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11053 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11054 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11055 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11056 default:
11057 {
11058 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11059 AssertRCReturn(rc, rc);
11060
11061 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11062 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11063 {
11064 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11065 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11066 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11067
11068 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11069 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11070 AssertRCReturn(rc, rc);
11071 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11072 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11073 0 /* GCPtrFaultAddress */);
11074 AssertRCReturn(rc, rc);
11075 }
11076 else
11077 {
11078 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11079 pVCpu->hm.s.u32HMError = uVector;
11080 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11081 }
11082 break;
11083 }
11084 }
11085 break;
11086 }
11087
11088 default:
11089 {
11090 pVCpu->hm.s.u32HMError = uExitIntInfo;
11091 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11092 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11093 break;
11094 }
11095 }
11096 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11097 return rc;
11098}
11099
11100
11101/**
11102 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11103 */
11104HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11105{
11106 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11107
11108 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11109 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11110
11111 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11112 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11113 return VINF_SUCCESS;
11114}
11115
11116
11117/**
11118 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11119 */
11120HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11121{
11122 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11123 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11124 {
11125 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11126 HMVMX_RETURN_UNEXPECTED_EXIT();
11127 }
11128
11129 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11130
11131 /*
11132 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11133 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11134 */
11135 uint32_t uIntrState = 0;
11136 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11137 AssertRCReturn(rc, rc);
11138
11139 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11140 if ( fBlockSti
11141 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11142 {
11143 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11144 }
11145
11146 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11147 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11148
11149 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11150 return VINF_SUCCESS;
11151}
11152
11153
11154/**
11155 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11156 */
11157HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11158{
11159 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11160 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11161 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11162}
11163
11164
11165/**
11166 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11167 */
11168HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11169{
11170 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11171 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11172 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11173}
11174
11175
11176/**
11177 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11178 */
11179HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11180{
11181 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11182 PVM pVM = pVCpu->CTX_SUFF(pVM);
11183 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11184 if (RT_LIKELY(rc == VINF_SUCCESS))
11185 {
11186 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11187 Assert(pVmxTransient->cbInstr == 2);
11188 }
11189 else
11190 {
11191 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11192 rc = VERR_EM_INTERPRETER;
11193 }
11194 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11195 return rc;
11196}
11197
11198
11199/**
11200 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11201 */
11202HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11203{
11204 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11205 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11206 AssertRCReturn(rc, rc);
11207
11208 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11209 return VINF_EM_RAW_EMULATE_INSTR;
11210
11211 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11212 HMVMX_RETURN_UNEXPECTED_EXIT();
11213}
11214
11215
11216/**
11217 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11218 */
11219HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11220{
11221 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11222 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
11223 AssertRCReturn(rc, rc);
11224
11225 PVM pVM = pVCpu->CTX_SUFF(pVM);
11226 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11227 if (RT_LIKELY(rc == VINF_SUCCESS))
11228 {
11229 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11230 Assert(pVmxTransient->cbInstr == 2);
11231 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11232 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11233 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11234 }
11235 else
11236 rc = VERR_EM_INTERPRETER;
11237 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11238 return rc;
11239}
11240
11241
11242/**
11243 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11244 */
11245HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11246{
11247 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11248 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
11249 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11250 AssertRCReturn(rc, rc);
11251
11252 PVM pVM = pVCpu->CTX_SUFF(pVM);
11253 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11254 if (RT_SUCCESS(rc))
11255 {
11256 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11257 Assert(pVmxTransient->cbInstr == 3);
11258 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11259 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11260 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11261 }
11262 else
11263 {
11264 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11265 rc = VERR_EM_INTERPRETER;
11266 }
11267 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11268 return rc;
11269}
11270
11271
11272/**
11273 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11274 */
11275HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11276{
11277 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11278 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
11279 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
11280 AssertRCReturn(rc, rc);
11281
11282 PVM pVM = pVCpu->CTX_SUFF(pVM);
11283 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11284 if (RT_LIKELY(rc == VINF_SUCCESS))
11285 {
11286 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11287 Assert(pVmxTransient->cbInstr == 2);
11288 }
11289 else
11290 {
11291 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11292 rc = VERR_EM_INTERPRETER;
11293 }
11294 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11295 return rc;
11296}
11297
11298
11299/**
11300 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11301 */
11302HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11303{
11304 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11305 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11306
11307 if (pVCpu->hm.s.fHypercallsEnabled)
11308 {
11309#if 0
11310 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11311 AssertRCReturn(rc, rc);
11312#else
11313 /* Aggressive state sync. for now. */
11314 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11315 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11316#endif
11317 rc |= hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11318 AssertRCReturn(rc, rc);
11319
11320 /** @todo pre-increment RIP before hypercall will break when we have to implement
11321 * continuing hypercalls (e.g. Hyper-V). */
11322 /** @todo r=bird: GIMHypercall will probably have to be able to return
11323 * informational status codes, so it should be made VBOXSTRICTRC. Not
11324 * doing that now because the status code handling isn't clean (i.e.
11325 * if you use RT_SUCCESS(rc) on the result of something, you don't
11326 * return rc in the success case, you return VINF_SUCCESS). */
11327 rc = GIMHypercall(pVCpu, pMixedCtx);
11328 /* If the hypercall changes anything other than guest general-purpose registers,
11329 we would need to reload the guest changed bits here before VM-entry. */
11330 return rc;
11331 }
11332
11333 Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11334 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11335 return VINF_SUCCESS;
11336}
11337
11338
11339/**
11340 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11341 */
11342HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11343{
11344 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11345 PVM pVM = pVCpu->CTX_SUFF(pVM);
11346 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11347
11348 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11349 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11350 AssertRCReturn(rc, rc);
11351
11352 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11353 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11354 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11355 else
11356 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11357 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11358 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
11359 return rcStrict;
11360}
11361
11362
11363/**
11364 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11365 */
11366HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11367{
11368 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11369 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11370 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11371 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11372 AssertRCReturn(rc, rc);
11373
11374 PVM pVM = pVCpu->CTX_SUFF(pVM);
11375 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11376 if (RT_LIKELY(rc == VINF_SUCCESS))
11377 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11378 else
11379 {
11380 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11381 rc = VERR_EM_INTERPRETER;
11382 }
11383 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11384 return rc;
11385}
11386
11387
11388/**
11389 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11390 */
11391HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11392{
11393 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11394 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11395 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11396 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11397 AssertRCReturn(rc, rc);
11398
11399 PVM pVM = pVCpu->CTX_SUFF(pVM);
11400 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11401 rc = VBOXSTRICTRC_VAL(rc2);
11402 if (RT_LIKELY( rc == VINF_SUCCESS
11403 || rc == VINF_EM_HALT))
11404 {
11405 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11406 AssertRCReturn(rc3, rc3);
11407
11408 if ( rc == VINF_EM_HALT
11409 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11410 {
11411 rc = VINF_SUCCESS;
11412 }
11413 }
11414 else
11415 {
11416 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11417 rc = VERR_EM_INTERPRETER;
11418 }
11419 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11420 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11421 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11422 return rc;
11423}
11424
11425
11426/**
11427 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11428 */
11429HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11430{
11431 /*
11432 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
11433 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
11434 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
11435 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
11436 */
11437 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11438 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11439 HMVMX_RETURN_UNEXPECTED_EXIT();
11440}
11441
11442
11443/**
11444 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11445 */
11446HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11447{
11448 /*
11449 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
11450 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
11451 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
11452 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
11453 */
11454 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11455 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11456 HMVMX_RETURN_UNEXPECTED_EXIT();
11457}
11458
11459
11460/**
11461 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11462 */
11463HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11464{
11465 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11466 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11467 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11468 HMVMX_RETURN_UNEXPECTED_EXIT();
11469}
11470
11471
11472/**
11473 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11474 */
11475HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11476{
11477 /*
11478 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
11479 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
11480 * See Intel spec. 25.3 "Other Causes of VM-exits".
11481 */
11482 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11483 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11484 HMVMX_RETURN_UNEXPECTED_EXIT();
11485}
11486
11487
11488/**
11489 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11490 * VM-exit.
11491 */
11492HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11493{
11494 /*
11495 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11496 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11497 *
11498 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11499 * See Intel spec. "23.8 Restrictions on VMX operation".
11500 */
11501 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11502 return VINF_SUCCESS;
11503}
11504
11505
11506/**
11507 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11508 * VM-exit.
11509 */
11510HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11511{
11512 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11513 return VINF_EM_RESET;
11514}
11515
11516
11517/**
11518 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11519 */
11520HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11521{
11522 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11523 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11524 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11525 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11526 AssertRCReturn(rc, rc);
11527
11528 pMixedCtx->rip++;
11529 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11530 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11531 rc = VINF_SUCCESS;
11532 else
11533 rc = VINF_EM_HALT;
11534
11535 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11536 if (rc != VINF_SUCCESS)
11537 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11538 return rc;
11539}
11540
11541
11542/**
11543 * VM-exit handler for instructions that result in a \#UD exception delivered to
11544 * the guest.
11545 */
11546HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11547{
11548 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11549 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11550 return VINF_SUCCESS;
11551}
11552
11553
11554/**
11555 * VM-exit handler for expiry of the VMX preemption timer.
11556 */
11557HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11558{
11559 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11560
11561 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11562 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11563
11564 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11565 PVM pVM = pVCpu->CTX_SUFF(pVM);
11566 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11567 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11568 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11569}
11570
11571
11572/**
11573 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11574 */
11575HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11576{
11577 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11578
11579 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11580 rc |= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, false /*fNeedRsp*/);
11581 rc |= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11582 AssertRCReturn(rc, rc);
11583
11584 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11585 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
11586
11587 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11588
11589 return rcStrict;
11590}
11591
11592
11593/**
11594 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11595 */
11596HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11597{
11598 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11599
11600 /* The guest should not invalidate the host CPU's TLBs, fallback to interpreter. */
11601 /** @todo implement EMInterpretInvpcid() */
11602 return VERR_EM_INTERPRETER;
11603}
11604
11605
11606/**
11607 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11608 * Error VM-exit.
11609 */
11610HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11611{
11612 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11613 AssertRCReturn(rc, rc);
11614
11615 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11616 AssertRCReturn(rc, rc);
11617
11618 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11619 NOREF(uInvalidReason);
11620
11621#ifdef VBOX_STRICT
11622 uint32_t uIntrState;
11623 RTHCUINTREG uHCReg;
11624 uint64_t u64Val;
11625 uint32_t u32Val;
11626
11627 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11628 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11629 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11630 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11631 AssertRCReturn(rc, rc);
11632
11633 Log4(("uInvalidReason %u\n", uInvalidReason));
11634 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11635 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11636 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11637 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
11638
11639 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11640 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11641 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11642 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11643 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11644 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11645 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11646 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11647 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11648 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11649 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11650 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11651#else
11652 NOREF(pVmxTransient);
11653#endif
11654
11655 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11656 return VERR_VMX_INVALID_GUEST_STATE;
11657}
11658
11659
11660/**
11661 * VM-exit handler for VM-entry failure due to an MSR-load
11662 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11663 */
11664HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11665{
11666 NOREF(pVmxTransient);
11667 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11668 HMVMX_RETURN_UNEXPECTED_EXIT();
11669}
11670
11671
11672/**
11673 * VM-exit handler for VM-entry failure due to a machine-check event
11674 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11675 */
11676HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11677{
11678 NOREF(pVmxTransient);
11679 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11680 HMVMX_RETURN_UNEXPECTED_EXIT();
11681}
11682
11683
11684/**
11685 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11686 * theory.
11687 */
11688HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11689{
11690 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
11691 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
11692 return VERR_VMX_UNDEFINED_EXIT_CODE;
11693}
11694
11695
11696/**
11697 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11698 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11699 * Conditional VM-exit.
11700 */
11701HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11702{
11703 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11704
11705 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
11706 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11707 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
11708 return VERR_EM_INTERPRETER;
11709 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11710 HMVMX_RETURN_UNEXPECTED_EXIT();
11711}
11712
11713
11714/**
11715 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11716 */
11717HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11718{
11719 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11720
11721 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
11722 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
11723 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
11724 return VERR_EM_INTERPRETER;
11725 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11726 HMVMX_RETURN_UNEXPECTED_EXIT();
11727}
11728
11729
11730/**
11731 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11732 */
11733HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11734{
11735 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11736
11737 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
11738 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11739 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11740 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11741 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11742 {
11743 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
11744 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
11745 }
11746 AssertRCReturn(rc, rc);
11747 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
11748
11749#ifdef VBOX_STRICT
11750 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
11751 {
11752 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
11753 && pMixedCtx->ecx != MSR_K6_EFER)
11754 {
11755 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11756 pMixedCtx->ecx));
11757 HMVMX_RETURN_UNEXPECTED_EXIT();
11758 }
11759# if HC_ARCH_BITS == 64
11760 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
11761 && hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11762 {
11763 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11764 HMVMX_RETURN_UNEXPECTED_EXIT();
11765 }
11766# endif
11767 }
11768#endif
11769
11770 PVM pVM = pVCpu->CTX_SUFF(pVM);
11771 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11772 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
11773 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
11774 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11775 if (RT_SUCCESS(rc))
11776 {
11777 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11778 Assert(pVmxTransient->cbInstr == 2);
11779 }
11780 return rc;
11781}
11782
11783
11784/**
11785 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11786 */
11787HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11788{
11789 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11790 PVM pVM = pVCpu->CTX_SUFF(pVM);
11791 int rc = VINF_SUCCESS;
11792
11793 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
11794 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11795 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11796 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11797 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11798 {
11799 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
11800 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
11801 }
11802 AssertRCReturn(rc, rc);
11803 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
11804
11805 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11806 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
11807 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
11808
11809 if (RT_SUCCESS(rc))
11810 {
11811 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11812
11813 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
11814 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
11815 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
11816 {
11817 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
11818 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
11819 EMInterpretWrmsr() changes it. */
11820 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
11821 }
11822 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
11823 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11824 else if (pMixedCtx->ecx == MSR_K6_EFER)
11825 {
11826 /*
11827 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
11828 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
11829 * the other bits as well, SCE and NXE. See @bugref{7368}.
11830 */
11831 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
11832 }
11833
11834 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
11835 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11836 {
11837 switch (pMixedCtx->ecx)
11838 {
11839 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
11840 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
11841 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
11842 case MSR_K8_FS_BASE: /* no break */
11843 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
11844 case MSR_K6_EFER: /* already handled above */ break;
11845 default:
11846 {
11847 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11848 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
11849#if HC_ARCH_BITS == 64
11850 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11851 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
11852#endif
11853 break;
11854 }
11855 }
11856 }
11857#ifdef VBOX_STRICT
11858 else
11859 {
11860 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
11861 switch (pMixedCtx->ecx)
11862 {
11863 case MSR_IA32_SYSENTER_CS:
11864 case MSR_IA32_SYSENTER_EIP:
11865 case MSR_IA32_SYSENTER_ESP:
11866 case MSR_K8_FS_BASE:
11867 case MSR_K8_GS_BASE:
11868 {
11869 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
11870 HMVMX_RETURN_UNEXPECTED_EXIT();
11871 }
11872
11873 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
11874 default:
11875 {
11876 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11877 {
11878 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
11879 if (pMixedCtx->ecx != MSR_K6_EFER)
11880 {
11881 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11882 pMixedCtx->ecx));
11883 HMVMX_RETURN_UNEXPECTED_EXIT();
11884 }
11885 }
11886
11887#if HC_ARCH_BITS == 64
11888 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11889 {
11890 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11891 HMVMX_RETURN_UNEXPECTED_EXIT();
11892 }
11893#endif
11894 break;
11895 }
11896 }
11897 }
11898#endif /* VBOX_STRICT */
11899 }
11900 return rc;
11901}
11902
11903
11904/**
11905 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
11906 */
11907HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11908{
11909 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11910
11911 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
11912 return VINF_EM_RAW_INTERRUPT;
11913}
11914
11915
11916/**
11917 * VM-exit handler for when the TPR value is lowered below the specified
11918 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
11919 */
11920HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11921{
11922 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11923 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
11924
11925 /*
11926 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
11927 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
11928 * resume guest execution.
11929 */
11930 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
11931 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
11932 return VINF_SUCCESS;
11933}
11934
11935
11936/**
11937 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
11938 * VM-exit.
11939 *
11940 * @retval VINF_SUCCESS when guest execution can continue.
11941 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
11942 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
11943 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
11944 * interpreter.
11945 */
11946HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11947{
11948 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11949 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
11950 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11951 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11952 AssertRCReturn(rc, rc);
11953
11954 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
11955 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
11956 PVM pVM = pVCpu->CTX_SUFF(pVM);
11957 VBOXSTRICTRC rcStrict;
11958 rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, true /*fNeedRsp*/);
11959 switch (uAccessType)
11960 {
11961 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
11962 {
11963 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11964 AssertRCReturn(rc, rc);
11965
11966 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
11967 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
11968 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
11969 AssertMsg( rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE
11970 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11971 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
11972 {
11973 case 0: /* CR0 */
11974 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11975 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
11976 break;
11977 case 2: /* CR2 */
11978 /* Nothing to do here, CR2 it's not part of the VMCS. */
11979 break;
11980 case 3: /* CR3 */
11981 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
11982 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
11983 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
11984 break;
11985 case 4: /* CR4 */
11986 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
11987 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
11988 VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
11989 break;
11990 case 8: /* CR8 */
11991 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
11992 /* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
11993 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
11994 break;
11995 default:
11996 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
11997 break;
11998 }
11999
12000 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12001 break;
12002 }
12003
12004 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12005 {
12006 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12007 AssertRCReturn(rc, rc);
12008
12009 Assert( !pVM->hm.s.fNestedPaging
12010 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12011 || pVCpu->hm.s.fUsingDebugLoop
12012 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12013
12014 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12015 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12016 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12017
12018 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12019 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12020 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12021 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12022 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12023 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12024 VBOXSTRICTRC_VAL(rcStrict)));
12025 break;
12026 }
12027
12028 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12029 {
12030 AssertRCReturn(rc, rc);
12031 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12032 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12033 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12034 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12035 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12036 break;
12037 }
12038
12039 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12040 {
12041 AssertRCReturn(rc, rc);
12042 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12043 VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12044 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE,
12045 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12046 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12047 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12048 break;
12049 }
12050
12051 default:
12052 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12053 VERR_VMX_UNEXPECTED_EXCEPTION);
12054 }
12055
12056 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12057 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12058 NOREF(pVM);
12059 return rcStrict;
12060}
12061
12062
12063/**
12064 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12065 * VM-exit.
12066 */
12067HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12068{
12069 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12070 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12071
12072 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12073 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12074 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12075 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12076 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12077 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12078 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12079 AssertRCReturn(rc2, rc2);
12080
12081 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12082 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12083 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12084 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12085 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12086 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12087 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12088 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12089 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12090
12091 /* I/O operation lookup arrays. */
12092 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12093 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12094
12095 VBOXSTRICTRC rcStrict;
12096 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12097 uint32_t const cbInstr = pVmxTransient->cbInstr;
12098 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12099 PVM pVM = pVCpu->CTX_SUFF(pVM);
12100 if (fIOString)
12101 {
12102#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12103 See @bugref{5752#c158}. Should work now. */
12104 /*
12105 * INS/OUTS - I/O String instruction.
12106 *
12107 * Use instruction-information if available, otherwise fall back on
12108 * interpreting the instruction.
12109 */
12110 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12111 fIOWrite ? 'w' : 'r'));
12112 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12113 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12114 {
12115 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12116 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12117 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12118 AssertRCReturn(rc2, rc2);
12119 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12120 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12121 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12122 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12123 if (fIOWrite)
12124 {
12125 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12126 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
12127 }
12128 else
12129 {
12130 /*
12131 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12132 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12133 * See Intel Instruction spec. for "INS".
12134 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12135 */
12136 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
12137 }
12138 }
12139 else
12140 {
12141 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12142 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12143 AssertRCReturn(rc2, rc2);
12144 rcStrict = IEMExecOne(pVCpu);
12145 }
12146 /** @todo IEM needs to be setting these flags somehow. */
12147 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12148 fUpdateRipAlready = true;
12149#else
12150 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12151 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12152 if (RT_SUCCESS(rcStrict))
12153 {
12154 if (fIOWrite)
12155 {
12156 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12157 (DISCPUMODE)pDis->uAddrMode, cbValue);
12158 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12159 }
12160 else
12161 {
12162 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12163 (DISCPUMODE)pDis->uAddrMode, cbValue);
12164 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12165 }
12166 }
12167 else
12168 {
12169 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12170 pMixedCtx->rip));
12171 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12172 }
12173#endif
12174 }
12175 else
12176 {
12177 /*
12178 * IN/OUT - I/O instruction.
12179 */
12180 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12181 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12182 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12183 if (fIOWrite)
12184 {
12185 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12186 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
12187 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12188 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12189 }
12190 else
12191 {
12192 uint32_t u32Result = 0;
12193 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12194 if (IOM_SUCCESS(rcStrict))
12195 {
12196 /* Save result of I/O IN instr. in AL/AX/EAX. */
12197 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12198 }
12199 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12200 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12201 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12202 }
12203 }
12204
12205 if (IOM_SUCCESS(rcStrict))
12206 {
12207 if (!fUpdateRipAlready)
12208 {
12209 pMixedCtx->rip += cbInstr;
12210 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12211 }
12212
12213 /*
12214 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12215 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12216 */
12217 if (fIOString)
12218 {
12219 /** @todo Single-step for INS/OUTS with REP prefix? */
12220 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12221 }
12222 else if ( !fDbgStepping
12223 && fGstStepping)
12224 {
12225 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12226 }
12227
12228 /*
12229 * If any I/O breakpoints are armed, we need to check if one triggered
12230 * and take appropriate action.
12231 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12232 */
12233 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12234 AssertRCReturn(rc2, rc2);
12235
12236 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12237 * execution engines about whether hyper BPs and such are pending. */
12238 uint32_t const uDr7 = pMixedCtx->dr[7];
12239 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12240 && X86_DR7_ANY_RW_IO(uDr7)
12241 && (pMixedCtx->cr4 & X86_CR4_DE))
12242 || DBGFBpIsHwIoArmed(pVM)))
12243 {
12244 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12245
12246 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12247 VMMRZCallRing3Disable(pVCpu);
12248 HM_DISABLE_PREEMPT();
12249
12250 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12251
12252 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12253 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12254 {
12255 /* Raise #DB. */
12256 if (fIsGuestDbgActive)
12257 ASMSetDR6(pMixedCtx->dr[6]);
12258 if (pMixedCtx->dr[7] != uDr7)
12259 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12260
12261 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12262 }
12263 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
12264 else if ( rcStrict2 != VINF_SUCCESS
12265 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12266 rcStrict = rcStrict2;
12267
12268 HM_RESTORE_PREEMPT();
12269 VMMRZCallRing3Enable(pVCpu);
12270 }
12271 }
12272
12273#ifdef VBOX_STRICT
12274 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12275 Assert(!fIOWrite);
12276 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
12277 Assert(fIOWrite);
12278 else
12279 {
12280#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12281 * statuses, that the VMM device and some others may return. See
12282 * IOM_SUCCESS() for guidance. */
12283 AssertMsg( RT_FAILURE(rcStrict)
12284 || rcStrict == VINF_SUCCESS
12285 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12286 || rcStrict == VINF_EM_DBG_BREAKPOINT
12287 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12288 || rcStrict == VINF_EM_RAW_TO_R3
12289 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12290#endif
12291 }
12292#endif
12293
12294 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12295 return rcStrict;
12296}
12297
12298
12299/**
12300 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12301 * VM-exit.
12302 */
12303HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12304{
12305 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12306
12307 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12308 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12309 AssertRCReturn(rc, rc);
12310 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12311 {
12312 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12313 AssertRCReturn(rc, rc);
12314 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12315 {
12316 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12317
12318 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12319 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12320
12321 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
12322 Assert(!pVCpu->hm.s.Event.fPending);
12323 pVCpu->hm.s.Event.fPending = true;
12324 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
12325 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12326 AssertRCReturn(rc, rc);
12327 if (fErrorCodeValid)
12328 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12329 else
12330 pVCpu->hm.s.Event.u32ErrCode = 0;
12331 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12332 && uVector == X86_XCPT_PF)
12333 {
12334 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
12335 }
12336
12337 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12338 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12339 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12340 }
12341 }
12342
12343 /** @todo Emulate task switch someday, currently just going back to ring-3 for
12344 * emulation. */
12345 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12346 return VERR_EM_INTERPRETER;
12347}
12348
12349
12350/**
12351 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12352 */
12353HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12354{
12355 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12356 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12357 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12358 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12359 AssertRCReturn(rc, rc);
12360 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12361 return VINF_EM_DBG_STEPPED;
12362}
12363
12364
12365/**
12366 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12367 */
12368HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12369{
12370 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12371
12372 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12373 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12374 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12375 { /* likely */ }
12376 else
12377 {
12378 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12379 rcStrict1 = VINF_SUCCESS;
12380 return rcStrict1;
12381 }
12382
12383#if 0
12384 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
12385 * just sync the whole thing. */
12386 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12387#else
12388 /* Aggressive state sync. for now. */
12389 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12390 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12391 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12392#endif
12393 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12394 AssertRCReturn(rc, rc);
12395
12396 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12397 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12398 VBOXSTRICTRC rcStrict2;
12399 switch (uAccessType)
12400 {
12401 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12402 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12403 {
12404 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12405 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != 0x80,
12406 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12407
12408 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
12409 GCPhys &= PAGE_BASE_GC_MASK;
12410 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12411 PVM pVM = pVCpu->CTX_SUFF(pVM);
12412 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
12413 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12414
12415 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12416 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12417 CPUMCTX2CORE(pMixedCtx), GCPhys);
12418 Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
12419 if ( rcStrict2 == VINF_SUCCESS
12420 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12421 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12422 {
12423 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12424 | HM_CHANGED_GUEST_RSP
12425 | HM_CHANGED_GUEST_RFLAGS
12426 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12427 rcStrict2 = VINF_SUCCESS;
12428 }
12429 break;
12430 }
12431
12432 default:
12433 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
12434 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12435 break;
12436 }
12437
12438 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12439 if (rcStrict2 != VINF_SUCCESS)
12440 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12441 return rcStrict2;
12442}
12443
12444
12445/**
12446 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12447 * VM-exit.
12448 */
12449HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12450{
12451 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12452
12453 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12454 if (pVmxTransient->fWasGuestDebugStateActive)
12455 {
12456 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12457 HMVMX_RETURN_UNEXPECTED_EXIT();
12458 }
12459
12460 if ( !pVCpu->hm.s.fSingleInstruction
12461 && !pVmxTransient->fWasHyperDebugStateActive)
12462 {
12463 Assert(!DBGFIsStepping(pVCpu));
12464 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12465
12466 /* Don't intercept MOV DRx any more. */
12467 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12468 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12469 AssertRCReturn(rc, rc);
12470
12471 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12472 VMMRZCallRing3Disable(pVCpu);
12473 HM_DISABLE_PREEMPT();
12474
12475 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12476 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12477 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12478
12479 HM_RESTORE_PREEMPT();
12480 VMMRZCallRing3Enable(pVCpu);
12481
12482#ifdef VBOX_WITH_STATISTICS
12483 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12484 AssertRCReturn(rc, rc);
12485 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12486 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12487 else
12488 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12489#endif
12490 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12491 return VINF_SUCCESS;
12492 }
12493
12494 /*
12495 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12496 * Update the segment registers and DR7 from the CPU.
12497 */
12498 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12499 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12500 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12501 AssertRCReturn(rc, rc);
12502 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12503
12504 PVM pVM = pVCpu->CTX_SUFF(pVM);
12505 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12506 {
12507 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12508 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
12509 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
12510 if (RT_SUCCESS(rc))
12511 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12512 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12513 }
12514 else
12515 {
12516 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12517 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
12518 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
12519 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12520 }
12521
12522 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12523 if (RT_SUCCESS(rc))
12524 {
12525 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12526 AssertRCReturn(rc2, rc2);
12527 return VINF_SUCCESS;
12528 }
12529 return rc;
12530}
12531
12532
12533/**
12534 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12535 * Conditional VM-exit.
12536 */
12537HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12538{
12539 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12540 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12541
12542 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12543 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12544 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12545 { /* likely */ }
12546 else
12547 {
12548 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12549 rcStrict1 = VINF_SUCCESS;
12550 return rcStrict1;
12551 }
12552
12553 RTGCPHYS GCPhys = 0;
12554 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12555
12556#if 0
12557 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12558#else
12559 /* Aggressive state sync. for now. */
12560 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12561 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12562 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12563#endif
12564 AssertRCReturn(rc, rc);
12565
12566 /*
12567 * If we succeed, resume guest execution.
12568 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12569 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12570 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12571 * weird case. See @bugref{6043}.
12572 */
12573 PVM pVM = pVCpu->CTX_SUFF(pVM);
12574 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12575 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
12576 if ( rcStrict2 == VINF_SUCCESS
12577 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12578 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12579 {
12580 /* Successfully handled MMIO operation. */
12581 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12582 | HM_CHANGED_GUEST_RSP
12583 | HM_CHANGED_GUEST_RFLAGS
12584 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12585 return VINF_SUCCESS;
12586 }
12587 return rcStrict2;
12588}
12589
12590
12591/**
12592 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12593 * VM-exit.
12594 */
12595HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12596{
12597 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12598 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12599
12600 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12601 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12602 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12603 { /* likely */ }
12604 else
12605 {
12606 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12607 rcStrict1 = VINF_SUCCESS;
12608 return rcStrict1;
12609 }
12610
12611 RTGCPHYS GCPhys = 0;
12612 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12613 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12614#if 0
12615 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12616#else
12617 /* Aggressive state sync. for now. */
12618 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12619 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12620 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12621#endif
12622 AssertRCReturn(rc, rc);
12623
12624 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12625 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12626
12627 RTGCUINT uErrorCode = 0;
12628 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
12629 uErrorCode |= X86_TRAP_PF_ID;
12630 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
12631 uErrorCode |= X86_TRAP_PF_RW;
12632 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
12633 uErrorCode |= X86_TRAP_PF_P;
12634
12635 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12636
12637 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12638 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12639
12640 /* Handle the pagefault trap for the nested shadow table. */
12641 PVM pVM = pVCpu->CTX_SUFF(pVM);
12642 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12643 TRPMResetTrap(pVCpu);
12644
12645 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12646 if ( rcStrict2 == VINF_SUCCESS
12647 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12648 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12649 {
12650 /* Successfully synced our nested page tables. */
12651 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12652 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12653 | HM_CHANGED_GUEST_RSP
12654 | HM_CHANGED_GUEST_RFLAGS);
12655 return VINF_SUCCESS;
12656 }
12657
12658 Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12659 return rcStrict2;
12660}
12661
12662/** @} */
12663
12664/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12665/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
12666/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12667
12668/** @name VM-exit exception handlers.
12669 * @{
12670 */
12671
12672/**
12673 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12674 */
12675static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12676{
12677 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12678 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12679
12680 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12681 AssertRCReturn(rc, rc);
12682
12683 if (!(pMixedCtx->cr0 & X86_CR0_NE))
12684 {
12685 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12686 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12687
12688 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12689 * provides VM-exit instruction length. If this causes problem later,
12690 * disassemble the instruction like it's done on AMD-V. */
12691 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12692 AssertRCReturn(rc2, rc2);
12693 return rc;
12694 }
12695
12696 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12697 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12698 return rc;
12699}
12700
12701
12702/**
12703 * VM-exit exception handler for \#BP (Breakpoint exception).
12704 */
12705static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12706{
12707 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12708 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12709
12710 /** @todo Try optimize this by not saving the entire guest state unless
12711 * really needed. */
12712 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12713 AssertRCReturn(rc, rc);
12714
12715 PVM pVM = pVCpu->CTX_SUFF(pVM);
12716 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12717 if (rc == VINF_EM_RAW_GUEST_TRAP)
12718 {
12719 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12720 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12721 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12722 AssertRCReturn(rc, rc);
12723
12724 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12725 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12726 }
12727
12728 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
12729 return rc;
12730}
12731
12732
12733/**
12734 * VM-exit exception handler for \#AC (alignment check exception).
12735 */
12736static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12737{
12738 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12739
12740 /*
12741 * Re-inject it. We'll detect any nesting before getting here.
12742 */
12743 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12744 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12745 AssertRCReturn(rc, rc);
12746 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
12747
12748 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12749 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12750 return VINF_SUCCESS;
12751}
12752
12753
12754/**
12755 * VM-exit exception handler for \#DB (Debug exception).
12756 */
12757static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12758{
12759 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12760 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
12761 Log6(("XcptDB\n"));
12762
12763 /*
12764 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
12765 * for processing.
12766 */
12767 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12768 AssertRCReturn(rc, rc);
12769
12770 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
12771 uint64_t uDR6 = X86_DR6_INIT_VAL;
12772 uDR6 |= ( pVmxTransient->uExitQualification
12773 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
12774
12775 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
12776 if (rc == VINF_EM_RAW_GUEST_TRAP)
12777 {
12778 /*
12779 * The exception was for the guest. Update DR6, DR7.GD and
12780 * IA32_DEBUGCTL.LBR before forwarding it.
12781 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
12782 */
12783 VMMRZCallRing3Disable(pVCpu);
12784 HM_DISABLE_PREEMPT();
12785
12786 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
12787 pMixedCtx->dr[6] |= uDR6;
12788 if (CPUMIsGuestDebugStateActive(pVCpu))
12789 ASMSetDR6(pMixedCtx->dr[6]);
12790
12791 HM_RESTORE_PREEMPT();
12792 VMMRZCallRing3Enable(pVCpu);
12793
12794 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12795 AssertRCReturn(rc, rc);
12796
12797 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
12798 pMixedCtx->dr[7] &= ~X86_DR7_GD;
12799
12800 /* Paranoia. */
12801 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
12802 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
12803
12804 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
12805 AssertRCReturn(rc, rc);
12806
12807 /*
12808 * Raise #DB in the guest.
12809 *
12810 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
12811 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
12812 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
12813 *
12814 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
12815 */
12816 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12817 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12818 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12819 AssertRCReturn(rc, rc);
12820 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12821 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12822 return VINF_SUCCESS;
12823 }
12824
12825 /*
12826 * Not a guest trap, must be a hypervisor related debug event then.
12827 * Update DR6 in case someone is interested in it.
12828 */
12829 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
12830 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
12831 CPUMSetHyperDR6(pVCpu, uDR6);
12832
12833 return rc;
12834}
12835
12836
12837/**
12838 * VM-exit exception handler for \#NM (Device-not-available exception: floating
12839 * point exception).
12840 */
12841static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12842{
12843 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12844
12845 /* We require CR0 and EFER. EFER is always up-to-date. */
12846 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12847 AssertRCReturn(rc, rc);
12848
12849 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
12850 VMMRZCallRing3Disable(pVCpu);
12851 HM_DISABLE_PREEMPT();
12852
12853 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
12854 if (pVmxTransient->fWasGuestFPUStateActive)
12855 {
12856 rc = VINF_EM_RAW_GUEST_TRAP;
12857 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
12858 }
12859 else
12860 {
12861#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12862 Assert(!pVmxTransient->fWasGuestFPUStateActive || pVCpu->hm.s.fUsingDebugLoop);
12863#endif
12864 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12865 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
12866 }
12867
12868 HM_RESTORE_PREEMPT();
12869 VMMRZCallRing3Enable(pVCpu);
12870
12871 if (rc == VINF_SUCCESS)
12872 {
12873 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
12874 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12875 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
12876 pVCpu->hm.s.fPreloadGuestFpu = true;
12877 }
12878 else
12879 {
12880 /* Forward #NM to the guest. */
12881 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
12882 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12883 AssertRCReturn(rc, rc);
12884 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12885 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
12886 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
12887 }
12888
12889 return VINF_SUCCESS;
12890}
12891
12892
12893/**
12894 * VM-exit exception handler for \#GP (General-protection exception).
12895 *
12896 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
12897 */
12898static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12899{
12900 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12901 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
12902
12903 int rc;
12904 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
12905 { /* likely */ }
12906 else
12907 {
12908#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12909 Assert(pVCpu->hm.s.fUsingDebugLoop);
12910#endif
12911 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
12912 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12913 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12914 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12915 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12916 AssertRCReturn(rc, rc);
12917 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
12918 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
12919 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12920 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12921 return rc;
12922 }
12923
12924 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
12925 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
12926
12927 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
12928 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12929 AssertRCReturn(rc, rc);
12930
12931 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12932 uint32_t cbOp = 0;
12933 PVM pVM = pVCpu->CTX_SUFF(pVM);
12934 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12935 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
12936 if (RT_SUCCESS(rc))
12937 {
12938 rc = VINF_SUCCESS;
12939 Assert(cbOp == pDis->cbInstr);
12940 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12941 switch (pDis->pCurInstr->uOpcode)
12942 {
12943 case OP_CLI:
12944 {
12945 pMixedCtx->eflags.Bits.u1IF = 0;
12946 pMixedCtx->eflags.Bits.u1RF = 0;
12947 pMixedCtx->rip += pDis->cbInstr;
12948 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12949 if ( !fDbgStepping
12950 && pMixedCtx->eflags.Bits.u1TF)
12951 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12952 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
12953 break;
12954 }
12955
12956 case OP_STI:
12957 {
12958 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
12959 pMixedCtx->eflags.Bits.u1IF = 1;
12960 pMixedCtx->eflags.Bits.u1RF = 0;
12961 pMixedCtx->rip += pDis->cbInstr;
12962 if (!fOldIF)
12963 {
12964 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
12965 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
12966 }
12967 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12968 if ( !fDbgStepping
12969 && pMixedCtx->eflags.Bits.u1TF)
12970 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12971 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
12972 break;
12973 }
12974
12975 case OP_HLT:
12976 {
12977 rc = VINF_EM_HALT;
12978 pMixedCtx->rip += pDis->cbInstr;
12979 pMixedCtx->eflags.Bits.u1RF = 0;
12980 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12981 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
12982 break;
12983 }
12984
12985 case OP_POPF:
12986 {
12987 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12988 uint32_t cbParm;
12989 uint32_t uMask;
12990 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12991 if (pDis->fPrefix & DISPREFIX_OPSIZE)
12992 {
12993 cbParm = 4;
12994 uMask = 0xffffffff;
12995 }
12996 else
12997 {
12998 cbParm = 2;
12999 uMask = 0xffff;
13000 }
13001
13002 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13003 RTGCPTR GCPtrStack = 0;
13004 X86EFLAGS Eflags;
13005 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13006 &GCPtrStack);
13007 if (RT_SUCCESS(rc))
13008 {
13009 Assert(sizeof(Eflags.u32) >= cbParm);
13010 Eflags.u32 = 0;
13011 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13012 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13013 }
13014 if (RT_FAILURE(rc))
13015 {
13016 rc = VERR_EM_INTERPRETER;
13017 break;
13018 }
13019 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13020 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13021 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13022 pMixedCtx->esp += cbParm;
13023 pMixedCtx->esp &= uMask;
13024 pMixedCtx->rip += pDis->cbInstr;
13025 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13026 | HM_CHANGED_GUEST_RSP
13027 | HM_CHANGED_GUEST_RFLAGS);
13028 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13029 POPF restores EFLAGS.TF. */
13030 if ( !fDbgStepping
13031 && fGstStepping)
13032 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13033 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13034 break;
13035 }
13036
13037 case OP_PUSHF:
13038 {
13039 uint32_t cbParm;
13040 uint32_t uMask;
13041 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13042 {
13043 cbParm = 4;
13044 uMask = 0xffffffff;
13045 }
13046 else
13047 {
13048 cbParm = 2;
13049 uMask = 0xffff;
13050 }
13051
13052 /* Get the stack pointer & push the contents of eflags onto the stack. */
13053 RTGCPTR GCPtrStack = 0;
13054 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13055 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13056 if (RT_FAILURE(rc))
13057 {
13058 rc = VERR_EM_INTERPRETER;
13059 break;
13060 }
13061 X86EFLAGS Eflags = pMixedCtx->eflags;
13062 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13063 Eflags.Bits.u1RF = 0;
13064 Eflags.Bits.u1VM = 0;
13065
13066 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13067 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13068 {
13069 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13070 rc = VERR_EM_INTERPRETER;
13071 break;
13072 }
13073 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13074 pMixedCtx->esp -= cbParm;
13075 pMixedCtx->esp &= uMask;
13076 pMixedCtx->rip += pDis->cbInstr;
13077 pMixedCtx->eflags.Bits.u1RF = 0;
13078 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13079 | HM_CHANGED_GUEST_RSP
13080 | HM_CHANGED_GUEST_RFLAGS);
13081 if ( !fDbgStepping
13082 && pMixedCtx->eflags.Bits.u1TF)
13083 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13084 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13085 break;
13086 }
13087
13088 case OP_IRET:
13089 {
13090 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13091 * instruction reference. */
13092 RTGCPTR GCPtrStack = 0;
13093 uint32_t uMask = 0xffff;
13094 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13095 uint16_t aIretFrame[3];
13096 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13097 {
13098 rc = VERR_EM_INTERPRETER;
13099 break;
13100 }
13101 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13102 &GCPtrStack);
13103 if (RT_SUCCESS(rc))
13104 {
13105 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13106 PGMACCESSORIGIN_HM));
13107 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13108 }
13109 if (RT_FAILURE(rc))
13110 {
13111 rc = VERR_EM_INTERPRETER;
13112 break;
13113 }
13114 pMixedCtx->eip = 0;
13115 pMixedCtx->ip = aIretFrame[0];
13116 pMixedCtx->cs.Sel = aIretFrame[1];
13117 pMixedCtx->cs.ValidSel = aIretFrame[1];
13118 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13119 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13120 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13121 pMixedCtx->sp += sizeof(aIretFrame);
13122 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13123 | HM_CHANGED_GUEST_SEGMENT_REGS
13124 | HM_CHANGED_GUEST_RSP
13125 | HM_CHANGED_GUEST_RFLAGS);
13126 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13127 if ( !fDbgStepping
13128 && fGstStepping)
13129 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13130 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13131 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13132 break;
13133 }
13134
13135 case OP_INT:
13136 {
13137 uint16_t uVector = pDis->Param1.uValue & 0xff;
13138 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13139 /* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
13140 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13141 break;
13142 }
13143
13144 case OP_INTO:
13145 {
13146 if (pMixedCtx->eflags.Bits.u1OF)
13147 {
13148 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13149 /* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
13150 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13151 }
13152 else
13153 {
13154 pMixedCtx->eflags.Bits.u1RF = 0;
13155 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13156 }
13157 break;
13158 }
13159
13160 default:
13161 {
13162 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13163 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13164 EMCODETYPE_SUPERVISOR);
13165 rc = VBOXSTRICTRC_VAL(rc2);
13166 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13167 /** @todo We have to set pending-debug exceptions here when the guest is
13168 * single-stepping depending on the instruction that was interpreted. */
13169 Log4(("#GP rc=%Rrc\n", rc));
13170 break;
13171 }
13172 }
13173 }
13174 else
13175 rc = VERR_EM_INTERPRETER;
13176
13177 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13178 ("#GP Unexpected rc=%Rrc\n", rc));
13179 return rc;
13180}
13181
13182
13183/**
13184 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13185 * the exception reported in the VMX transient structure back into the VM.
13186 *
13187 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13188 * up-to-date.
13189 */
13190static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13191{
13192 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13193#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13194 Assert(pVCpu->hm.s.fUsingDebugLoop);
13195#endif
13196
13197 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13198 hmR0VmxCheckExitDueToEventDelivery(). */
13199 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13200 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13201 AssertRCReturn(rc, rc);
13202 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13203
13204#ifdef DEBUG_ramshankar
13205 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13206 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13207 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13208#endif
13209
13210 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13211 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13212 return VINF_SUCCESS;
13213}
13214
13215
13216/**
13217 * VM-exit exception handler for \#PF (Page-fault exception).
13218 */
13219static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13220{
13221 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13222 PVM pVM = pVCpu->CTX_SUFF(pVM);
13223 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13224 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13225 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13226 AssertRCReturn(rc, rc);
13227
13228 if (!pVM->hm.s.fNestedPaging)
13229 { /* likely */ }
13230 else
13231 {
13232#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13233 Assert(pVCpu->hm.s.fUsingDebugLoop);
13234#endif
13235 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13236 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13237 {
13238 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13239 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13240 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13241 }
13242 else
13243 {
13244 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13245 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13246 Log4(("Pending #DF due to vectoring #PF. NP\n"));
13247 }
13248 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13249 return rc;
13250 }
13251
13252 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13253 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13254 if (pVmxTransient->fVectoringPF)
13255 {
13256 Assert(pVCpu->hm.s.Event.fPending);
13257 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13258 }
13259
13260 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13261 AssertRCReturn(rc, rc);
13262
13263 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13264 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13265
13266 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13267 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13268 (RTGCPTR)pVmxTransient->uExitQualification);
13269
13270 Log4(("#PF: rc=%Rrc\n", rc));
13271 if (rc == VINF_SUCCESS)
13272 {
13273 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13274 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
13275 * memory? We don't update the whole state here... */
13276 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13277 | HM_CHANGED_GUEST_RSP
13278 | HM_CHANGED_GUEST_RFLAGS
13279 | HM_CHANGED_VMX_GUEST_APIC_STATE);
13280 TRPMResetTrap(pVCpu);
13281 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13282 return rc;
13283 }
13284
13285 if (rc == VINF_EM_RAW_GUEST_TRAP)
13286 {
13287 if (!pVmxTransient->fVectoringDoublePF)
13288 {
13289 /* It's a guest page fault and needs to be reflected to the guest. */
13290 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13291 TRPMResetTrap(pVCpu);
13292 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13293 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13294 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13295 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13296 }
13297 else
13298 {
13299 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13300 TRPMResetTrap(pVCpu);
13301 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13302 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13303 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
13304 }
13305
13306 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13307 return VINF_SUCCESS;
13308 }
13309
13310 TRPMResetTrap(pVCpu);
13311 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13312 return rc;
13313}
13314
13315/** @} */
13316
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette