VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 59176

Last change on this file since 59176 was 59149, checked in by vboxsync, 9 years ago

VMM/HMVMXR0: Fewer branches where applicable.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 571.5 KB
Line 
1/* $Id: HMVMXR0.cpp 59149 2015-12-16 11:12:58Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2015 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#include <iprt/x86.h>
24#include <iprt/asm-amd64-x86.h>
25#include <iprt/thread.h>
26
27#include <VBox/vmm/pdmapi.h>
28#include <VBox/vmm/dbgf.h>
29#include <VBox/vmm/iem.h>
30#include <VBox/vmm/iom.h>
31#include <VBox/vmm/selm.h>
32#include <VBox/vmm/tm.h>
33#include <VBox/vmm/gim.h>
34#ifdef VBOX_WITH_REM
35# include <VBox/vmm/rem.h>
36#endif
37#include "HMInternal.h"
38#include <VBox/vmm/vm.h>
39#include "HMVMXR0.h"
40#include "dtrace/VBoxVMM.h"
41
42#ifdef DEBUG_ramshankar
43# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
44# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
45# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
46# define HMVMX_ALWAYS_CHECK_GUEST_STATE
47# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
48# define HMVMX_ALWAYS_TRAP_PF
49# define HMVMX_ALWAYS_SWAP_FPU_STATE
50# define HMVMX_ALWAYS_FLUSH_TLB
51# define HMVMX_ALWAYS_SWAP_EFER
52#endif
53
54
55/*********************************************************************************************************************************
56* Defined Constants And Macros *
57*********************************************************************************************************************************/
58/** Use the function table. */
59#define HMVMX_USE_FUNCTION_TABLE
60
61/** Determine which tagged-TLB flush handler to use. */
62#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
63#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
64#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
65#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
66
67/** @name Updated-guest-state flags.
68 * @{ */
69#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
70#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
71#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
72#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
73#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
74#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
75#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
76#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
77#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
78#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
79#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
80#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
81#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
82#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
83#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
84#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
85#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
86#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
87#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
88#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
89#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
90 | HMVMX_UPDATED_GUEST_RSP \
91 | HMVMX_UPDATED_GUEST_RFLAGS \
92 | HMVMX_UPDATED_GUEST_CR0 \
93 | HMVMX_UPDATED_GUEST_CR3 \
94 | HMVMX_UPDATED_GUEST_CR4 \
95 | HMVMX_UPDATED_GUEST_GDTR \
96 | HMVMX_UPDATED_GUEST_IDTR \
97 | HMVMX_UPDATED_GUEST_LDTR \
98 | HMVMX_UPDATED_GUEST_TR \
99 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
100 | HMVMX_UPDATED_GUEST_DEBUG \
101 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
102 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
103 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
104 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
105 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
106 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
107 | HMVMX_UPDATED_GUEST_INTR_STATE \
108 | HMVMX_UPDATED_GUEST_APIC_STATE)
109/** @} */
110
111/** @name
112 * Flags to skip redundant reads of some common VMCS fields that are not part of
113 * the guest-CPU state but are in the transient structure.
114 */
115#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
116#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
117#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
118#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
119#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
120#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
122/** @} */
123
124/** @name
125 * States of the VMCS.
126 *
127 * This does not reflect all possible VMCS states but currently only those
128 * needed for maintaining the VMCS consistently even when thread-context hooks
129 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
130 */
131#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
132#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
133#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
134/** @} */
135
136/**
137 * Exception bitmap mask for real-mode guests (real-on-v86).
138 *
139 * We need to intercept all exceptions manually except:
140 * - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
141 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
142 * due to bugs in Intel CPUs.
143 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
144 * support.
145 */
146#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
147 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
148 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
149 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
150 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
151 /* RT_BIT(X86_XCPT_MF) always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
152 | RT_BIT(X86_XCPT_XF))
153
154/**
155 * Exception bitmap mask for all contributory exceptions.
156 *
157 * Page fault is deliberately excluded here as it's conditional as to whether
158 * it's contributory or benign. Page faults are handled separately.
159 */
160#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
161 | RT_BIT(X86_XCPT_DE))
162
163/** Maximum VM-instruction error number. */
164#define HMVMX_INSTR_ERROR_MAX 28
165
166/** Profiling macro. */
167#ifdef HM_PROFILE_EXIT_DISPATCH
168# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
169# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
170#else
171# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
172# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
173#endif
174
175/** Assert that preemption is disabled or covered by thread-context hooks. */
176#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
177 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
178
179/** Assert that we haven't migrated CPUs when thread-context hooks are not
180 * used. */
181#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
182 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
183 ("Illegal migration! Entered on CPU %u Current %u\n", \
184 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
185
186/** Helper macro for VM-exit handlers called unexpectedly. */
187#define HMVMX_RETURN_UNEXPECTED_EXIT() \
188 do { \
189 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
190 return VERR_VMX_UNEXPECTED_EXIT; \
191 } while (0)
192
193
194/*********************************************************************************************************************************
195* Structures and Typedefs *
196*********************************************************************************************************************************/
197/**
198 * VMX transient state.
199 *
200 * A state structure for holding miscellaneous information across
201 * VMX non-root operation and restored after the transition.
202 */
203typedef struct VMXTRANSIENT
204{
205 /** The host's rflags/eflags. */
206 RTCCUINTREG fEFlags;
207#if HC_ARCH_BITS == 32
208 uint32_t u32Alignment0;
209#endif
210 /** The guest's TPR value used for TPR shadowing. */
211 uint8_t u8GuestTpr;
212 /** Alignment. */
213 uint8_t abAlignment0[7];
214
215 /** The basic VM-exit reason. */
216 uint16_t uExitReason;
217 /** Alignment. */
218 uint16_t u16Alignment0;
219 /** The VM-exit interruption error code. */
220 uint32_t uExitIntErrorCode;
221 /** The VM-exit exit code qualification. */
222 uint64_t uExitQualification;
223
224 /** The VM-exit interruption-information field. */
225 uint32_t uExitIntInfo;
226 /** The VM-exit instruction-length field. */
227 uint32_t cbInstr;
228 /** The VM-exit instruction-information field. */
229 union
230 {
231 /** Plain unsigned int representation. */
232 uint32_t u;
233 /** INS and OUTS information. */
234 struct
235 {
236 uint32_t u6Reserved0 : 7;
237 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
238 uint32_t u3AddrSize : 3;
239 uint32_t u5Reserved1 : 5;
240 /** The segment register (X86_SREG_XXX). */
241 uint32_t iSegReg : 3;
242 uint32_t uReserved2 : 14;
243 } StrIo;
244 } ExitInstrInfo;
245 /** Whether the VM-entry failed or not. */
246 bool fVMEntryFailed;
247 /** Alignment. */
248 uint8_t abAlignment1[3];
249
250 /** The VM-entry interruption-information field. */
251 uint32_t uEntryIntInfo;
252 /** The VM-entry exception error code field. */
253 uint32_t uEntryXcptErrorCode;
254 /** The VM-entry instruction length field. */
255 uint32_t cbEntryInstr;
256
257 /** IDT-vectoring information field. */
258 uint32_t uIdtVectoringInfo;
259 /** IDT-vectoring error code. */
260 uint32_t uIdtVectoringErrorCode;
261
262 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
263 uint32_t fVmcsFieldsRead;
264
265 /** Whether the guest FPU was active at the time of VM-exit. */
266 bool fWasGuestFPUStateActive;
267 /** Whether the guest debug state was active at the time of VM-exit. */
268 bool fWasGuestDebugStateActive;
269 /** Whether the hyper debug state was active at the time of VM-exit. */
270 bool fWasHyperDebugStateActive;
271 /** Whether TSC-offsetting should be setup before VM-entry. */
272 bool fUpdateTscOffsettingAndPreemptTimer;
273 /** Whether the VM-exit was caused by a page-fault during delivery of a
274 * contributory exception or a page-fault. */
275 bool fVectoringDoublePF;
276 /** Whether the VM-exit was caused by a page-fault during delivery of an
277 * external interrupt or NMI. */
278 bool fVectoringPF;
279} VMXTRANSIENT;
280AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
281AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
282AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
283AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
284AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
285/** Pointer to VMX transient state. */
286typedef VMXTRANSIENT *PVMXTRANSIENT;
287
288
289/**
290 * MSR-bitmap read permissions.
291 */
292typedef enum VMXMSREXITREAD
293{
294 /** Reading this MSR causes a VM-exit. */
295 VMXMSREXIT_INTERCEPT_READ = 0xb,
296 /** Reading this MSR does not cause a VM-exit. */
297 VMXMSREXIT_PASSTHRU_READ
298} VMXMSREXITREAD;
299/** Pointer to MSR-bitmap read permissions. */
300typedef VMXMSREXITREAD* PVMXMSREXITREAD;
301
302/**
303 * MSR-bitmap write permissions.
304 */
305typedef enum VMXMSREXITWRITE
306{
307 /** Writing to this MSR causes a VM-exit. */
308 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
309 /** Writing to this MSR does not cause a VM-exit. */
310 VMXMSREXIT_PASSTHRU_WRITE
311} VMXMSREXITWRITE;
312/** Pointer to MSR-bitmap write permissions. */
313typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
314
315
316/**
317 * VMX VM-exit handler.
318 *
319 * @returns Strict VBox status code (i.e. informational status codes too).
320 * @param pVCpu The cross context virtual CPU structure.
321 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
322 * out-of-sync. Make sure to update the required
323 * fields before using them.
324 * @param pVmxTransient Pointer to the VMX-transient structure.
325 */
326#ifndef HMVMX_USE_FUNCTION_TABLE
327typedef DECLINLINE(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
328#else
329typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
330/** Pointer to VM-exit handler. */
331typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
332#endif
333
334/**
335 * VMX VM-exit handler, non-strict status code.
336 *
337 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
338 *
339 * @returns VBox status code, no informational status code returned.
340 * @param pVCpu The cross context virtual CPU structure.
341 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
342 * out-of-sync. Make sure to update the required
343 * fields before using them.
344 * @param pVmxTransient Pointer to the VMX-transient structure.
345 *
346 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
347 * use of that status code will be replaced with VINF_EM_SOMETHING
348 * later when switching over to IEM.
349 */
350#ifndef HMVMX_USE_FUNCTION_TABLE
351typedef DECLINLINE(int) FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
352#else
353typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
354#endif
355
356
357/*********************************************************************************************************************************
358* Internal Functions *
359*********************************************************************************************************************************/
360static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
361static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
362static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
363static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
364 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
365 bool fStepping, uint32_t *puIntState);
366#if HC_ARCH_BITS == 32
367static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
368#endif
369#ifndef HMVMX_USE_FUNCTION_TABLE
370DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
371# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
372# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
373#else
374# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
375# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
376#endif
377
378
379/** @name VM-exit handlers.
380 * @{
381 */
382static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
383static FNVMXEXITHANDLER hmR0VmxExitExtInt;
384static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
385static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
386static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
387static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
388static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
389static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
390static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
391static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
392static FNVMXEXITHANDLER hmR0VmxExitCpuid;
393static FNVMXEXITHANDLER hmR0VmxExitGetsec;
394static FNVMXEXITHANDLER hmR0VmxExitHlt;
395static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
396static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
397static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
398static FNVMXEXITHANDLER hmR0VmxExitVmcall;
399static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
400static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
401static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
402static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
403static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
404static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
405static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
406static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
407static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
408static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
409static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
410static FNVMXEXITHANDLER hmR0VmxExitMwait;
411static FNVMXEXITHANDLER hmR0VmxExitMtf;
412static FNVMXEXITHANDLER hmR0VmxExitMonitor;
413static FNVMXEXITHANDLER hmR0VmxExitPause;
414static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
415static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
416static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
417static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
418static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
419static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
420static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
421static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
422static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
423static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
424static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
425static FNVMXEXITHANDLER hmR0VmxExitRdrand;
426static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
427/** @} */
428
429static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
430static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
431static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
432static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
433static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
434static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
435static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
436static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
437static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
438
439
440/*********************************************************************************************************************************
441* Global Variables *
442*********************************************************************************************************************************/
443#ifdef HMVMX_USE_FUNCTION_TABLE
444
445/**
446 * VMX_EXIT dispatch table.
447 */
448static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
449{
450 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
451 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
452 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
453 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
454 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
455 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
456 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
457 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
458 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
459 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
460 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
461 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
462 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
463 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
464 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
465 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
466 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
467 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
468 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
469 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
470 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
471 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
472 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
473 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
474 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
475 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
476 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
477 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
478 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
479 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
480 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
481 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
482 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
483 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
484 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
485 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
486 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
487 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
488 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
489 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
490 /* 40 UNDEFINED */ hmR0VmxExitPause,
491 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
492 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
493 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
494 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
495 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
496 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
497 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
498 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
499 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
500 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
501 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
502 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
503 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
504 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
505 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
506 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
507 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
508 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
509 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
510 /* 60 VMX_EXIT_RESERVED_60 */ hmR0VmxExitErrUndefined,
511 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
512 /* 62 VMX_EXIT_RESERVED_62 */ hmR0VmxExitErrUndefined,
513 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
514 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
515};
516#endif /* HMVMX_USE_FUNCTION_TABLE */
517
518#ifdef VBOX_STRICT
519static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
520{
521 /* 0 */ "(Not Used)",
522 /* 1 */ "VMCALL executed in VMX root operation.",
523 /* 2 */ "VMCLEAR with invalid physical address.",
524 /* 3 */ "VMCLEAR with VMXON pointer.",
525 /* 4 */ "VMLAUNCH with non-clear VMCS.",
526 /* 5 */ "VMRESUME with non-launched VMCS.",
527 /* 6 */ "VMRESUME after VMXOFF",
528 /* 7 */ "VM-entry with invalid control fields.",
529 /* 8 */ "VM-entry with invalid host state fields.",
530 /* 9 */ "VMPTRLD with invalid physical address.",
531 /* 10 */ "VMPTRLD with VMXON pointer.",
532 /* 11 */ "VMPTRLD with incorrect revision identifier.",
533 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
534 /* 13 */ "VMWRITE to read-only VMCS component.",
535 /* 14 */ "(Not Used)",
536 /* 15 */ "VMXON executed in VMX root operation.",
537 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
538 /* 17 */ "VM-entry with non-launched executing VMCS.",
539 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
540 /* 19 */ "VMCALL with non-clear VMCS.",
541 /* 20 */ "VMCALL with invalid VM-exit control fields.",
542 /* 21 */ "(Not Used)",
543 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
544 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
545 /* 24 */ "VMCALL with invalid SMM-monitor features.",
546 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
547 /* 26 */ "VM-entry with events blocked by MOV SS.",
548 /* 27 */ "(Not Used)",
549 /* 28 */ "Invalid operand to INVEPT/INVVPID."
550};
551#endif /* VBOX_STRICT */
552
553
554
555/**
556 * Updates the VM's last error record.
557 *
558 * If there was a VMX instruction error, reads the error data from the VMCS and
559 * updates VCPU's last error record as well.
560 *
561 * @param pVM The cross context VM structure.
562 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
563 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
564 * VERR_VMX_INVALID_VMCS_FIELD.
565 * @param rc The error code.
566 */
567static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
568{
569 AssertPtr(pVM);
570 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
571 || rc == VERR_VMX_UNABLE_TO_START_VM)
572 {
573 AssertPtrReturnVoid(pVCpu);
574 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
575 }
576 pVM->hm.s.lLastError = rc;
577}
578
579
580/**
581 * Reads the VM-entry interruption-information field from the VMCS into the VMX
582 * transient structure.
583 *
584 * @returns VBox status code.
585 * @param pVmxTransient Pointer to the VMX transient structure.
586 *
587 * @remarks No-long-jump zone!!!
588 */
589DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
590{
591 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
592 AssertRCReturn(rc, rc);
593 return VINF_SUCCESS;
594}
595
596
597/**
598 * Reads the VM-entry exception error code field from the VMCS into
599 * the VMX transient structure.
600 *
601 * @returns VBox status code.
602 * @param pVmxTransient Pointer to the VMX transient structure.
603 *
604 * @remarks No-long-jump zone!!!
605 */
606DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
607{
608 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
609 AssertRCReturn(rc, rc);
610 return VINF_SUCCESS;
611}
612
613
614/**
615 * Reads the VM-entry exception error code field from the VMCS into
616 * the VMX transient structure.
617 *
618 * @returns VBox status code.
619 * @param pVmxTransient Pointer to the VMX transient structure.
620 *
621 * @remarks No-long-jump zone!!!
622 */
623DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
624{
625 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
626 AssertRCReturn(rc, rc);
627 return VINF_SUCCESS;
628}
629
630
631/**
632 * Reads the VM-exit interruption-information field from the VMCS into the VMX
633 * transient structure.
634 *
635 * @returns VBox status code.
636 * @param pVmxTransient Pointer to the VMX transient structure.
637 */
638DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
639{
640 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
641 {
642 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
643 AssertRCReturn(rc, rc);
644 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
645 }
646 return VINF_SUCCESS;
647}
648
649
650/**
651 * Reads the VM-exit interruption error code from the VMCS into the VMX
652 * transient structure.
653 *
654 * @returns VBox status code.
655 * @param pVmxTransient Pointer to the VMX transient structure.
656 */
657DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
658{
659 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
660 {
661 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
662 AssertRCReturn(rc, rc);
663 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
664 }
665 return VINF_SUCCESS;
666}
667
668
669/**
670 * Reads the VM-exit instruction length field from the VMCS into the VMX
671 * transient structure.
672 *
673 * @returns VBox status code.
674 * @param pVmxTransient Pointer to the VMX transient structure.
675 */
676DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
677{
678 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
679 {
680 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
681 AssertRCReturn(rc, rc);
682 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
683 }
684 return VINF_SUCCESS;
685}
686
687
688/**
689 * Reads the VM-exit instruction-information field from the VMCS into
690 * the VMX transient structure.
691 *
692 * @returns VBox status code.
693 * @param pVmxTransient Pointer to the VMX transient structure.
694 */
695DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
696{
697 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
698 {
699 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
700 AssertRCReturn(rc, rc);
701 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
702 }
703 return VINF_SUCCESS;
704}
705
706
707/**
708 * Reads the exit code qualification from the VMCS into the VMX transient
709 * structure.
710 *
711 * @returns VBox status code.
712 * @param pVCpu The cross context virtual CPU structure of the
713 * calling EMT. (Required for the VMCS cache case.)
714 * @param pVmxTransient Pointer to the VMX transient structure.
715 */
716DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
717{
718 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
719 {
720 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
721 AssertRCReturn(rc, rc);
722 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
723 }
724 return VINF_SUCCESS;
725}
726
727
728/**
729 * Reads the IDT-vectoring information field from the VMCS into the VMX
730 * transient structure.
731 *
732 * @returns VBox status code.
733 * @param pVmxTransient Pointer to the VMX transient structure.
734 *
735 * @remarks No-long-jump zone!!!
736 */
737DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
738{
739 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
740 {
741 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
742 AssertRCReturn(rc, rc);
743 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
744 }
745 return VINF_SUCCESS;
746}
747
748
749/**
750 * Reads the IDT-vectoring error code from the VMCS into the VMX
751 * transient structure.
752 *
753 * @returns VBox status code.
754 * @param pVmxTransient Pointer to the VMX transient structure.
755 */
756DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
757{
758 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
759 {
760 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
761 AssertRCReturn(rc, rc);
762 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
763 }
764 return VINF_SUCCESS;
765}
766
767
768/**
769 * Enters VMX root mode operation on the current CPU.
770 *
771 * @returns VBox status code.
772 * @param pVM The cross context VM structure. Can be
773 * NULL, after a resume.
774 * @param HCPhysCpuPage Physical address of the VMXON region.
775 * @param pvCpuPage Pointer to the VMXON region.
776 */
777static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
778{
779 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
780 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
781 Assert(pvCpuPage);
782 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
783
784 if (pVM)
785 {
786 /* Write the VMCS revision dword to the VMXON region. */
787 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
788 }
789
790 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
791 RTCCUINTREG fEFlags = ASMIntDisableFlags();
792
793 /* Enable the VMX bit in CR4 if necessary. */
794 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
795
796 /* Enter VMX root mode. */
797 int rc = VMXEnable(HCPhysCpuPage);
798 if (RT_FAILURE(rc))
799 {
800 if (!(uOldCr4 & X86_CR4_VMXE))
801 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
802
803 if (pVM)
804 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
805 }
806
807 /* Restore interrupts. */
808 ASMSetFlags(fEFlags);
809 return rc;
810}
811
812
813/**
814 * Exits VMX root mode operation on the current CPU.
815 *
816 * @returns VBox status code.
817 */
818static int hmR0VmxLeaveRootMode(void)
819{
820 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
821
822 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
823 RTCCUINTREG fEFlags = ASMIntDisableFlags();
824
825 /* If we're for some reason not in VMX root mode, then don't leave it. */
826 RTCCUINTREG uHostCR4 = ASMGetCR4();
827
828 int rc;
829 if (uHostCR4 & X86_CR4_VMXE)
830 {
831 /* Exit VMX root mode and clear the VMX bit in CR4. */
832 VMXDisable();
833 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
834 rc = VINF_SUCCESS;
835 }
836 else
837 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
838
839 /* Restore interrupts. */
840 ASMSetFlags(fEFlags);
841 return rc;
842}
843
844
845/**
846 * Allocates and maps one physically contiguous page. The allocated page is
847 * zero'd out. (Used by various VT-x structures).
848 *
849 * @returns IPRT status code.
850 * @param pMemObj Pointer to the ring-0 memory object.
851 * @param ppVirt Where to store the virtual address of the
852 * allocation.
853 * @param pHCPhys Where to store the physical address of the
854 * allocation.
855 */
856DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
857{
858 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
859 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
860 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
861
862 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
863 if (RT_FAILURE(rc))
864 return rc;
865 *ppVirt = RTR0MemObjAddress(*pMemObj);
866 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
867 ASMMemZero32(*ppVirt, PAGE_SIZE);
868 return VINF_SUCCESS;
869}
870
871
872/**
873 * Frees and unmaps an allocated physical page.
874 *
875 * @param pMemObj Pointer to the ring-0 memory object.
876 * @param ppVirt Where to re-initialize the virtual address of
877 * allocation as 0.
878 * @param pHCPhys Where to re-initialize the physical address of the
879 * allocation as 0.
880 */
881DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
882{
883 AssertPtr(pMemObj);
884 AssertPtr(ppVirt);
885 AssertPtr(pHCPhys);
886 if (*pMemObj != NIL_RTR0MEMOBJ)
887 {
888 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
889 AssertRC(rc);
890 *pMemObj = NIL_RTR0MEMOBJ;
891 *ppVirt = 0;
892 *pHCPhys = 0;
893 }
894}
895
896
897/**
898 * Worker function to free VT-x related structures.
899 *
900 * @returns IPRT status code.
901 * @param pVM The cross context VM structure.
902 */
903static void hmR0VmxStructsFree(PVM pVM)
904{
905 for (VMCPUID i = 0; i < pVM->cCpus; i++)
906 {
907 PVMCPU pVCpu = &pVM->aCpus[i];
908 AssertPtr(pVCpu);
909
910 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
911 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
912
913 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
914 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
915
916 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
917 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
918 }
919
920 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
921#ifdef VBOX_WITH_CRASHDUMP_MAGIC
922 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
923#endif
924}
925
926
927/**
928 * Worker function to allocate VT-x related VM structures.
929 *
930 * @returns IPRT status code.
931 * @param pVM The cross context VM structure.
932 */
933static int hmR0VmxStructsAlloc(PVM pVM)
934{
935 /*
936 * Initialize members up-front so we can cleanup properly on allocation failure.
937 */
938#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
939 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
940 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
941 pVM->hm.s.vmx.HCPhys##a_Name = 0;
942
943#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
944 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
945 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
946 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
947
948#ifdef VBOX_WITH_CRASHDUMP_MAGIC
949 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
950#endif
951 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
952
953 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
954 for (VMCPUID i = 0; i < pVM->cCpus; i++)
955 {
956 PVMCPU pVCpu = &pVM->aCpus[i];
957 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
958 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
959 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
960 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
961 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
962 }
963#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
964#undef VMXLOCAL_INIT_VM_MEMOBJ
965
966 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
967 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
968 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
969 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
970
971 /*
972 * Allocate all the VT-x structures.
973 */
974 int rc = VINF_SUCCESS;
975#ifdef VBOX_WITH_CRASHDUMP_MAGIC
976 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
977 if (RT_FAILURE(rc))
978 goto cleanup;
979 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
980 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
981#endif
982
983 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
984 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
985 {
986 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
987 &pVM->hm.s.vmx.HCPhysApicAccess);
988 if (RT_FAILURE(rc))
989 goto cleanup;
990 }
991
992 /*
993 * Initialize per-VCPU VT-x structures.
994 */
995 for (VMCPUID i = 0; i < pVM->cCpus; i++)
996 {
997 PVMCPU pVCpu = &pVM->aCpus[i];
998 AssertPtr(pVCpu);
999
1000 /* Allocate the VM control structure (VMCS). */
1001 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1002 if (RT_FAILURE(rc))
1003 goto cleanup;
1004
1005 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
1006 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1007 {
1008 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1009 &pVCpu->hm.s.vmx.HCPhysVirtApic);
1010 if (RT_FAILURE(rc))
1011 goto cleanup;
1012 }
1013
1014 /*
1015 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1016 * transparent accesses of specific MSRs.
1017 *
1018 * If the condition for enabling MSR bitmaps changes here, don't forget to
1019 * update HMAreMsrBitmapsAvailable().
1020 */
1021 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1022 {
1023 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1024 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1025 if (RT_FAILURE(rc))
1026 goto cleanup;
1027 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1028 }
1029
1030 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1031 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1032 if (RT_FAILURE(rc))
1033 goto cleanup;
1034
1035 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1036 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1037 if (RT_FAILURE(rc))
1038 goto cleanup;
1039 }
1040
1041 return VINF_SUCCESS;
1042
1043cleanup:
1044 hmR0VmxStructsFree(pVM);
1045 return rc;
1046}
1047
1048
1049/**
1050 * Does global VT-x initialization (called during module initialization).
1051 *
1052 * @returns VBox status code.
1053 */
1054VMMR0DECL(int) VMXR0GlobalInit(void)
1055{
1056#ifdef HMVMX_USE_FUNCTION_TABLE
1057 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1058# ifdef VBOX_STRICT
1059 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1060 Assert(g_apfnVMExitHandlers[i]);
1061# endif
1062#endif
1063 return VINF_SUCCESS;
1064}
1065
1066
1067/**
1068 * Does global VT-x termination (called during module termination).
1069 */
1070VMMR0DECL(void) VMXR0GlobalTerm()
1071{
1072 /* Nothing to do currently. */
1073}
1074
1075
1076/**
1077 * Sets up and activates VT-x on the current CPU.
1078 *
1079 * @returns VBox status code.
1080 * @param pCpu Pointer to the global CPU info struct.
1081 * @param pVM The cross context VM structure. Can be
1082 * NULL after a host resume operation.
1083 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1084 * fEnabledByHost is @c true).
1085 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1086 * @a fEnabledByHost is @c true).
1087 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1088 * enable VT-x on the host.
1089 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1090 */
1091VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1092 void *pvMsrs)
1093{
1094 Assert(pCpu);
1095 Assert(pvMsrs);
1096 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1097
1098 /* Enable VT-x if it's not already enabled by the host. */
1099 if (!fEnabledByHost)
1100 {
1101 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1102 if (RT_FAILURE(rc))
1103 return rc;
1104 }
1105
1106 /*
1107 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1108 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1109 */
1110 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1111 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1112 {
1113 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1114 pCpu->fFlushAsidBeforeUse = false;
1115 }
1116 else
1117 pCpu->fFlushAsidBeforeUse = true;
1118
1119 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1120 ++pCpu->cTlbFlushes;
1121
1122 return VINF_SUCCESS;
1123}
1124
1125
1126/**
1127 * Deactivates VT-x on the current CPU.
1128 *
1129 * @returns VBox status code.
1130 * @param pCpu Pointer to the global CPU info struct.
1131 * @param pvCpuPage Pointer to the VMXON region.
1132 * @param HCPhysCpuPage Physical address of the VMXON region.
1133 *
1134 * @remarks This function should never be called when SUPR0EnableVTx() or
1135 * similar was used to enable VT-x on the host.
1136 */
1137VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1138{
1139 NOREF(pCpu);
1140 NOREF(pvCpuPage);
1141 NOREF(HCPhysCpuPage);
1142
1143 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1144 return hmR0VmxLeaveRootMode();
1145}
1146
1147
1148/**
1149 * Sets the permission bits for the specified MSR in the MSR bitmap.
1150 *
1151 * @param pVCpu The cross context virtual CPU structure.
1152 * @param uMsr The MSR value.
1153 * @param enmRead Whether reading this MSR causes a VM-exit.
1154 * @param enmWrite Whether writing this MSR causes a VM-exit.
1155 */
1156static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1157{
1158 int32_t iBit;
1159 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1160
1161 /*
1162 * Layout:
1163 * 0x000 - 0x3ff - Low MSR read bits
1164 * 0x400 - 0x7ff - High MSR read bits
1165 * 0x800 - 0xbff - Low MSR write bits
1166 * 0xc00 - 0xfff - High MSR write bits
1167 */
1168 if (uMsr <= 0x00001FFF)
1169 iBit = uMsr;
1170 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1171 {
1172 iBit = uMsr - UINT32_C(0xC0000000);
1173 pbMsrBitmap += 0x400;
1174 }
1175 else
1176 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1177
1178 Assert(iBit <= 0x1fff);
1179 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1180 ASMBitSet(pbMsrBitmap, iBit);
1181 else
1182 ASMBitClear(pbMsrBitmap, iBit);
1183
1184 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1185 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1186 else
1187 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1188}
1189
1190
1191#ifdef VBOX_STRICT
1192/**
1193 * Gets the permission bits for the specified MSR in the MSR bitmap.
1194 *
1195 * @returns VBox status code.
1196 * @retval VINF_SUCCESS if the specified MSR is found.
1197 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1198 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1199 *
1200 * @param pVCpu The cross context virtual CPU structure.
1201 * @param uMsr The MSR.
1202 * @param penmRead Where to store the read permissions.
1203 * @param penmWrite Where to store the write permissions.
1204 */
1205static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1206{
1207 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1208 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1209 int32_t iBit;
1210 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1211
1212 /* See hmR0VmxSetMsrPermission() for the layout. */
1213 if (uMsr <= 0x00001FFF)
1214 iBit = uMsr;
1215 else if ( uMsr >= 0xC0000000
1216 && uMsr <= 0xC0001FFF)
1217 {
1218 iBit = (uMsr - 0xC0000000);
1219 pbMsrBitmap += 0x400;
1220 }
1221 else
1222 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1223
1224 Assert(iBit <= 0x1fff);
1225 if (ASMBitTest(pbMsrBitmap, iBit))
1226 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1227 else
1228 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1229
1230 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1231 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1232 else
1233 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1234 return VINF_SUCCESS;
1235}
1236#endif /* VBOX_STRICT */
1237
1238
1239/**
1240 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1241 * area.
1242 *
1243 * @returns VBox status code.
1244 * @param pVCpu The cross context virtual CPU structure.
1245 * @param cMsrs The number of MSRs.
1246 */
1247DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1248{
1249 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1250 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1251 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1252 {
1253 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1254 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1255 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1256 }
1257
1258 /* Update number of guest MSRs to load/store across the world-switch. */
1259 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1260 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1261
1262 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1263 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1264 AssertRCReturn(rc, rc);
1265
1266 /* Update the VCPU's copy of the MSR count. */
1267 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1268
1269 return VINF_SUCCESS;
1270}
1271
1272
1273/**
1274 * Adds a new (or updates the value of an existing) guest/host MSR
1275 * pair to be swapped during the world-switch as part of the
1276 * auto-load/store MSR area in the VMCS.
1277 *
1278 * @returns VBox status code.
1279 * @param pVCpu The cross context virtual CPU structure.
1280 * @param uMsr The MSR.
1281 * @param uGuestMsrValue Value of the guest MSR.
1282 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1283 * necessary.
1284 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1285 * its value was updated. Optional, can be NULL.
1286 */
1287static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1288 bool *pfAddedAndUpdated)
1289{
1290 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1291 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1292 uint32_t i;
1293 for (i = 0; i < cMsrs; i++)
1294 {
1295 if (pGuestMsr->u32Msr == uMsr)
1296 break;
1297 pGuestMsr++;
1298 }
1299
1300 bool fAdded = false;
1301 if (i == cMsrs)
1302 {
1303 ++cMsrs;
1304 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1305 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1306
1307 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1308 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1309 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1310
1311 fAdded = true;
1312 }
1313
1314 /* Update the MSR values in the auto-load/store MSR area. */
1315 pGuestMsr->u32Msr = uMsr;
1316 pGuestMsr->u64Value = uGuestMsrValue;
1317
1318 /* Create/update the MSR slot in the host MSR area. */
1319 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1320 pHostMsr += i;
1321 pHostMsr->u32Msr = uMsr;
1322
1323 /*
1324 * Update the host MSR only when requested by the caller AND when we're
1325 * adding it to the auto-load/store area. Otherwise, it would have been
1326 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1327 */
1328 bool fUpdatedMsrValue = false;
1329 if ( fAdded
1330 && fUpdateHostMsr)
1331 {
1332 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1333 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1334 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1335 fUpdatedMsrValue = true;
1336 }
1337
1338 if (pfAddedAndUpdated)
1339 *pfAddedAndUpdated = fUpdatedMsrValue;
1340 return VINF_SUCCESS;
1341}
1342
1343
1344/**
1345 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1346 * auto-load/store MSR area in the VMCS.
1347 *
1348 * @returns VBox status code.
1349 * @param pVCpu The cross context virtual CPU structure.
1350 * @param uMsr The MSR.
1351 */
1352static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1353{
1354 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1355 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1356 for (uint32_t i = 0; i < cMsrs; i++)
1357 {
1358 /* Find the MSR. */
1359 if (pGuestMsr->u32Msr == uMsr)
1360 {
1361 /* If it's the last MSR, simply reduce the count. */
1362 if (i == cMsrs - 1)
1363 {
1364 --cMsrs;
1365 break;
1366 }
1367
1368 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1369 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1370 pLastGuestMsr += cMsrs - 1;
1371 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1372 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1373
1374 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1375 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1376 pLastHostMsr += cMsrs - 1;
1377 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1378 pHostMsr->u64Value = pLastHostMsr->u64Value;
1379 --cMsrs;
1380 break;
1381 }
1382 pGuestMsr++;
1383 }
1384
1385 /* Update the VMCS if the count changed (meaning the MSR was found). */
1386 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1387 {
1388 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1389 AssertRCReturn(rc, rc);
1390
1391 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1392 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1393 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1394
1395 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1396 return VINF_SUCCESS;
1397 }
1398
1399 return VERR_NOT_FOUND;
1400}
1401
1402
1403/**
1404 * Checks if the specified guest MSR is part of the auto-load/store area in
1405 * the VMCS.
1406 *
1407 * @returns true if found, false otherwise.
1408 * @param pVCpu The cross context virtual CPU structure.
1409 * @param uMsr The MSR to find.
1410 */
1411static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1412{
1413 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1414 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1415
1416 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1417 {
1418 if (pGuestMsr->u32Msr == uMsr)
1419 return true;
1420 }
1421 return false;
1422}
1423
1424
1425/**
1426 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1427 *
1428 * @param pVCpu The cross context virtual CPU structure.
1429 *
1430 * @remarks No-long-jump zone!!!
1431 */
1432static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1433{
1434 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1435 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1436 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1437 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1438
1439 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1440 {
1441 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1442
1443 /*
1444 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1445 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1446 */
1447 if (pHostMsr->u32Msr == MSR_K6_EFER)
1448 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1449 else
1450 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1451 }
1452
1453 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1454}
1455
1456
1457#if HC_ARCH_BITS == 64
1458/**
1459 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1460 * perform lazy restoration of the host MSRs while leaving VT-x.
1461 *
1462 * @param pVCpu The cross context virtual CPU structure.
1463 *
1464 * @remarks No-long-jump zone!!!
1465 */
1466static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1467{
1468 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1469
1470 /*
1471 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1472 */
1473 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1474 {
1475 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1476 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1477 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1478 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1479 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1480 }
1481}
1482
1483
1484/**
1485 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1486 * lazily while leaving VT-x.
1487 *
1488 * @returns true if it does, false otherwise.
1489 * @param pVCpu The cross context virtual CPU structure.
1490 * @param uMsr The MSR to check.
1491 */
1492static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1493{
1494 NOREF(pVCpu);
1495 switch (uMsr)
1496 {
1497 case MSR_K8_LSTAR:
1498 case MSR_K6_STAR:
1499 case MSR_K8_SF_MASK:
1500 case MSR_K8_KERNEL_GS_BASE:
1501 return true;
1502 }
1503 return false;
1504}
1505
1506
1507/**
1508 * Saves a set of guest MSRs back into the guest-CPU context.
1509 *
1510 * @param pVCpu The cross context virtual CPU structure.
1511 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1512 * out-of-sync. Make sure to update the required fields
1513 * before using them.
1514 *
1515 * @remarks No-long-jump zone!!!
1516 */
1517static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1518{
1519 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1520 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1521
1522 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1523 {
1524 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1525 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1526 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1527 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1528 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1529 }
1530}
1531
1532
1533/**
1534 * Loads a set of guests MSRs to allow read/passthru to the guest.
1535 *
1536 * The name of this function is slightly confusing. This function does NOT
1537 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1538 * common prefix for functions dealing with "lazy restoration" of the shared
1539 * MSRs.
1540 *
1541 * @param pVCpu The cross context virtual CPU structure.
1542 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1543 * out-of-sync. Make sure to update the required fields
1544 * before using them.
1545 *
1546 * @remarks No-long-jump zone!!!
1547 */
1548static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1549{
1550 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1551 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1552
1553#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1554 do { \
1555 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1556 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1557 else \
1558 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1559 } while (0)
1560
1561 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1562 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1563 {
1564 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1565 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1566 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1567 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1568 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1569 }
1570 else
1571 {
1572 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1573 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1574 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1575 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1576 }
1577
1578#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1579}
1580
1581
1582/**
1583 * Performs lazy restoration of the set of host MSRs if they were previously
1584 * loaded with guest MSR values.
1585 *
1586 * @param pVCpu The cross context virtual CPU structure.
1587 *
1588 * @remarks No-long-jump zone!!!
1589 * @remarks The guest MSRs should have been saved back into the guest-CPU
1590 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1591 */
1592static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1593{
1594 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1595 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1596
1597 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1598 {
1599 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1600 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1601 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1602 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1603 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1604 }
1605 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1606}
1607#endif /* HC_ARCH_BITS == 64 */
1608
1609
1610/**
1611 * Verifies that our cached values of the VMCS controls are all
1612 * consistent with what's actually present in the VMCS.
1613 *
1614 * @returns VBox status code.
1615 * @param pVCpu The cross context virtual CPU structure.
1616 */
1617static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1618{
1619 uint32_t u32Val;
1620 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1621 AssertRCReturn(rc, rc);
1622 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1623 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1624
1625 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1626 AssertRCReturn(rc, rc);
1627 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1628 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1629
1630 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1631 AssertRCReturn(rc, rc);
1632 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1633 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1634
1635 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1636 AssertRCReturn(rc, rc);
1637 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1638 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1639
1640 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1641 {
1642 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1643 AssertRCReturn(rc, rc);
1644 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1645 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1646 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1647 }
1648
1649 return VINF_SUCCESS;
1650}
1651
1652
1653#ifdef VBOX_STRICT
1654/**
1655 * Verifies that our cached host EFER value has not changed
1656 * since we cached it.
1657 *
1658 * @param pVCpu The cross context virtual CPU structure.
1659 */
1660static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1661{
1662 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1663
1664 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1665 {
1666 uint64_t u64Val;
1667 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, &u64Val);
1668 AssertRC(rc);
1669
1670 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1671 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1672 }
1673}
1674
1675
1676/**
1677 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1678 * VMCS are correct.
1679 *
1680 * @param pVCpu The cross context virtual CPU structure.
1681 */
1682static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1683{
1684 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1685
1686 /* Verify MSR counts in the VMCS are what we think it should be. */
1687 uint32_t cMsrs;
1688 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1689 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1690
1691 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1692 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1693
1694 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1695 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1696
1697 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1698 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1699 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1700 {
1701 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1702 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1703 pGuestMsr->u32Msr, cMsrs));
1704
1705 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1706 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1707 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1708
1709 /* Verify that the permissions are as expected in the MSR bitmap. */
1710 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1711 {
1712 VMXMSREXITREAD enmRead;
1713 VMXMSREXITWRITE enmWrite;
1714 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1715 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1716 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1717 {
1718 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1719 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1720 }
1721 else
1722 {
1723 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1724 pGuestMsr->u32Msr, cMsrs));
1725 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1726 pGuestMsr->u32Msr, cMsrs));
1727 }
1728 }
1729 }
1730}
1731#endif /* VBOX_STRICT */
1732
1733
1734/**
1735 * Flushes the TLB using EPT.
1736 *
1737 * @returns VBox status code.
1738 * @param pVCpu The cross context virtual CPU structure of the calling
1739 * EMT. Can be NULL depending on @a enmFlush.
1740 * @param enmFlush Type of flush.
1741 *
1742 * @remarks Caller is responsible for making sure this function is called only
1743 * when NestedPaging is supported and providing @a enmFlush that is
1744 * supported by the CPU.
1745 * @remarks Can be called with interrupts disabled.
1746 */
1747static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1748{
1749 uint64_t au64Descriptor[2];
1750 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1751 au64Descriptor[0] = 0;
1752 else
1753 {
1754 Assert(pVCpu);
1755 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1756 }
1757 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1758
1759 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1760 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1761 rc));
1762 if ( RT_SUCCESS(rc)
1763 && pVCpu)
1764 {
1765 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1766 }
1767}
1768
1769
1770/**
1771 * Flushes the TLB using VPID.
1772 *
1773 * @returns VBox status code.
1774 * @param pVM The cross context VM structure.
1775 * @param pVCpu The cross context virtual CPU structure of the calling
1776 * EMT. Can be NULL depending on @a enmFlush.
1777 * @param enmFlush Type of flush.
1778 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1779 * on @a enmFlush).
1780 *
1781 * @remarks Can be called with interrupts disabled.
1782 */
1783static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1784{
1785 NOREF(pVM);
1786 AssertPtr(pVM);
1787 Assert(pVM->hm.s.vmx.fVpid);
1788
1789 uint64_t au64Descriptor[2];
1790 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1791 {
1792 au64Descriptor[0] = 0;
1793 au64Descriptor[1] = 0;
1794 }
1795 else
1796 {
1797 AssertPtr(pVCpu);
1798 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1799 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1800 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1801 au64Descriptor[1] = GCPtr;
1802 }
1803
1804 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1805 AssertMsg(rc == VINF_SUCCESS,
1806 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1807 if ( RT_SUCCESS(rc)
1808 && pVCpu)
1809 {
1810 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1811 }
1812}
1813
1814
1815/**
1816 * Invalidates a guest page by guest virtual address. Only relevant for
1817 * EPT/VPID, otherwise there is nothing really to invalidate.
1818 *
1819 * @returns VBox status code.
1820 * @param pVM The cross context VM structure.
1821 * @param pVCpu The cross context virtual CPU structure.
1822 * @param GCVirt Guest virtual address of the page to invalidate.
1823 */
1824VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1825{
1826 AssertPtr(pVM);
1827 AssertPtr(pVCpu);
1828 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1829
1830 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1831 if (!fFlushPending)
1832 {
1833 /*
1834 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1835 * See @bugref{6043} and @bugref{6177}.
1836 *
1837 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1838 * function maybe called in a loop with individual addresses.
1839 */
1840 if (pVM->hm.s.vmx.fVpid)
1841 {
1842 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1843 {
1844 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1845 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1846 }
1847 else
1848 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1849 }
1850 else if (pVM->hm.s.fNestedPaging)
1851 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1852 }
1853
1854 return VINF_SUCCESS;
1855}
1856
1857
1858/**
1859 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1860 * otherwise there is nothing really to invalidate.
1861 *
1862 * @returns VBox status code.
1863 * @param pVM The cross context VM structure.
1864 * @param pVCpu The cross context virtual CPU structure.
1865 * @param GCPhys Guest physical address of the page to invalidate.
1866 */
1867VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1868{
1869 NOREF(pVM); NOREF(GCPhys);
1870 LogFlowFunc(("%RGp\n", GCPhys));
1871
1872 /*
1873 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1874 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1875 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1876 */
1877 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1878 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1879 return VINF_SUCCESS;
1880}
1881
1882
1883/**
1884 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1885 * case where neither EPT nor VPID is supported by the CPU.
1886 *
1887 * @param pVM The cross context VM structure.
1888 * @param pVCpu The cross context virtual CPU structure.
1889 * @param pCpu Pointer to the global HM struct.
1890 *
1891 * @remarks Called with interrupts disabled.
1892 */
1893static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1894{
1895 AssertPtr(pVCpu);
1896 AssertPtr(pCpu);
1897 NOREF(pVM);
1898
1899 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1900
1901 Assert(pCpu->idCpu != NIL_RTCPUID);
1902 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1903 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1904 pVCpu->hm.s.fForceTLBFlush = false;
1905 return;
1906}
1907
1908
1909/**
1910 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1911 *
1912 * @param pVM The cross context VM structure.
1913 * @param pVCpu The cross context virtual CPU structure.
1914 * @param pCpu Pointer to the global HM CPU struct.
1915 * @remarks All references to "ASID" in this function pertains to "VPID" in
1916 * Intel's nomenclature. The reason is, to avoid confusion in compare
1917 * statements since the host-CPU copies are named "ASID".
1918 *
1919 * @remarks Called with interrupts disabled.
1920 */
1921static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1922{
1923#ifdef VBOX_WITH_STATISTICS
1924 bool fTlbFlushed = false;
1925# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1926# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1927 if (!fTlbFlushed) \
1928 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1929 } while (0)
1930#else
1931# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1932# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1933#endif
1934
1935 AssertPtr(pVM);
1936 AssertPtr(pCpu);
1937 AssertPtr(pVCpu);
1938 Assert(pCpu->idCpu != NIL_RTCPUID);
1939
1940 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1941 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1942 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1943
1944 /*
1945 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1946 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1947 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1948 */
1949 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1950 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1951 {
1952 ++pCpu->uCurrentAsid;
1953 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1954 {
1955 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1956 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1957 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1958 }
1959
1960 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1961 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1962 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1963
1964 /*
1965 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1966 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1967 */
1968 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1969 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1970 HMVMX_SET_TAGGED_TLB_FLUSHED();
1971 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1972 }
1973
1974 /* Check for explicit TLB flushes. */
1975 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1976 {
1977 /*
1978 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1979 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1980 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1981 * but not guest-physical mappings.
1982 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1983 */
1984 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1985 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1986 HMVMX_SET_TAGGED_TLB_FLUSHED();
1987 }
1988
1989 pVCpu->hm.s.fForceTLBFlush = false;
1990 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1991
1992 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1993 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1994 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1995 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1996 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1997 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
1998 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
1999 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2000 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2001
2002 /* Update VMCS with the VPID. */
2003 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2004 AssertRC(rc);
2005
2006#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2007}
2008
2009
2010/**
2011 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2012 *
2013 * @returns VBox status code.
2014 * @param pVM The cross context VM structure.
2015 * @param pVCpu The cross context virtual CPU structure.
2016 * @param pCpu Pointer to the global HM CPU struct.
2017 *
2018 * @remarks Called with interrupts disabled.
2019 */
2020static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2021{
2022 AssertPtr(pVM);
2023 AssertPtr(pVCpu);
2024 AssertPtr(pCpu);
2025 Assert(pCpu->idCpu != NIL_RTCPUID);
2026 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2027 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2028
2029 /*
2030 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2031 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2032 */
2033 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2034 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2035 {
2036 pVCpu->hm.s.fForceTLBFlush = true;
2037 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2038 }
2039
2040 /* Check for explicit TLB flushes. */
2041 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2042 {
2043 pVCpu->hm.s.fForceTLBFlush = true;
2044 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2045 }
2046
2047 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2048 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2049
2050 if (pVCpu->hm.s.fForceTLBFlush)
2051 {
2052 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2053 pVCpu->hm.s.fForceTLBFlush = false;
2054 }
2055}
2056
2057
2058/**
2059 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2060 *
2061 * @returns VBox status code.
2062 * @param pVM The cross context VM structure.
2063 * @param pVCpu The cross context virtual CPU structure.
2064 * @param pCpu Pointer to the global HM CPU struct.
2065 *
2066 * @remarks Called with interrupts disabled.
2067 */
2068static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2069{
2070 AssertPtr(pVM);
2071 AssertPtr(pVCpu);
2072 AssertPtr(pCpu);
2073 Assert(pCpu->idCpu != NIL_RTCPUID);
2074 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2075 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2076
2077 /*
2078 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2079 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2080 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2081 */
2082 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2083 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2084 {
2085 pVCpu->hm.s.fForceTLBFlush = true;
2086 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2087 }
2088
2089 /* Check for explicit TLB flushes. */
2090 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2091 {
2092 /*
2093 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2094 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2095 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2096 */
2097 pVCpu->hm.s.fForceTLBFlush = true;
2098 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2099 }
2100
2101 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2102 if (pVCpu->hm.s.fForceTLBFlush)
2103 {
2104 ++pCpu->uCurrentAsid;
2105 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2106 {
2107 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2108 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2109 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2110 }
2111
2112 pVCpu->hm.s.fForceTLBFlush = false;
2113 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2114 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2115 if (pCpu->fFlushAsidBeforeUse)
2116 {
2117 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2118 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2119 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2120 {
2121 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2122 pCpu->fFlushAsidBeforeUse = false;
2123 }
2124 else
2125 {
2126 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2127 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2128 }
2129 }
2130 }
2131
2132 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2133 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2134 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2135 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2136 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2137 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2138 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2139
2140 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2141 AssertRC(rc);
2142}
2143
2144
2145/**
2146 * Flushes the guest TLB entry based on CPU capabilities.
2147 *
2148 * @param pVCpu The cross context virtual CPU structure.
2149 * @param pCpu Pointer to the global HM CPU struct.
2150 */
2151DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2152{
2153#ifdef HMVMX_ALWAYS_FLUSH_TLB
2154 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2155#endif
2156 PVM pVM = pVCpu->CTX_SUFF(pVM);
2157 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2158 {
2159 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2160 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2161 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2162 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2163 default:
2164 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2165 break;
2166 }
2167
2168 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2169}
2170
2171
2172/**
2173 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2174 * TLB entries from the host TLB before VM-entry.
2175 *
2176 * @returns VBox status code.
2177 * @param pVM The cross context VM structure.
2178 */
2179static int hmR0VmxSetupTaggedTlb(PVM pVM)
2180{
2181 /*
2182 * Determine optimal flush type for Nested Paging.
2183 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2184 * guest execution (see hmR3InitFinalizeR0()).
2185 */
2186 if (pVM->hm.s.fNestedPaging)
2187 {
2188 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2189 {
2190 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2191 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2192 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2193 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2194 else
2195 {
2196 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2197 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2198 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2199 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2200 }
2201
2202 /* Make sure the write-back cacheable memory type for EPT is supported. */
2203 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2204 {
2205 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2206 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2207 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2208 }
2209
2210 /* EPT requires a page-walk length of 4. */
2211 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2212 {
2213 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2214 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2215 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2216 }
2217 }
2218 else
2219 {
2220 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2221 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2222 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2223 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2224 }
2225 }
2226
2227 /*
2228 * Determine optimal flush type for VPID.
2229 */
2230 if (pVM->hm.s.vmx.fVpid)
2231 {
2232 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2233 {
2234 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2235 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2236 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2237 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2238 else
2239 {
2240 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2241 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2242 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2243 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2244 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2245 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2246 pVM->hm.s.vmx.fVpid = false;
2247 }
2248 }
2249 else
2250 {
2251 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2252 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2253 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2254 pVM->hm.s.vmx.fVpid = false;
2255 }
2256 }
2257
2258 /*
2259 * Setup the handler for flushing tagged-TLBs.
2260 */
2261 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2262 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2263 else if (pVM->hm.s.fNestedPaging)
2264 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2265 else if (pVM->hm.s.vmx.fVpid)
2266 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2267 else
2268 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2269 return VINF_SUCCESS;
2270}
2271
2272
2273/**
2274 * Sets up pin-based VM-execution controls in the VMCS.
2275 *
2276 * @returns VBox status code.
2277 * @param pVM The cross context VM structure.
2278 * @param pVCpu The cross context virtual CPU structure.
2279 */
2280static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2281{
2282 AssertPtr(pVM);
2283 AssertPtr(pVCpu);
2284
2285 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2286 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2287
2288 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2289 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2290
2291 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2292 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2293
2294 /* Enable the VMX preemption timer. */
2295 if (pVM->hm.s.vmx.fUsePreemptTimer)
2296 {
2297 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2298 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2299 }
2300
2301 if ((val & zap) != val)
2302 {
2303 LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2304 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2305 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2306 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2307 }
2308
2309 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2310 AssertRCReturn(rc, rc);
2311
2312 pVCpu->hm.s.vmx.u32PinCtls = val;
2313 return rc;
2314}
2315
2316
2317/**
2318 * Sets up processor-based VM-execution controls in the VMCS.
2319 *
2320 * @returns VBox status code.
2321 * @param pVM The cross context VM structure.
2322 * @param pVCpu The cross context virtual CPU structure.
2323 */
2324static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2325{
2326 AssertPtr(pVM);
2327 AssertPtr(pVCpu);
2328
2329 int rc = VERR_INTERNAL_ERROR_5;
2330 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2331 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2332
2333 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2334 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2335 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2336 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2337 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2338 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2339 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2340
2341 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2342 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2343 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2344 {
2345 LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2346 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2347 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2348 }
2349
2350 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2351 if (!pVM->hm.s.fNestedPaging)
2352 {
2353 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2354 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2355 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2356 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2357 }
2358
2359 /* Use TPR shadowing if supported by the CPU. */
2360 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2361 {
2362 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2363 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2364 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2365 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2366 AssertRCReturn(rc, rc);
2367
2368 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2369 /* CR8 writes cause a VM-exit based on TPR threshold. */
2370 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2371 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2372 }
2373 else
2374 {
2375 /*
2376 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2377 * Set this control only for 64-bit guests.
2378 */
2379 if (pVM->hm.s.fAllow64BitGuests)
2380 {
2381 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2382 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2383 }
2384 }
2385
2386 /* Use MSR-bitmaps if supported by the CPU. */
2387 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2388 {
2389 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2390
2391 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2392 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2393 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2394 AssertRCReturn(rc, rc);
2395
2396 /*
2397 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2398 * automatically using dedicated fields in the VMCS.
2399 */
2400 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2401 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2402 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2403 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2404 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2405
2406#if HC_ARCH_BITS == 64
2407 /*
2408 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2409 */
2410 if (pVM->hm.s.fAllow64BitGuests)
2411 {
2412 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2413 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2414 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2415 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2416 }
2417#endif
2418 }
2419
2420 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2421 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2422 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2423
2424 if ((val & zap) != val)
2425 {
2426 LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2427 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2428 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2429 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2430 }
2431
2432 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2433 AssertRCReturn(rc, rc);
2434
2435 pVCpu->hm.s.vmx.u32ProcCtls = val;
2436
2437 /*
2438 * Secondary processor-based VM-execution controls.
2439 */
2440 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2441 {
2442 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2443 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2444
2445 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2446 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2447
2448 if (pVM->hm.s.fNestedPaging)
2449 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2450 else
2451 {
2452 /*
2453 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2454 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2455 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2456 */
2457 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2458 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2459 }
2460
2461 if (pVM->hm.s.vmx.fVpid)
2462 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2463
2464 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2465 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2466
2467 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2468 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2469 * done dynamically. */
2470 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2471 {
2472 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2473 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2474 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2475 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2476 AssertRCReturn(rc, rc);
2477 }
2478
2479 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2480 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2481
2482 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2483 && pVM->hm.s.vmx.cPleGapTicks
2484 && pVM->hm.s.vmx.cPleWindowTicks)
2485 {
2486 val |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2487
2488 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2489 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2490 AssertRCReturn(rc, rc);
2491 }
2492
2493 if ((val & zap) != val)
2494 {
2495 LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2496 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2497 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2498 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2499 }
2500
2501 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2502 AssertRCReturn(rc, rc);
2503
2504 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2505 }
2506 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2507 {
2508 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2509 "available\n"));
2510 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2511 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2512 }
2513
2514 return VINF_SUCCESS;
2515}
2516
2517
2518/**
2519 * Sets up miscellaneous (everything other than Pin & Processor-based
2520 * VM-execution) control fields in the VMCS.
2521 *
2522 * @returns VBox status code.
2523 * @param pVM The cross context VM structure.
2524 * @param pVCpu The cross context virtual CPU structure.
2525 */
2526static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2527{
2528 NOREF(pVM);
2529 AssertPtr(pVM);
2530 AssertPtr(pVCpu);
2531
2532 int rc = VERR_GENERAL_FAILURE;
2533
2534 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2535#if 0
2536 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2537 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2538 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2539
2540 /*
2541 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2542 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2543 * We thus use the exception bitmap to control it rather than use both.
2544 */
2545 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2546 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2547
2548 /** @todo Explore possibility of using IO-bitmaps. */
2549 /* All IO & IOIO instructions cause VM-exits. */
2550 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2551 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2552
2553 /* Initialize the MSR-bitmap area. */
2554 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2555 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2556 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2557 AssertRCReturn(rc, rc);
2558#endif
2559
2560 /* Setup MSR auto-load/store area. */
2561 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2562 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2563 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2564 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2565 AssertRCReturn(rc, rc);
2566
2567 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2568 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2569 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2570 AssertRCReturn(rc, rc);
2571
2572 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2573 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2574 AssertRCReturn(rc, rc);
2575
2576 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2577#if 0
2578 /* Setup debug controls */
2579 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2580 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2581 AssertRCReturn(rc, rc);
2582#endif
2583
2584 return rc;
2585}
2586
2587
2588/**
2589 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2590 *
2591 * @returns VBox status code.
2592 * @param pVM The cross context VM structure.
2593 * @param pVCpu The cross context virtual CPU structure.
2594 */
2595static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2596{
2597 AssertPtr(pVM);
2598 AssertPtr(pVCpu);
2599
2600 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2601
2602 uint32_t u32XcptBitmap = pVCpu->hm.s.fGIMTrapXcptUD ? RT_BIT(X86_XCPT_UD) : 0;
2603
2604 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2605 u32XcptBitmap |= RT_BIT_32(X86_XCPT_AC);
2606
2607 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2608 and writes, and because recursive #DBs can cause the CPU hang, we must always
2609 intercept #DB. */
2610 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2611
2612 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2613 if (!pVM->hm.s.fNestedPaging)
2614 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2615
2616 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2617 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2618 AssertRCReturn(rc, rc);
2619 return rc;
2620}
2621
2622
2623/**
2624 * Sets up the initial guest-state mask. The guest-state mask is consulted
2625 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2626 * for the nested virtualization case (as it would cause a VM-exit).
2627 *
2628 * @param pVCpu The cross context virtual CPU structure.
2629 */
2630static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2631{
2632 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2633 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2634 return VINF_SUCCESS;
2635}
2636
2637
2638/**
2639 * Does per-VM VT-x initialization.
2640 *
2641 * @returns VBox status code.
2642 * @param pVM The cross context VM structure.
2643 */
2644VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2645{
2646 LogFlowFunc(("pVM=%p\n", pVM));
2647
2648 int rc = hmR0VmxStructsAlloc(pVM);
2649 if (RT_FAILURE(rc))
2650 {
2651 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2652 return rc;
2653 }
2654
2655 return VINF_SUCCESS;
2656}
2657
2658
2659/**
2660 * Does per-VM VT-x termination.
2661 *
2662 * @returns VBox status code.
2663 * @param pVM The cross context VM structure.
2664 */
2665VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2666{
2667 LogFlowFunc(("pVM=%p\n", pVM));
2668
2669#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2670 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2671 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2672#endif
2673 hmR0VmxStructsFree(pVM);
2674 return VINF_SUCCESS;
2675}
2676
2677
2678/**
2679 * Sets up the VM for execution under VT-x.
2680 * This function is only called once per-VM during initialization.
2681 *
2682 * @returns VBox status code.
2683 * @param pVM The cross context VM structure.
2684 */
2685VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2686{
2687 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2688 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2689
2690 LogFlowFunc(("pVM=%p\n", pVM));
2691
2692 /*
2693 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2694 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2695 */
2696 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2697 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2698 || !pVM->hm.s.vmx.pRealModeTSS))
2699 {
2700 LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2701 return VERR_INTERNAL_ERROR;
2702 }
2703
2704 /* Initialize these always, see hmR3InitFinalizeR0().*/
2705 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2706 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2707
2708 /* Setup the tagged-TLB flush handlers. */
2709 int rc = hmR0VmxSetupTaggedTlb(pVM);
2710 if (RT_FAILURE(rc))
2711 {
2712 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2713 return rc;
2714 }
2715
2716 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2717 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2718#if HC_ARCH_BITS == 64
2719 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2720 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2721 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2722 {
2723 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2724 }
2725#endif
2726
2727 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2728 {
2729 PVMCPU pVCpu = &pVM->aCpus[i];
2730 AssertPtr(pVCpu);
2731 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2732
2733 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2734 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2735
2736 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2737 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2738 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2739
2740 /* Set revision dword at the beginning of the VMCS structure. */
2741 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2742
2743 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2744 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2745 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2746 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2747
2748 /* Load this VMCS as the current VMCS. */
2749 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2750 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2751 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2752
2753 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2754 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2755 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2756
2757 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2758 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2759 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2760
2761 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2762 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2763 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2764
2765 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2766 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2767 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2768
2769 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2770 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2771 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2772
2773#if HC_ARCH_BITS == 32
2774 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2775 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2776 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2777#endif
2778
2779 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2780 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2781 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2782 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2783
2784 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2785
2786 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2787 }
2788
2789 return VINF_SUCCESS;
2790}
2791
2792
2793/**
2794 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2795 * the VMCS.
2796 *
2797 * @returns VBox status code.
2798 * @param pVM The cross context VM structure.
2799 * @param pVCpu The cross context virtual CPU structure.
2800 */
2801DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2802{
2803 NOREF(pVM); NOREF(pVCpu);
2804
2805 RTCCUINTREG uReg = ASMGetCR0();
2806 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2807 AssertRCReturn(rc, rc);
2808
2809 uReg = ASMGetCR3();
2810 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2811 AssertRCReturn(rc, rc);
2812
2813 uReg = ASMGetCR4();
2814 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2815 AssertRCReturn(rc, rc);
2816 return rc;
2817}
2818
2819
2820#if HC_ARCH_BITS == 64
2821/**
2822 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2823 * requirements. See hmR0VmxSaveHostSegmentRegs().
2824 */
2825# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2826 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2827 { \
2828 bool fValidSelector = true; \
2829 if ((selValue) & X86_SEL_LDT) \
2830 { \
2831 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2832 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2833 } \
2834 if (fValidSelector) \
2835 { \
2836 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2837 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2838 } \
2839 (selValue) = 0; \
2840 }
2841#endif
2842
2843
2844/**
2845 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2846 * the host-state area in the VMCS.
2847 *
2848 * @returns VBox status code.
2849 * @param pVM The cross context VM structure.
2850 * @param pVCpu The cross context virtual CPU structure.
2851 */
2852DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2853{
2854 int rc = VERR_INTERNAL_ERROR_5;
2855
2856#if HC_ARCH_BITS == 64
2857 /*
2858 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2859 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2860 */
2861 AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2862 ("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2863#endif
2864
2865 /*
2866 * Host DS, ES, FS and GS segment registers.
2867 */
2868#if HC_ARCH_BITS == 64
2869 RTSEL uSelDS = ASMGetDS();
2870 RTSEL uSelES = ASMGetES();
2871 RTSEL uSelFS = ASMGetFS();
2872 RTSEL uSelGS = ASMGetGS();
2873#else
2874 RTSEL uSelDS = 0;
2875 RTSEL uSelES = 0;
2876 RTSEL uSelFS = 0;
2877 RTSEL uSelGS = 0;
2878#endif
2879
2880 /* Recalculate which host-state bits need to be manually restored. */
2881 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2882
2883 /*
2884 * Host CS and SS segment registers.
2885 */
2886 RTSEL uSelCS = ASMGetCS();
2887 RTSEL uSelSS = ASMGetSS();
2888
2889 /*
2890 * Host TR segment register.
2891 */
2892 RTSEL uSelTR = ASMGetTR();
2893
2894#if HC_ARCH_BITS == 64
2895 /*
2896 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2897 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2898 */
2899 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2900 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2901 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2902 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2903# undef VMXLOCAL_ADJUST_HOST_SEG
2904#endif
2905
2906 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2907 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2908 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2909 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2910 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2911 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2912 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2913 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2914 Assert(uSelCS);
2915 Assert(uSelTR);
2916
2917 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2918#if 0
2919 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2920 Assert(uSelSS != 0);
2921#endif
2922
2923 /* Write these host selector fields into the host-state area in the VMCS. */
2924 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS);
2925 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS);
2926#if HC_ARCH_BITS == 64
2927 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS);
2928 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES);
2929 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS);
2930 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS);
2931#else
2932 NOREF(uSelDS);
2933 NOREF(uSelES);
2934 NOREF(uSelFS);
2935 NOREF(uSelGS);
2936#endif
2937 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR);
2938 AssertRCReturn(rc, rc);
2939
2940 /*
2941 * Host GDTR and IDTR.
2942 */
2943 RTGDTR Gdtr;
2944 RTIDTR Idtr;
2945 RT_ZERO(Gdtr);
2946 RT_ZERO(Idtr);
2947 ASMGetGDTR(&Gdtr);
2948 ASMGetIDTR(&Idtr);
2949 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2950 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2951 AssertRCReturn(rc, rc);
2952
2953#if HC_ARCH_BITS == 64
2954 /*
2955 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2956 * maximum limit (0xffff) on every VM-exit.
2957 */
2958 if (Gdtr.cbGdt != 0xffff)
2959 {
2960 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2961 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2962 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2963 }
2964
2965 /*
2966 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
2967 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
2968 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
2969 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
2970 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
2971 * hosts where we are pretty sure it won't cause trouble.
2972 */
2973# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
2974 if (Idtr.cbIdt < 0x0fff)
2975# else
2976 if (Idtr.cbIdt != 0xffff)
2977# endif
2978 {
2979 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
2980 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2981 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2982 }
2983#endif
2984
2985 /*
2986 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2987 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2988 */
2989 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
2990 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
2991 VERR_VMX_INVALID_HOST_STATE);
2992
2993 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2994#if HC_ARCH_BITS == 64
2995 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
2996
2997 /*
2998 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2999 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3000 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3001 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3002 *
3003 * [1] See Intel spec. 3.5 "System Descriptor Types".
3004 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3005 */
3006 Assert(pDesc->System.u4Type == 11);
3007 if ( pDesc->System.u16LimitLow != 0x67
3008 || pDesc->System.u4LimitHigh)
3009 {
3010 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3011 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3012 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3013 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3014 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3015
3016 /* Store the GDTR here as we need it while restoring TR. */
3017 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3018 }
3019#else
3020 NOREF(pVM);
3021 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3022#endif
3023 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3024 AssertRCReturn(rc, rc);
3025
3026 /*
3027 * Host FS base and GS base.
3028 */
3029#if HC_ARCH_BITS == 64
3030 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3031 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3032 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3033 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3034 AssertRCReturn(rc, rc);
3035
3036 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3037 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3038 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3039 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3040 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3041#endif
3042 return rc;
3043}
3044
3045
3046/**
3047 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3048 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3049 * the host after every successful VM-exit.
3050 *
3051 * @returns VBox status code.
3052 * @param pVM The cross context VM structure.
3053 * @param pVCpu The cross context virtual CPU structure.
3054 *
3055 * @remarks No-long-jump zone!!!
3056 */
3057DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3058{
3059 NOREF(pVM);
3060
3061 AssertPtr(pVCpu);
3062 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3063
3064 int rc = VINF_SUCCESS;
3065#if HC_ARCH_BITS == 64
3066 if (pVM->hm.s.fAllow64BitGuests)
3067 hmR0VmxLazySaveHostMsrs(pVCpu);
3068#endif
3069
3070 /*
3071 * Host Sysenter MSRs.
3072 */
3073 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3074#if HC_ARCH_BITS == 32
3075 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3076 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3077#else
3078 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3079 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3080#endif
3081 AssertRCReturn(rc, rc);
3082
3083 /*
3084 * Host EFER MSR.
3085 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3086 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3087 */
3088 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3089 {
3090 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3091 AssertRCReturn(rc, rc);
3092 }
3093
3094 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3095 * hmR0VmxLoadGuestExitCtls() !! */
3096
3097 return rc;
3098}
3099
3100
3101/**
3102 * Figures out if we need to swap the EFER MSR which is
3103 * particularly expensive.
3104 *
3105 * We check all relevant bits. For now, that's everything
3106 * besides LMA/LME, as these two bits are handled by VM-entry,
3107 * see hmR0VmxLoadGuestExitCtls() and
3108 * hmR0VMxLoadGuestEntryCtls().
3109 *
3110 * @returns true if we need to load guest EFER, false otherwise.
3111 * @param pVCpu The cross context virtual CPU structure.
3112 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3113 * out-of-sync. Make sure to update the required fields
3114 * before using them.
3115 *
3116 * @remarks Requires EFER, CR4.
3117 * @remarks No-long-jump zone!!!
3118 */
3119static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3120{
3121#ifdef HMVMX_ALWAYS_SWAP_EFER
3122 return true;
3123#endif
3124
3125#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3126 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3127 if (CPUMIsGuestInLongMode(pVCpu))
3128 return false;
3129#endif
3130
3131 PVM pVM = pVCpu->CTX_SUFF(pVM);
3132 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3133 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3134
3135 /*
3136 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3137 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3138 */
3139 if ( CPUMIsGuestInLongMode(pVCpu)
3140 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3141 {
3142 return true;
3143 }
3144
3145 /*
3146 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3147 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3148 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3149 */
3150 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3151 && (pMixedCtx->cr0 & X86_CR0_PG)
3152 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3153 {
3154 /* Assert that host is PAE capable. */
3155 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3156 return true;
3157 }
3158
3159 /** @todo Check the latest Intel spec. for any other bits,
3160 * like SMEP/SMAP? */
3161 return false;
3162}
3163
3164
3165/**
3166 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3167 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3168 * controls".
3169 *
3170 * @returns VBox status code.
3171 * @param pVCpu The cross context virtual CPU structure.
3172 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3173 * out-of-sync. Make sure to update the required fields
3174 * before using them.
3175 *
3176 * @remarks Requires EFER.
3177 * @remarks No-long-jump zone!!!
3178 */
3179DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3180{
3181 int rc = VINF_SUCCESS;
3182 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3183 {
3184 PVM pVM = pVCpu->CTX_SUFF(pVM);
3185 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3186 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3187
3188 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3189 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3190
3191 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3192 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3193 {
3194 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3195 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3196 }
3197 else
3198 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3199
3200 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3201 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3202 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3203 {
3204 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3205 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3206 }
3207
3208 /*
3209 * The following should -not- be set (since we're not in SMM mode):
3210 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3211 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3212 */
3213
3214 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3215 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3216
3217 if ((val & zap) != val)
3218 {
3219 LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3220 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3221 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3222 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3223 }
3224
3225 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3226 AssertRCReturn(rc, rc);
3227
3228 pVCpu->hm.s.vmx.u32EntryCtls = val;
3229 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3230 }
3231 return rc;
3232}
3233
3234
3235/**
3236 * Sets up the VM-exit controls in the VMCS.
3237 *
3238 * @returns VBox status code.
3239 * @param pVCpu The cross context virtual CPU structure.
3240 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3241 * out-of-sync. Make sure to update the required fields
3242 * before using them.
3243 *
3244 * @remarks Requires EFER.
3245 */
3246DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3247{
3248 NOREF(pMixedCtx);
3249
3250 int rc = VINF_SUCCESS;
3251 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3252 {
3253 PVM pVM = pVCpu->CTX_SUFF(pVM);
3254 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3255 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3256
3257 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3258 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3259
3260 /*
3261 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3262 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3263 */
3264#if HC_ARCH_BITS == 64
3265 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3266 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3267#else
3268 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3269 {
3270 /* The switcher returns to long mode, EFER is managed by the switcher. */
3271 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3272 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3273 }
3274 else
3275 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3276#endif
3277
3278 /* If the newer VMCS fields for managing EFER exists, use it. */
3279 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3280 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3281 {
3282 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3283 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3284 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3285 }
3286
3287 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3288 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3289
3290 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3291 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3292 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3293
3294 if ( pVM->hm.s.vmx.fUsePreemptTimer
3295 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3296 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3297
3298 if ((val & zap) != val)
3299 {
3300 LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3301 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3302 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3303 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3304 }
3305
3306 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3307 AssertRCReturn(rc, rc);
3308
3309 pVCpu->hm.s.vmx.u32ExitCtls = val;
3310 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3311 }
3312 return rc;
3313}
3314
3315
3316/**
3317 * Loads the guest APIC and related state.
3318 *
3319 * @returns VBox status code.
3320 * @param pVCpu The cross context virtual CPU structure.
3321 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3322 * out-of-sync. Make sure to update the required fields
3323 * before using them.
3324 */
3325DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3326{
3327 NOREF(pMixedCtx);
3328
3329 int rc = VINF_SUCCESS;
3330 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3331 {
3332 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3333 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3334 {
3335 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3336
3337 bool fPendingIntr = false;
3338 uint8_t u8Tpr = 0;
3339 uint8_t u8PendingIntr = 0;
3340 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3341 AssertRCReturn(rc, rc);
3342
3343 /*
3344 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3345 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3346 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3347 * the interrupt when we VM-exit for other reasons.
3348 */
3349 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3350 uint32_t u32TprThreshold = 0;
3351 if (fPendingIntr)
3352 {
3353 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3354 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3355 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3356 if (u8PendingPriority <= u8TprPriority)
3357 u32TprThreshold = u8PendingPriority;
3358 else
3359 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3360 }
3361 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3362
3363 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3364 AssertRCReturn(rc, rc);
3365 }
3366
3367 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3368 }
3369 return rc;
3370}
3371
3372
3373/**
3374 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3375 *
3376 * @returns Guest's interruptibility-state.
3377 * @param pVCpu The cross context virtual CPU structure.
3378 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3379 * out-of-sync. Make sure to update the required fields
3380 * before using them.
3381 *
3382 * @remarks No-long-jump zone!!!
3383 */
3384DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3385{
3386 /*
3387 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3388 */
3389 uint32_t uIntrState = 0;
3390 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3391 {
3392 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3393 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3394 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3395 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3396 {
3397 if (pMixedCtx->eflags.Bits.u1IF)
3398 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3399 else
3400 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3401 }
3402 /* else: Although we can clear the force-flag here, let's keep this side-effects free. */
3403 }
3404
3405 /*
3406 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3407 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3408 * setting this would block host-NMIs and IRET will not clear the blocking.
3409 *
3410 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3411 */
3412 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3413 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3414 {
3415 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3416 }
3417
3418 return uIntrState;
3419}
3420
3421
3422/**
3423 * Loads the guest's interruptibility-state into the guest-state area in the
3424 * VMCS.
3425 *
3426 * @returns VBox status code.
3427 * @param pVCpu The cross context virtual CPU structure.
3428 * @param uIntrState The interruptibility-state to set.
3429 */
3430static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3431{
3432 NOREF(pVCpu);
3433 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3434 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3435 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3436 AssertRC(rc);
3437 return rc;
3438}
3439
3440
3441/**
3442 * Loads the exception intercepts required for guest execution in the VMCS.
3443 *
3444 * @returns VBox status code.
3445 * @param pVCpu The cross context virtual CPU structure.
3446 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3447 * out-of-sync. Make sure to update the required fields
3448 * before using them.
3449 */
3450static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3451{
3452 NOREF(pMixedCtx);
3453 int rc = VINF_SUCCESS;
3454 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3455 {
3456 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3457 if (pVCpu->hm.s.fGIMTrapXcptUD)
3458 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3459#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3460 else
3461 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3462#endif
3463
3464 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3465 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3466
3467 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3468 AssertRCReturn(rc, rc);
3469
3470 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3471 Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3472 pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3473 }
3474 return rc;
3475}
3476
3477
3478/**
3479 * Loads the guest's RIP into the guest-state area in the VMCS.
3480 *
3481 * @returns VBox status code.
3482 * @param pVCpu The cross context virtual CPU structure.
3483 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3484 * out-of-sync. Make sure to update the required fields
3485 * before using them.
3486 *
3487 * @remarks No-long-jump zone!!!
3488 */
3489static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3490{
3491 int rc = VINF_SUCCESS;
3492 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3493 {
3494 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3495 AssertRCReturn(rc, rc);
3496
3497 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3498 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3499 HMCPU_CF_VALUE(pVCpu)));
3500 }
3501 return rc;
3502}
3503
3504
3505/**
3506 * Loads the guest's RSP into the guest-state area in the VMCS.
3507 *
3508 * @returns VBox status code.
3509 * @param pVCpu The cross context virtual CPU structure.
3510 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3511 * out-of-sync. Make sure to update the required fields
3512 * before using them.
3513 *
3514 * @remarks No-long-jump zone!!!
3515 */
3516static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3517{
3518 int rc = VINF_SUCCESS;
3519 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3520 {
3521 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3522 AssertRCReturn(rc, rc);
3523
3524 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3525 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3526 }
3527 return rc;
3528}
3529
3530
3531/**
3532 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3533 *
3534 * @returns VBox status code.
3535 * @param pVCpu The cross context virtual CPU structure.
3536 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3537 * out-of-sync. Make sure to update the required fields
3538 * before using them.
3539 *
3540 * @remarks No-long-jump zone!!!
3541 */
3542static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3543{
3544 int rc = VINF_SUCCESS;
3545 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3546 {
3547 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3548 Let us assert it as such and use 32-bit VMWRITE. */
3549 Assert(!(pMixedCtx->rflags.u64 >> 32));
3550 X86EFLAGS Eflags = pMixedCtx->eflags;
3551 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3552 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3553 * These will never be cleared/set, unless some other part of the VMM
3554 * code is buggy - in which case we're better of finding and fixing
3555 * those bugs than hiding them. */
3556 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3557 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3558 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3559 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3560
3561 /*
3562 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3563 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3564 */
3565 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3566 {
3567 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3568 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3569 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3570 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3571 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3572 }
3573
3574 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3575 AssertRCReturn(rc, rc);
3576
3577 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3578 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3579 }
3580 return rc;
3581}
3582
3583
3584/**
3585 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3586 *
3587 * @returns VBox status code.
3588 * @param pVCpu The cross context virtual CPU structure.
3589 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3590 * out-of-sync. Make sure to update the required fields
3591 * before using them.
3592 *
3593 * @remarks No-long-jump zone!!!
3594 */
3595DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3596{
3597 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3598 rc |= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3599 rc |= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3600 AssertRCReturn(rc, rc);
3601 return rc;
3602}
3603
3604
3605/**
3606 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3607 * CR0 is partially shared with the host and we have to consider the FPU bits.
3608 *
3609 * @returns VBox status code.
3610 * @param pVCpu The cross context virtual CPU structure.
3611 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3612 * out-of-sync. Make sure to update the required fields
3613 * before using them.
3614 *
3615 * @remarks No-long-jump zone!!!
3616 */
3617static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3618{
3619 /*
3620 * Guest CR0.
3621 * Guest FPU.
3622 */
3623 int rc = VINF_SUCCESS;
3624 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3625 {
3626 Assert(!(pMixedCtx->cr0 >> 32));
3627 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3628 PVM pVM = pVCpu->CTX_SUFF(pVM);
3629
3630 /* The guest's view (read access) of its CR0 is unblemished. */
3631 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3632 AssertRCReturn(rc, rc);
3633 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3634
3635 /* Setup VT-x's view of the guest CR0. */
3636 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3637 if (pVM->hm.s.fNestedPaging)
3638 {
3639 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3640 {
3641 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3642 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3643 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3644 }
3645 else
3646 {
3647 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3648 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3649 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3650 }
3651
3652 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3653 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3654 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3655
3656 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3657 AssertRCReturn(rc, rc);
3658 }
3659 else
3660 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3661
3662 /*
3663 * Guest FPU bits.
3664 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3665 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3666 */
3667 u32GuestCR0 |= X86_CR0_NE;
3668 bool fInterceptNM = false;
3669 if (CPUMIsGuestFPUStateActive(pVCpu))
3670 {
3671 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3672 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3673 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3674 }
3675 else
3676 {
3677 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3678 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3679 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3680 }
3681
3682 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3683 bool fInterceptMF = false;
3684 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3685 fInterceptMF = true;
3686
3687 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3688 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3689 {
3690 Assert(PDMVmmDevHeapIsEnabled(pVM));
3691 Assert(pVM->hm.s.vmx.pRealModeTSS);
3692 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3693 fInterceptNM = true;
3694 fInterceptMF = true;
3695 }
3696 else
3697 {
3698 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3699 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3700 }
3701 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3702
3703 if (fInterceptNM)
3704 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3705 else
3706 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3707
3708 if (fInterceptMF)
3709 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3710 else
3711 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3712
3713 /* Additional intercepts for debugging, define these yourself explicitly. */
3714#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3715 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3716 | RT_BIT(X86_XCPT_BP)
3717 | RT_BIT(X86_XCPT_DE)
3718 | RT_BIT(X86_XCPT_NM)
3719 | RT_BIT(X86_XCPT_TS)
3720 | RT_BIT(X86_XCPT_UD)
3721 | RT_BIT(X86_XCPT_NP)
3722 | RT_BIT(X86_XCPT_SS)
3723 | RT_BIT(X86_XCPT_GP)
3724 | RT_BIT(X86_XCPT_PF)
3725 | RT_BIT(X86_XCPT_MF)
3726 ;
3727#elif defined(HMVMX_ALWAYS_TRAP_PF)
3728 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3729#endif
3730
3731 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3732
3733 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3734 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3735 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3736 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3737 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3738 else
3739 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3740
3741 u32GuestCR0 |= uSetCR0;
3742 u32GuestCR0 &= uZapCR0;
3743 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3744
3745 /* Write VT-x's view of the guest CR0 into the VMCS. */
3746 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3747 AssertRCReturn(rc, rc);
3748 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3749 uZapCR0));
3750
3751 /*
3752 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3753 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3754 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3755 */
3756 uint32_t u32CR0Mask = 0;
3757 u32CR0Mask = X86_CR0_PE
3758 | X86_CR0_NE
3759 | X86_CR0_WP
3760 | X86_CR0_PG
3761 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3762 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3763 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3764
3765 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3766 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3767 * and @bugref{6944}. */
3768#if 0
3769 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3770 u32CR0Mask &= ~X86_CR0_PE;
3771#endif
3772 if (pVM->hm.s.fNestedPaging)
3773 u32CR0Mask &= ~X86_CR0_WP;
3774
3775 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3776 if (fInterceptNM)
3777 {
3778 u32CR0Mask |= X86_CR0_TS
3779 | X86_CR0_MP;
3780 }
3781
3782 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3783 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3784 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3785 AssertRCReturn(rc, rc);
3786 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3787
3788 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3789 }
3790 return rc;
3791}
3792
3793
3794/**
3795 * Loads the guest control registers (CR3, CR4) into the guest-state area
3796 * in the VMCS.
3797 *
3798 * @returns VBox status code.
3799 * @param pVCpu The cross context virtual CPU structure.
3800 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3801 * out-of-sync. Make sure to update the required fields
3802 * before using them.
3803 *
3804 * @remarks No-long-jump zone!!!
3805 */
3806static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3807{
3808 int rc = VINF_SUCCESS;
3809 PVM pVM = pVCpu->CTX_SUFF(pVM);
3810
3811 /*
3812 * Guest CR2.
3813 * It's always loaded in the assembler code. Nothing to do here.
3814 */
3815
3816 /*
3817 * Guest CR3.
3818 */
3819 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3820 {
3821 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3822 if (pVM->hm.s.fNestedPaging)
3823 {
3824 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3825
3826 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3827 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3828 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3829 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3830
3831 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3832 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3833 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3834
3835 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3836 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3837 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3838 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3839 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3840 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3841 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3842
3843 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3844 AssertRCReturn(rc, rc);
3845 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3846
3847 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3848 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3849 {
3850 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3851 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3852 {
3853 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3854 AssertRCReturn(rc, rc);
3855 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3856 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3857 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3858 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3859 AssertRCReturn(rc, rc);
3860 }
3861
3862 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3863 have Unrestricted Execution to handle the guest when it's not using paging. */
3864 GCPhysGuestCR3 = pMixedCtx->cr3;
3865 }
3866 else
3867 {
3868 /*
3869 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3870 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3871 * EPT takes care of translating it to host-physical addresses.
3872 */
3873 RTGCPHYS GCPhys;
3874 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3875 Assert(PDMVmmDevHeapIsEnabled(pVM));
3876
3877 /* We obtain it here every time as the guest could have relocated this PCI region. */
3878 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3879 AssertRCReturn(rc, rc);
3880
3881 GCPhysGuestCR3 = GCPhys;
3882 }
3883
3884 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3885 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3886 }
3887 else
3888 {
3889 /* Non-nested paging case, just use the hypervisor's CR3. */
3890 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3891
3892 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
3893 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3894 }
3895 AssertRCReturn(rc, rc);
3896
3897 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3898 }
3899
3900 /*
3901 * Guest CR4.
3902 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3903 */
3904 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3905 {
3906 Assert(!(pMixedCtx->cr4 >> 32));
3907 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3908
3909 /* The guest's view of its CR4 is unblemished. */
3910 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3911 AssertRCReturn(rc, rc);
3912 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
3913
3914 /* Setup VT-x's view of the guest CR4. */
3915 /*
3916 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3917 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3918 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3919 */
3920 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3921 {
3922 Assert(pVM->hm.s.vmx.pRealModeTSS);
3923 Assert(PDMVmmDevHeapIsEnabled(pVM));
3924 u32GuestCR4 &= ~X86_CR4_VME;
3925 }
3926
3927 if (pVM->hm.s.fNestedPaging)
3928 {
3929 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3930 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3931 {
3932 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3933 u32GuestCR4 |= X86_CR4_PSE;
3934 /* Our identity mapping is a 32-bit page directory. */
3935 u32GuestCR4 &= ~X86_CR4_PAE;
3936 }
3937 /* else use guest CR4.*/
3938 }
3939 else
3940 {
3941 /*
3942 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3943 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3944 */
3945 switch (pVCpu->hm.s.enmShadowMode)
3946 {
3947 case PGMMODE_REAL: /* Real-mode. */
3948 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3949 case PGMMODE_32_BIT: /* 32-bit paging. */
3950 {
3951 u32GuestCR4 &= ~X86_CR4_PAE;
3952 break;
3953 }
3954
3955 case PGMMODE_PAE: /* PAE paging. */
3956 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3957 {
3958 u32GuestCR4 |= X86_CR4_PAE;
3959 break;
3960 }
3961
3962 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3963 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3964#ifdef VBOX_ENABLE_64_BITS_GUESTS
3965 break;
3966#endif
3967 default:
3968 AssertFailed();
3969 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3970 }
3971 }
3972
3973 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3974 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3975 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3976 u32GuestCR4 |= uSetCR4;
3977 u32GuestCR4 &= uZapCR4;
3978
3979 /* Write VT-x's view of the guest CR4 into the VMCS. */
3980 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
3981 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3982 AssertRCReturn(rc, rc);
3983
3984 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
3985 uint32_t u32CR4Mask = X86_CR4_VME
3986 | X86_CR4_PAE
3987 | X86_CR4_PGE
3988 | X86_CR4_PSE
3989 | X86_CR4_VMXE;
3990 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
3991 u32CR4Mask |= X86_CR4_OSXSAVE;
3992 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3993 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3994 AssertRCReturn(rc, rc);
3995
3996 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
3997 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
3998
3999 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4000 }
4001 return rc;
4002}
4003
4004
4005/**
4006 * Loads the guest debug registers into the guest-state area in the VMCS.
4007 *
4008 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4009 *
4010 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4011 *
4012 * @returns VBox status code.
4013 * @param pVCpu The cross context virtual CPU structure.
4014 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4015 * out-of-sync. Make sure to update the required fields
4016 * before using them.
4017 *
4018 * @remarks No-long-jump zone!!!
4019 */
4020static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4021{
4022 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4023 return VINF_SUCCESS;
4024
4025#ifdef VBOX_STRICT
4026 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4027 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4028 {
4029 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4030 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4031 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4032 }
4033#endif
4034
4035 int rc;
4036 PVM pVM = pVCpu->CTX_SUFF(pVM);
4037 bool fSteppingDB = false;
4038 bool fInterceptMovDRx = false;
4039 if (pVCpu->hm.s.fSingleInstruction)
4040 {
4041 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4042 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4043 {
4044 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4045 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4046 AssertRCReturn(rc, rc);
4047 Assert(fSteppingDB == false);
4048 }
4049 else
4050 {
4051 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4052 pVCpu->hm.s.fClearTrapFlag = true;
4053 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4054 fSteppingDB = true;
4055 }
4056 }
4057
4058 if ( fSteppingDB
4059 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4060 {
4061 /*
4062 * Use the combined guest and host DRx values found in the hypervisor
4063 * register set because the debugger has breakpoints active or someone
4064 * is single stepping on the host side without a monitor trap flag.
4065 *
4066 * Note! DBGF expects a clean DR6 state before executing guest code.
4067 */
4068#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4069 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4070 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4071 {
4072 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4073 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4074 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4075 }
4076 else
4077#endif
4078 if (!CPUMIsHyperDebugStateActive(pVCpu))
4079 {
4080 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4081 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4082 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4083 }
4084
4085 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4086 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4087 AssertRCReturn(rc, rc);
4088
4089 pVCpu->hm.s.fUsingHyperDR7 = true;
4090 fInterceptMovDRx = true;
4091 }
4092 else
4093 {
4094 /*
4095 * If the guest has enabled debug registers, we need to load them prior to
4096 * executing guest code so they'll trigger at the right time.
4097 */
4098 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4099 {
4100#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4101 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4102 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4103 {
4104 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4105 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4106 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4107 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4108 }
4109 else
4110#endif
4111 if (!CPUMIsGuestDebugStateActive(pVCpu))
4112 {
4113 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4114 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4115 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4116 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4117 }
4118 Assert(!fInterceptMovDRx);
4119 }
4120 /*
4121 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4122 * must intercept #DB in order to maintain a correct DR6 guest value, and
4123 * because we need to intercept it to prevent nested #DBs from hanging the
4124 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4125 */
4126#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4127 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4128 && !CPUMIsGuestDebugStateActive(pVCpu))
4129#else
4130 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4131#endif
4132 {
4133 fInterceptMovDRx = true;
4134 }
4135
4136 /* Update guest DR7. */
4137 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4138 AssertRCReturn(rc, rc);
4139
4140 pVCpu->hm.s.fUsingHyperDR7 = false;
4141 }
4142
4143 /*
4144 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4145 */
4146 if (fInterceptMovDRx)
4147 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4148 else
4149 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4150 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4151 AssertRCReturn(rc, rc);
4152
4153 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4154 return VINF_SUCCESS;
4155}
4156
4157
4158#ifdef VBOX_STRICT
4159/**
4160 * Strict function to validate segment registers.
4161 *
4162 * @remarks ASSUMES CR0 is up to date.
4163 */
4164static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4165{
4166 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4167 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4168 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4169 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4170 && ( !CPUMIsGuestInRealModeEx(pCtx)
4171 && !CPUMIsGuestInV86ModeEx(pCtx)))
4172 {
4173 /* Protected mode checks */
4174 /* CS */
4175 Assert(pCtx->cs.Attr.n.u1Present);
4176 Assert(!(pCtx->cs.Attr.u & 0xf00));
4177 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4178 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4179 || !(pCtx->cs.Attr.n.u1Granularity));
4180 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4181 || (pCtx->cs.Attr.n.u1Granularity));
4182 /* CS cannot be loaded with NULL in protected mode. */
4183 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4184 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4185 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4186 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4187 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4188 else
4189 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4190 /* SS */
4191 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4192 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4193 if ( !(pCtx->cr0 & X86_CR0_PE)
4194 || pCtx->cs.Attr.n.u4Type == 3)
4195 {
4196 Assert(!pCtx->ss.Attr.n.u2Dpl);
4197 }
4198 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4199 {
4200 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4201 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4202 Assert(pCtx->ss.Attr.n.u1Present);
4203 Assert(!(pCtx->ss.Attr.u & 0xf00));
4204 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4205 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4206 || !(pCtx->ss.Attr.n.u1Granularity));
4207 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4208 || (pCtx->ss.Attr.n.u1Granularity));
4209 }
4210 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4211 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4212 {
4213 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4214 Assert(pCtx->ds.Attr.n.u1Present);
4215 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4216 Assert(!(pCtx->ds.Attr.u & 0xf00));
4217 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4218 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4219 || !(pCtx->ds.Attr.n.u1Granularity));
4220 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4221 || (pCtx->ds.Attr.n.u1Granularity));
4222 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4223 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4224 }
4225 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4226 {
4227 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4228 Assert(pCtx->es.Attr.n.u1Present);
4229 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4230 Assert(!(pCtx->es.Attr.u & 0xf00));
4231 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4232 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4233 || !(pCtx->es.Attr.n.u1Granularity));
4234 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4235 || (pCtx->es.Attr.n.u1Granularity));
4236 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4237 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4238 }
4239 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4240 {
4241 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4242 Assert(pCtx->fs.Attr.n.u1Present);
4243 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4244 Assert(!(pCtx->fs.Attr.u & 0xf00));
4245 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4246 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4247 || !(pCtx->fs.Attr.n.u1Granularity));
4248 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4249 || (pCtx->fs.Attr.n.u1Granularity));
4250 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4251 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4252 }
4253 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4254 {
4255 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4256 Assert(pCtx->gs.Attr.n.u1Present);
4257 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4258 Assert(!(pCtx->gs.Attr.u & 0xf00));
4259 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4260 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4261 || !(pCtx->gs.Attr.n.u1Granularity));
4262 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4263 || (pCtx->gs.Attr.n.u1Granularity));
4264 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4265 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4266 }
4267 /* 64-bit capable CPUs. */
4268# if HC_ARCH_BITS == 64
4269 Assert(!(pCtx->cs.u64Base >> 32));
4270 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4271 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4272 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4273# endif
4274 }
4275 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4276 || ( CPUMIsGuestInRealModeEx(pCtx)
4277 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4278 {
4279 /* Real and v86 mode checks. */
4280 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4281 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4282 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4283 {
4284 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4285 }
4286 else
4287 {
4288 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4289 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4290 }
4291
4292 /* CS */
4293 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4294 Assert(pCtx->cs.u32Limit == 0xffff);
4295 Assert(u32CSAttr == 0xf3);
4296 /* SS */
4297 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4298 Assert(pCtx->ss.u32Limit == 0xffff);
4299 Assert(u32SSAttr == 0xf3);
4300 /* DS */
4301 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4302 Assert(pCtx->ds.u32Limit == 0xffff);
4303 Assert(u32DSAttr == 0xf3);
4304 /* ES */
4305 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4306 Assert(pCtx->es.u32Limit == 0xffff);
4307 Assert(u32ESAttr == 0xf3);
4308 /* FS */
4309 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4310 Assert(pCtx->fs.u32Limit == 0xffff);
4311 Assert(u32FSAttr == 0xf3);
4312 /* GS */
4313 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4314 Assert(pCtx->gs.u32Limit == 0xffff);
4315 Assert(u32GSAttr == 0xf3);
4316 /* 64-bit capable CPUs. */
4317# if HC_ARCH_BITS == 64
4318 Assert(!(pCtx->cs.u64Base >> 32));
4319 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4320 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4321 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4322# endif
4323 }
4324}
4325#endif /* VBOX_STRICT */
4326
4327
4328/**
4329 * Writes a guest segment register into the guest-state area in the VMCS.
4330 *
4331 * @returns VBox status code.
4332 * @param pVCpu The cross context virtual CPU structure.
4333 * @param idxSel Index of the selector in the VMCS.
4334 * @param idxLimit Index of the segment limit in the VMCS.
4335 * @param idxBase Index of the segment base in the VMCS.
4336 * @param idxAccess Index of the access rights of the segment in the VMCS.
4337 * @param pSelReg Pointer to the segment selector.
4338 *
4339 * @remarks No-long-jump zone!!!
4340 */
4341static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4342 uint32_t idxAccess, PCPUMSELREG pSelReg)
4343{
4344 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4345 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4346 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4347 AssertRCReturn(rc, rc);
4348
4349 uint32_t u32Access = pSelReg->Attr.u;
4350 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4351 {
4352 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4353 u32Access = 0xf3;
4354 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4355 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4356 }
4357 else
4358 {
4359 /*
4360 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4361 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4362 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4363 * loaded in protected-mode have their attribute as 0.
4364 */
4365 if (!u32Access)
4366 u32Access = X86DESCATTR_UNUSABLE;
4367 }
4368
4369 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4370 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4371 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4372
4373 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4374 AssertRCReturn(rc, rc);
4375 return rc;
4376}
4377
4378
4379/**
4380 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4381 * into the guest-state area in the VMCS.
4382 *
4383 * @returns VBox status code.
4384 * @param pVCpu The cross context virtual CPU structure.
4385 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4386 * out-of-sync. Make sure to update the required fields
4387 * before using them.
4388 *
4389 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4390 * @remarks No-long-jump zone!!!
4391 */
4392static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4393{
4394 int rc = VERR_INTERNAL_ERROR_5;
4395 PVM pVM = pVCpu->CTX_SUFF(pVM);
4396
4397 /*
4398 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4399 */
4400 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4401 {
4402 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4403 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4404 {
4405 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4406 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4407 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4408 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4409 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4410 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4411 }
4412
4413#ifdef VBOX_WITH_REM
4414 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4415 {
4416 Assert(pVM->hm.s.vmx.pRealModeTSS);
4417 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4418 if ( pVCpu->hm.s.vmx.fWasInRealMode
4419 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4420 {
4421 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4422 in real-mode (e.g. OpenBSD 4.0) */
4423 REMFlushTBs(pVM);
4424 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4425 pVCpu->hm.s.vmx.fWasInRealMode = false;
4426 }
4427 }
4428#endif
4429 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4430 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4431 AssertRCReturn(rc, rc);
4432 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4433 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4434 AssertRCReturn(rc, rc);
4435 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4436 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4437 AssertRCReturn(rc, rc);
4438 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4439 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4440 AssertRCReturn(rc, rc);
4441 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4442 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4443 AssertRCReturn(rc, rc);
4444 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4445 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4446 AssertRCReturn(rc, rc);
4447
4448#ifdef VBOX_STRICT
4449 /* Validate. */
4450 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4451#endif
4452
4453 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4454 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4455 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4456 }
4457
4458 /*
4459 * Guest TR.
4460 */
4461 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4462 {
4463 /*
4464 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4465 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4466 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4467 */
4468 uint16_t u16Sel = 0;
4469 uint32_t u32Limit = 0;
4470 uint64_t u64Base = 0;
4471 uint32_t u32AccessRights = 0;
4472
4473 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4474 {
4475 u16Sel = pMixedCtx->tr.Sel;
4476 u32Limit = pMixedCtx->tr.u32Limit;
4477 u64Base = pMixedCtx->tr.u64Base;
4478 u32AccessRights = pMixedCtx->tr.Attr.u;
4479 }
4480 else
4481 {
4482 Assert(pVM->hm.s.vmx.pRealModeTSS);
4483 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4484
4485 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4486 RTGCPHYS GCPhys;
4487 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4488 AssertRCReturn(rc, rc);
4489
4490 X86DESCATTR DescAttr;
4491 DescAttr.u = 0;
4492 DescAttr.n.u1Present = 1;
4493 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4494
4495 u16Sel = 0;
4496 u32Limit = HM_VTX_TSS_SIZE;
4497 u64Base = GCPhys; /* in real-mode phys = virt. */
4498 u32AccessRights = DescAttr.u;
4499 }
4500
4501 /* Validate. */
4502 Assert(!(u16Sel & RT_BIT(2)));
4503 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4504 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4505 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4506 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4507 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4508 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4509 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4510 Assert( (u32Limit & 0xfff) == 0xfff
4511 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4512 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4513 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4514
4515 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel);
4516 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4517 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4518 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4519 AssertRCReturn(rc, rc);
4520
4521 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4522 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4523 }
4524
4525 /*
4526 * Guest GDTR.
4527 */
4528 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4529 {
4530 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4531 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4532 AssertRCReturn(rc, rc);
4533
4534 /* Validate. */
4535 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4536
4537 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4538 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4539 }
4540
4541 /*
4542 * Guest LDTR.
4543 */
4544 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4545 {
4546 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4547 uint32_t u32Access = 0;
4548 if (!pMixedCtx->ldtr.Attr.u)
4549 u32Access = X86DESCATTR_UNUSABLE;
4550 else
4551 u32Access = pMixedCtx->ldtr.Attr.u;
4552
4553 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel);
4554 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4555 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4556 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4557 AssertRCReturn(rc, rc);
4558
4559 /* Validate. */
4560 if (!(u32Access & X86DESCATTR_UNUSABLE))
4561 {
4562 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4563 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4564 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4565 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4566 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4567 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4568 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4569 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4570 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4571 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4572 }
4573
4574 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4575 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4576 }
4577
4578 /*
4579 * Guest IDTR.
4580 */
4581 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4582 {
4583 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4584 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4585 AssertRCReturn(rc, rc);
4586
4587 /* Validate. */
4588 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4589
4590 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4591 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4592 }
4593
4594 return VINF_SUCCESS;
4595}
4596
4597
4598/**
4599 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4600 * areas.
4601 *
4602 * These MSRs will automatically be loaded to the host CPU on every successful
4603 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4604 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4605 * -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4606 *
4607 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4608 *
4609 * @returns VBox status code.
4610 * @param pVCpu The cross context virtual CPU structure.
4611 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4612 * out-of-sync. Make sure to update the required fields
4613 * before using them.
4614 *
4615 * @remarks No-long-jump zone!!!
4616 */
4617static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4618{
4619 AssertPtr(pVCpu);
4620 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4621
4622 /*
4623 * MSRs that we use the auto-load/store MSR area in the VMCS.
4624 */
4625 PVM pVM = pVCpu->CTX_SUFF(pVM);
4626 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4627 {
4628 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4629#if HC_ARCH_BITS == 32
4630 if (pVM->hm.s.fAllow64BitGuests)
4631 {
4632 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4633 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4634 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4635 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4636 AssertRCReturn(rc, rc);
4637# ifdef LOG_ENABLED
4638 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4639 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4640 {
4641 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4642 pMsr->u64Value));
4643 }
4644# endif
4645 }
4646#endif
4647 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4648 }
4649
4650 /*
4651 * Guest Sysenter MSRs.
4652 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4653 * VM-exits on WRMSRs for these MSRs.
4654 */
4655 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4656 {
4657 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4658 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4659 }
4660
4661 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4662 {
4663 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4664 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4665 }
4666
4667 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4668 {
4669 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4670 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4671 }
4672
4673 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4674 {
4675 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4676 {
4677 /*
4678 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4679 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4680 */
4681 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4682 {
4683 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4684 AssertRCReturn(rc,rc);
4685 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4686 }
4687 else
4688 {
4689 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4690 NULL /* pfAddedAndUpdated */);
4691 AssertRCReturn(rc, rc);
4692
4693 /* We need to intercept reads too, see @bugref{7386#c16}. */
4694 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4695 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4696 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4697 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4698 }
4699 }
4700 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4701 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4702 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4703 }
4704
4705 return VINF_SUCCESS;
4706}
4707
4708
4709/**
4710 * Loads the guest activity state into the guest-state area in the VMCS.
4711 *
4712 * @returns VBox status code.
4713 * @param pVCpu The cross context virtual CPU structure.
4714 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4715 * out-of-sync. Make sure to update the required fields
4716 * before using them.
4717 *
4718 * @remarks No-long-jump zone!!!
4719 */
4720static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4721{
4722 NOREF(pMixedCtx);
4723 /** @todo See if we can make use of other states, e.g.
4724 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4725 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4726 {
4727 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4728 AssertRCReturn(rc, rc);
4729
4730 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4731 }
4732 return VINF_SUCCESS;
4733}
4734
4735
4736/**
4737 * Sets up the appropriate function to run guest code.
4738 *
4739 * @returns VBox status code.
4740 * @param pVCpu The cross context virtual CPU structure.
4741 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4742 * out-of-sync. Make sure to update the required fields
4743 * before using them.
4744 *
4745 * @remarks No-long-jump zone!!!
4746 */
4747static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4748{
4749 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4750 {
4751#ifndef VBOX_ENABLE_64_BITS_GUESTS
4752 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4753#endif
4754 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4755#if HC_ARCH_BITS == 32
4756 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4757 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4758 {
4759 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4760 {
4761 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4762 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4763 | HM_CHANGED_VMX_ENTRY_CTLS
4764 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4765 }
4766 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4767 }
4768#else
4769 /* 64-bit host. */
4770 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4771#endif
4772 }
4773 else
4774 {
4775 /* Guest is not in long mode, use the 32-bit handler. */
4776#if HC_ARCH_BITS == 32
4777 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4778 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4779 {
4780 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4781 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4782 | HM_CHANGED_VMX_ENTRY_CTLS
4783 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4784 }
4785#endif
4786 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4787 }
4788 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4789 return VINF_SUCCESS;
4790}
4791
4792
4793/**
4794 * Wrapper for running the guest code in VT-x.
4795 *
4796 * @returns VBox status code, no informational status codes.
4797 * @param pVM The cross context VM structure.
4798 * @param pVCpu The cross context virtual CPU structure.
4799 * @param pCtx Pointer to the guest-CPU context.
4800 *
4801 * @remarks No-long-jump zone!!!
4802 */
4803DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4804{
4805 /*
4806 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4807 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4808 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4809 */
4810 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4811 /** @todo Add stats for resume vs launch. */
4812#ifdef VBOX_WITH_KERNEL_USING_XMM
4813 int rc = HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4814#else
4815 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4816#endif
4817 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4818 return rc;
4819}
4820
4821
4822/**
4823 * Reports world-switch error and dumps some useful debug info.
4824 *
4825 * @param pVM The cross context VM structure.
4826 * @param pVCpu The cross context virtual CPU structure.
4827 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4828 * @param pCtx Pointer to the guest-CPU context.
4829 * @param pVmxTransient Pointer to the VMX transient structure (only
4830 * exitReason updated).
4831 */
4832static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4833{
4834 Assert(pVM);
4835 Assert(pVCpu);
4836 Assert(pCtx);
4837 Assert(pVmxTransient);
4838 HMVMX_ASSERT_PREEMPT_SAFE();
4839
4840 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4841 switch (rcVMRun)
4842 {
4843 case VERR_VMX_INVALID_VMXON_PTR:
4844 AssertFailed();
4845 break;
4846 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4847 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4848 {
4849 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4850 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4851 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4852 AssertRC(rc);
4853
4854 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4855 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4856 Cannot do it here as we may have been long preempted. */
4857
4858#ifdef VBOX_STRICT
4859 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4860 pVmxTransient->uExitReason));
4861 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4862 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4863 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4864 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4865 else
4866 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4867 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4868 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4869
4870 /* VMX control bits. */
4871 uint32_t u32Val;
4872 uint64_t u64Val;
4873 RTHCUINTREG uHCReg;
4874 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4875 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4876 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4877 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4878 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4879 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4880 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4881 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4882 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4883 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4884 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4885 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4886 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4887 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4888 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4889 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4890 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4891 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4892 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4893 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4894 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4895 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4896 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4897 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4898 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4899 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4900 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4901 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4902 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4903 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4904 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4905 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4906 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4907 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4908 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4909 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4910 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4911 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4912 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4913 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4914 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4915 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4916
4917 /* Guest bits. */
4918 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4919 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4920 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4921 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4922 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4923 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4924 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4925 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4926
4927 /* Host bits. */
4928 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4929 Log4(("Host CR0 %#RHr\n", uHCReg));
4930 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4931 Log4(("Host CR3 %#RHr\n", uHCReg));
4932 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4933 Log4(("Host CR4 %#RHr\n", uHCReg));
4934
4935 RTGDTR HostGdtr;
4936 PCX86DESCHC pDesc;
4937 ASMGetGDTR(&HostGdtr);
4938 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4939 Log4(("Host CS %#08x\n", u32Val));
4940 if (u32Val < HostGdtr.cbGdt)
4941 {
4942 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4943 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4944 }
4945
4946 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4947 Log4(("Host DS %#08x\n", u32Val));
4948 if (u32Val < HostGdtr.cbGdt)
4949 {
4950 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4951 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4952 }
4953
4954 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4955 Log4(("Host ES %#08x\n", u32Val));
4956 if (u32Val < HostGdtr.cbGdt)
4957 {
4958 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4959 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4960 }
4961
4962 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4963 Log4(("Host FS %#08x\n", u32Val));
4964 if (u32Val < HostGdtr.cbGdt)
4965 {
4966 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4967 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4968 }
4969
4970 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4971 Log4(("Host GS %#08x\n", u32Val));
4972 if (u32Val < HostGdtr.cbGdt)
4973 {
4974 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4975 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4976 }
4977
4978 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4979 Log4(("Host SS %#08x\n", u32Val));
4980 if (u32Val < HostGdtr.cbGdt)
4981 {
4982 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4983 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4984 }
4985
4986 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4987 Log4(("Host TR %#08x\n", u32Val));
4988 if (u32Val < HostGdtr.cbGdt)
4989 {
4990 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4991 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4992 }
4993
4994 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4995 Log4(("Host TR Base %#RHv\n", uHCReg));
4996 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4997 Log4(("Host GDTR Base %#RHv\n", uHCReg));
4998 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4999 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5000 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5001 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5002 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5003 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5004 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5005 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5006 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5007 Log4(("Host RSP %#RHv\n", uHCReg));
5008 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5009 Log4(("Host RIP %#RHv\n", uHCReg));
5010# if HC_ARCH_BITS == 64
5011 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5012 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5013 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5014 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5015 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5016 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5017# endif
5018#endif /* VBOX_STRICT */
5019 break;
5020 }
5021
5022 default:
5023 /* Impossible */
5024 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5025 break;
5026 }
5027 NOREF(pVM); NOREF(pCtx);
5028}
5029
5030
5031#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5032#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5033# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5034#endif
5035#ifdef VBOX_STRICT
5036static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5037{
5038 switch (idxField)
5039 {
5040 case VMX_VMCS_GUEST_RIP:
5041 case VMX_VMCS_GUEST_RSP:
5042 case VMX_VMCS_GUEST_SYSENTER_EIP:
5043 case VMX_VMCS_GUEST_SYSENTER_ESP:
5044 case VMX_VMCS_GUEST_GDTR_BASE:
5045 case VMX_VMCS_GUEST_IDTR_BASE:
5046 case VMX_VMCS_GUEST_CS_BASE:
5047 case VMX_VMCS_GUEST_DS_BASE:
5048 case VMX_VMCS_GUEST_ES_BASE:
5049 case VMX_VMCS_GUEST_FS_BASE:
5050 case VMX_VMCS_GUEST_GS_BASE:
5051 case VMX_VMCS_GUEST_SS_BASE:
5052 case VMX_VMCS_GUEST_LDTR_BASE:
5053 case VMX_VMCS_GUEST_TR_BASE:
5054 case VMX_VMCS_GUEST_CR3:
5055 return true;
5056 }
5057 return false;
5058}
5059
5060static bool hmR0VmxIsValidReadField(uint32_t idxField)
5061{
5062 switch (idxField)
5063 {
5064 /* Read-only fields. */
5065 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5066 return true;
5067 }
5068 /* Remaining readable fields should also be writable. */
5069 return hmR0VmxIsValidWriteField(idxField);
5070}
5071#endif /* VBOX_STRICT */
5072
5073
5074/**
5075 * Executes the specified handler in 64-bit mode.
5076 *
5077 * @returns VBox status code (no informational status codes).
5078 * @param pVM The cross context VM structure.
5079 * @param pVCpu The cross context virtual CPU structure.
5080 * @param pCtx Pointer to the guest CPU context.
5081 * @param enmOp The operation to perform.
5082 * @param cParams Number of parameters.
5083 * @param paParam Array of 32-bit parameters.
5084 */
5085VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5086 uint32_t cParams, uint32_t *paParam)
5087{
5088 NOREF(pCtx);
5089
5090 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5091 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5092 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5093 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5094
5095#ifdef VBOX_STRICT
5096 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5097 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5098
5099 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5100 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5101#endif
5102
5103 /* Disable interrupts. */
5104 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5105
5106#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5107 RTCPUID idHostCpu = RTMpCpuId();
5108 CPUMR0SetLApic(pVCpu, idHostCpu);
5109#endif
5110
5111 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5112 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5113
5114 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5115 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5116
5117 /* Leave VMX Root Mode. */
5118 VMXDisable();
5119
5120 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5121
5122 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5123 CPUMSetHyperEIP(pVCpu, enmOp);
5124 for (int i = (int)cParams - 1; i >= 0; i--)
5125 CPUMPushHyper(pVCpu, paParam[i]);
5126
5127 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5128
5129 /* Call the switcher. */
5130 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5131 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5132
5133 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5134 /* Make sure the VMX instructions don't cause #UD faults. */
5135 SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
5136
5137 /* Re-enter VMX Root Mode */
5138 int rc2 = VMXEnable(HCPhysCpuPage);
5139 if (RT_FAILURE(rc2))
5140 {
5141 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5142 ASMSetFlags(fOldEFlags);
5143 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5144 return rc2;
5145 }
5146
5147 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5148 AssertRC(rc2);
5149 Assert(!(ASMGetFlags() & X86_EFL_IF));
5150 ASMSetFlags(fOldEFlags);
5151 return rc;
5152}
5153
5154
5155/**
5156 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5157 * supporting 64-bit guests.
5158 *
5159 * @returns VBox status code.
5160 * @param fResume Whether to VMLAUNCH or VMRESUME.
5161 * @param pCtx Pointer to the guest-CPU context.
5162 * @param pCache Pointer to the VMCS cache.
5163 * @param pVM The cross context VM structure.
5164 * @param pVCpu The cross context virtual CPU structure.
5165 */
5166DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5167{
5168 NOREF(fResume);
5169
5170 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5171 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5172
5173#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5174 pCache->uPos = 1;
5175 pCache->interPD = PGMGetInterPaeCR3(pVM);
5176 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5177#endif
5178
5179#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5180 pCache->TestIn.HCPhysCpuPage = 0;
5181 pCache->TestIn.HCPhysVmcs = 0;
5182 pCache->TestIn.pCache = 0;
5183 pCache->TestOut.HCPhysVmcs = 0;
5184 pCache->TestOut.pCache = 0;
5185 pCache->TestOut.pCtx = 0;
5186 pCache->TestOut.eflags = 0;
5187#else
5188 NOREF(pCache);
5189#endif
5190
5191 uint32_t aParam[10];
5192 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5193 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5194 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5195 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5196 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5197 aParam[5] = 0;
5198 aParam[6] = VM_RC_ADDR(pVM, pVM);
5199 aParam[7] = 0;
5200 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5201 aParam[9] = 0;
5202
5203#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5204 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5205 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5206#endif
5207 int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5208
5209#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5210 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5211 Assert(pCtx->dr[4] == 10);
5212 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5213#endif
5214
5215#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5216 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5217 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5218 pVCpu->hm.s.vmx.HCPhysVmcs));
5219 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5220 pCache->TestOut.HCPhysVmcs));
5221 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5222 pCache->TestOut.pCache));
5223 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5224 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5225 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5226 pCache->TestOut.pCtx));
5227 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5228#endif
5229 return rc;
5230}
5231
5232
5233/**
5234 * Initialize the VMCS-Read cache.
5235 *
5236 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5237 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5238 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5239 * (those that have a 32-bit FULL & HIGH part).
5240 *
5241 * @returns VBox status code.
5242 * @param pVM The cross context VM structure.
5243 * @param pVCpu The cross context virtual CPU structure.
5244 */
5245static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5246{
5247#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5248{ \
5249 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5250 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5251 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5252 ++cReadFields; \
5253}
5254
5255 AssertPtr(pVM);
5256 AssertPtr(pVCpu);
5257 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5258 uint32_t cReadFields = 0;
5259
5260 /*
5261 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5262 * and serve to indicate exceptions to the rules.
5263 */
5264
5265 /* Guest-natural selector base fields. */
5266#if 0
5267 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5268 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5269 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5270#endif
5271 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5272 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5273 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5274 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5275 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5276 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5277 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5278 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5279 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5280 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5281 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5282 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5283#if 0
5284 /* Unused natural width guest-state fields. */
5285 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5286 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5287#endif
5288 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5289 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5290
5291 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5292#if 0
5293 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5294 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5295 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5296 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5297 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5298 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5299 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5300 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5301 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5302#endif
5303
5304 /* Natural width guest-state fields. */
5305 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5306#if 0
5307 /* Currently unused field. */
5308 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5309#endif
5310
5311 if (pVM->hm.s.fNestedPaging)
5312 {
5313 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5314 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5315 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5316 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5317 }
5318 else
5319 {
5320 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5321 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5322 }
5323
5324#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5325 return VINF_SUCCESS;
5326}
5327
5328
5329/**
5330 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5331 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5332 * darwin, running 64-bit guests).
5333 *
5334 * @returns VBox status code.
5335 * @param pVCpu The cross context virtual CPU structure.
5336 * @param idxField The VMCS field encoding.
5337 * @param u64Val 16, 32 or 64-bit value.
5338 */
5339VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5340{
5341 int rc;
5342 switch (idxField)
5343 {
5344 /*
5345 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5346 */
5347 /* 64-bit Control fields. */
5348 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5349 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5350 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5351 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5352 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5353 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5354 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5355 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5356 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5357 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5358 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5359 case VMX_VMCS64_CTRL_EPTP_FULL:
5360 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5361 /* 64-bit Guest-state fields. */
5362 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5363 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5364 case VMX_VMCS64_GUEST_PAT_FULL:
5365 case VMX_VMCS64_GUEST_EFER_FULL:
5366 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5367 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5368 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5369 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5370 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5371 /* 64-bit Host-state fields. */
5372 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5373 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5374 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5375 {
5376 rc = VMXWriteVmcs32(idxField, u64Val);
5377 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5378 break;
5379 }
5380
5381 /*
5382 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5383 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5384 */
5385 /* Natural-width Guest-state fields. */
5386 case VMX_VMCS_GUEST_CR3:
5387 case VMX_VMCS_GUEST_ES_BASE:
5388 case VMX_VMCS_GUEST_CS_BASE:
5389 case VMX_VMCS_GUEST_SS_BASE:
5390 case VMX_VMCS_GUEST_DS_BASE:
5391 case VMX_VMCS_GUEST_FS_BASE:
5392 case VMX_VMCS_GUEST_GS_BASE:
5393 case VMX_VMCS_GUEST_LDTR_BASE:
5394 case VMX_VMCS_GUEST_TR_BASE:
5395 case VMX_VMCS_GUEST_GDTR_BASE:
5396 case VMX_VMCS_GUEST_IDTR_BASE:
5397 case VMX_VMCS_GUEST_RSP:
5398 case VMX_VMCS_GUEST_RIP:
5399 case VMX_VMCS_GUEST_SYSENTER_ESP:
5400 case VMX_VMCS_GUEST_SYSENTER_EIP:
5401 {
5402 if (!(u64Val >> 32))
5403 {
5404 /* If this field is 64-bit, VT-x will zero out the top bits. */
5405 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5406 }
5407 else
5408 {
5409 /* Assert that only the 32->64 switcher case should ever come here. */
5410 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5411 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5412 }
5413 break;
5414 }
5415
5416 default:
5417 {
5418 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5419 rc = VERR_INVALID_PARAMETER;
5420 break;
5421 }
5422 }
5423 AssertRCReturn(rc, rc);
5424 return rc;
5425}
5426
5427
5428/**
5429 * Queue up a VMWRITE by using the VMCS write cache.
5430 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5431 *
5432 * @param pVCpu The cross context virtual CPU structure.
5433 * @param idxField The VMCS field encoding.
5434 * @param u64Val 16, 32 or 64-bit value.
5435 */
5436VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5437{
5438 AssertPtr(pVCpu);
5439 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5440
5441 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5442 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5443
5444 /* Make sure there are no duplicates. */
5445 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5446 {
5447 if (pCache->Write.aField[i] == idxField)
5448 {
5449 pCache->Write.aFieldVal[i] = u64Val;
5450 return VINF_SUCCESS;
5451 }
5452 }
5453
5454 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5455 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5456 pCache->Write.cValidEntries++;
5457 return VINF_SUCCESS;
5458}
5459#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5460
5461
5462/**
5463 * Sets up the usage of TSC-offsetting and updates the VMCS.
5464 *
5465 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5466 * VMX preemption timer.
5467 *
5468 * @returns VBox status code.
5469 * @param pVM The cross context VM structure.
5470 * @param pVCpu The cross context virtual CPU structure.
5471 *
5472 * @remarks No-long-jump zone!!!
5473 */
5474static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5475{
5476 int rc;
5477 bool fOffsettedTsc;
5478 bool fParavirtTsc;
5479 if (pVM->hm.s.vmx.fUsePreemptTimer)
5480 {
5481 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5482 &fOffsettedTsc, &fParavirtTsc);
5483
5484 /* Make sure the returned values have sane upper and lower boundaries. */
5485 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5486 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5487 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5488 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5489
5490 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5491 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5492 }
5493 else
5494 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5495
5496 /** @todo later optimize this to be done elsewhere and not before every
5497 * VM-entry. */
5498 if (fParavirtTsc)
5499 {
5500 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5501 information before every VM-entry, hence disable it for performance sake. */
5502#if 0
5503 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5504 AssertRC(rc);
5505#endif
5506 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5507 }
5508
5509 if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5510 {
5511 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5512 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5513
5514 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5515 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5516 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5517 }
5518 else
5519 {
5520 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5521 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5522 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5523 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5524 }
5525}
5526
5527
5528/**
5529 * Determines if an exception is a contributory exception.
5530 *
5531 * Contributory exceptions are ones which can cause double-faults unless the
5532 * original exception was a benign exception. Page-fault is intentionally not
5533 * included here as it's a conditional contributory exception.
5534 *
5535 * @returns true if the exception is contributory, false otherwise.
5536 * @param uVector The exception vector.
5537 */
5538DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5539{
5540 switch (uVector)
5541 {
5542 case X86_XCPT_GP:
5543 case X86_XCPT_SS:
5544 case X86_XCPT_NP:
5545 case X86_XCPT_TS:
5546 case X86_XCPT_DE:
5547 return true;
5548 default:
5549 break;
5550 }
5551 return false;
5552}
5553
5554
5555/**
5556 * Sets an event as a pending event to be injected into the guest.
5557 *
5558 * @param pVCpu The cross context virtual CPU structure.
5559 * @param u32IntInfo The VM-entry interruption-information field.
5560 * @param cbInstr The VM-entry instruction length in bytes (for software
5561 * interrupts, exceptions and privileged software
5562 * exceptions).
5563 * @param u32ErrCode The VM-entry exception error code.
5564 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5565 * page-fault.
5566 *
5567 * @remarks Statistics counter assumes this is a guest event being injected or
5568 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5569 * always incremented.
5570 */
5571DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5572 RTGCUINTPTR GCPtrFaultAddress)
5573{
5574 Assert(!pVCpu->hm.s.Event.fPending);
5575 pVCpu->hm.s.Event.fPending = true;
5576 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5577 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5578 pVCpu->hm.s.Event.cbInstr = cbInstr;
5579 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5580
5581 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5582}
5583
5584
5585/**
5586 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5587 *
5588 * @param pVCpu The cross context virtual CPU structure.
5589 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5590 * out-of-sync. Make sure to update the required fields
5591 * before using them.
5592 */
5593DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5594{
5595 NOREF(pMixedCtx);
5596 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5597 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5598 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5599 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5600}
5601
5602
5603/**
5604 * Handle a condition that occurred while delivering an event through the guest
5605 * IDT.
5606 *
5607 * @returns Strict VBox status code (i.e. informational status codes too).
5608 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5609 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5610 * to continue execution of the guest which will delivery the \#DF.
5611 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5612 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5613 *
5614 * @param pVCpu The cross context virtual CPU structure.
5615 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5616 * out-of-sync. Make sure to update the required fields
5617 * before using them.
5618 * @param pVmxTransient Pointer to the VMX transient structure.
5619 *
5620 * @remarks No-long-jump zone!!!
5621 */
5622static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5623{
5624 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5625
5626 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5627 rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5628
5629 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5630 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5631 {
5632 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5633 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5634
5635 typedef enum
5636 {
5637 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5638 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5639 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5640 VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
5641 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5642 } VMXREFLECTXCPT;
5643
5644 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5645 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5646 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5647 {
5648 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5649 {
5650 enmReflect = VMXREFLECTXCPT_XCPT;
5651#ifdef VBOX_STRICT
5652 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5653 && uExitVector == X86_XCPT_PF)
5654 {
5655 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5656 }
5657#endif
5658 if ( uExitVector == X86_XCPT_PF
5659 && uIdtVector == X86_XCPT_PF)
5660 {
5661 pVmxTransient->fVectoringDoublePF = true;
5662 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5663 }
5664 else if ( uExitVector == X86_XCPT_AC
5665 && uIdtVector == X86_XCPT_AC)
5666 {
5667 enmReflect = VMXREFLECTXCPT_HANG;
5668 Log4(("IDT: Nested #AC - Bad guest\n"));
5669 }
5670 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5671 && hmR0VmxIsContributoryXcpt(uExitVector)
5672 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5673 || uIdtVector == X86_XCPT_PF))
5674 {
5675 enmReflect = VMXREFLECTXCPT_DF;
5676 }
5677 else if (uIdtVector == X86_XCPT_DF)
5678 enmReflect = VMXREFLECTXCPT_TF;
5679 }
5680 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5681 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5682 {
5683 /*
5684 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5685 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5686 */
5687 enmReflect = VMXREFLECTXCPT_XCPT;
5688
5689 if (uExitVector == X86_XCPT_PF)
5690 {
5691 pVmxTransient->fVectoringPF = true;
5692 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5693 }
5694 }
5695 }
5696 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5697 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5698 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5699 {
5700 /*
5701 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5702 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5703 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5704 */
5705 enmReflect = VMXREFLECTXCPT_XCPT;
5706 }
5707
5708 /*
5709 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5710 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5711 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5712 *
5713 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5714 */
5715 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5716 && enmReflect == VMXREFLECTXCPT_XCPT
5717 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5718 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5719 {
5720 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5721 }
5722
5723 switch (enmReflect)
5724 {
5725 case VMXREFLECTXCPT_XCPT:
5726 {
5727 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5728 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5729 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5730
5731 uint32_t u32ErrCode = 0;
5732 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5733 {
5734 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5735 AssertRCReturn(rc2, rc2);
5736 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5737 }
5738
5739 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5740 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5741 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5742 rcStrict = VINF_SUCCESS;
5743 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5744 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5745
5746 break;
5747 }
5748
5749 case VMXREFLECTXCPT_DF:
5750 {
5751 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5752 rcStrict = VINF_HM_DOUBLE_FAULT;
5753 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5754 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5755
5756 break;
5757 }
5758
5759 case VMXREFLECTXCPT_TF:
5760 {
5761 rcStrict = VINF_EM_RESET;
5762 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5763 uExitVector));
5764 break;
5765 }
5766
5767 case VMXREFLECTXCPT_HANG:
5768 {
5769 rcStrict = VERR_EM_GUEST_CPU_HANG;
5770 break;
5771 }
5772
5773 default:
5774 Assert(rcStrict == VINF_SUCCESS);
5775 break;
5776 }
5777 }
5778 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5779 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5780 && uExitVector != X86_XCPT_DF
5781 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5782 {
5783 /*
5784 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5785 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5786 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5787 */
5788 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5789 {
5790 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
5791 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5792 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5793 }
5794 }
5795
5796 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
5797 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
5798 return rcStrict;
5799}
5800
5801
5802/**
5803 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5804 *
5805 * @returns VBox status code.
5806 * @param pVCpu The cross context virtual CPU structure.
5807 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5808 * out-of-sync. Make sure to update the required fields
5809 * before using them.
5810 *
5811 * @remarks No-long-jump zone!!!
5812 */
5813static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5814{
5815 NOREF(pMixedCtx);
5816
5817 /*
5818 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5819 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5820 */
5821 VMMRZCallRing3Disable(pVCpu);
5822 HM_DISABLE_PREEMPT();
5823
5824 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5825 {
5826 uint32_t uVal = 0;
5827 uint32_t uShadow = 0;
5828 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5829 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5830 AssertRCReturn(rc, rc);
5831
5832 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5833 CPUMSetGuestCR0(pVCpu, uVal);
5834 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5835 }
5836
5837 HM_RESTORE_PREEMPT();
5838 VMMRZCallRing3Enable(pVCpu);
5839 return VINF_SUCCESS;
5840}
5841
5842
5843/**
5844 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5845 *
5846 * @returns VBox status code.
5847 * @param pVCpu The cross context virtual CPU structure.
5848 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5849 * out-of-sync. Make sure to update the required fields
5850 * before using them.
5851 *
5852 * @remarks No-long-jump zone!!!
5853 */
5854static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5855{
5856 NOREF(pMixedCtx);
5857
5858 int rc = VINF_SUCCESS;
5859 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5860 {
5861 uint32_t uVal = 0;
5862 uint32_t uShadow = 0;
5863 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5864 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5865 AssertRCReturn(rc, rc);
5866
5867 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5868 CPUMSetGuestCR4(pVCpu, uVal);
5869 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5870 }
5871 return rc;
5872}
5873
5874
5875/**
5876 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5877 *
5878 * @returns VBox status code.
5879 * @param pVCpu The cross context virtual CPU structure.
5880 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5881 * out-of-sync. Make sure to update the required fields
5882 * before using them.
5883 *
5884 * @remarks No-long-jump zone!!!
5885 */
5886static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5887{
5888 int rc = VINF_SUCCESS;
5889 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
5890 {
5891 uint64_t u64Val = 0;
5892 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5893 AssertRCReturn(rc, rc);
5894
5895 pMixedCtx->rip = u64Val;
5896 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
5897 }
5898 return rc;
5899}
5900
5901
5902/**
5903 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
5904 *
5905 * @returns VBox status code.
5906 * @param pVCpu The cross context virtual CPU structure.
5907 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5908 * out-of-sync. Make sure to update the required fields
5909 * before using them.
5910 *
5911 * @remarks No-long-jump zone!!!
5912 */
5913static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5914{
5915 int rc = VINF_SUCCESS;
5916 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
5917 {
5918 uint64_t u64Val = 0;
5919 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5920 AssertRCReturn(rc, rc);
5921
5922 pMixedCtx->rsp = u64Val;
5923 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
5924 }
5925 return rc;
5926}
5927
5928
5929/**
5930 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5931 *
5932 * @returns VBox status code.
5933 * @param pVCpu The cross context virtual CPU structure.
5934 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5935 * out-of-sync. Make sure to update the required fields
5936 * before using them.
5937 *
5938 * @remarks No-long-jump zone!!!
5939 */
5940static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5941{
5942 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
5943 {
5944 uint32_t uVal = 0;
5945 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5946 AssertRCReturn(rc, rc);
5947
5948 pMixedCtx->eflags.u32 = uVal;
5949 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5950 {
5951 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5952 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5953
5954 pMixedCtx->eflags.Bits.u1VM = 0;
5955 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5956 }
5957
5958 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
5959 }
5960 return VINF_SUCCESS;
5961}
5962
5963
5964/**
5965 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5966 * guest-CPU context.
5967 */
5968DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5969{
5970 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5971 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5972 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5973 return rc;
5974}
5975
5976
5977/**
5978 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5979 * from the guest-state area in the VMCS.
5980 *
5981 * @param pVCpu The cross context virtual CPU structure.
5982 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5983 * out-of-sync. Make sure to update the required fields
5984 * before using them.
5985 *
5986 * @remarks No-long-jump zone!!!
5987 */
5988static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5989{
5990 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
5991 {
5992 uint32_t uIntrState = 0;
5993 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5994 AssertRC(rc);
5995
5996 if (!uIntrState)
5997 {
5998 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5999 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6000
6001 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6002 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6003 }
6004 else
6005 {
6006 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6007 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6008 {
6009 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6010 AssertRC(rc);
6011 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6012 AssertRC(rc);
6013
6014 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6015 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6016 }
6017 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6018 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6019
6020 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6021 {
6022 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6023 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6024 }
6025 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6026 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6027 }
6028
6029 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6030 }
6031}
6032
6033
6034/**
6035 * Saves the guest's activity state.
6036 *
6037 * @returns VBox status code.
6038 * @param pVCpu The cross context virtual CPU structure.
6039 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6040 * out-of-sync. Make sure to update the required fields
6041 * before using them.
6042 *
6043 * @remarks No-long-jump zone!!!
6044 */
6045static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6046{
6047 NOREF(pMixedCtx);
6048 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6049 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6050 return VINF_SUCCESS;
6051}
6052
6053
6054/**
6055 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6056 * the current VMCS into the guest-CPU context.
6057 *
6058 * @returns VBox status code.
6059 * @param pVCpu The cross context virtual CPU structure.
6060 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6061 * out-of-sync. Make sure to update the required fields
6062 * before using them.
6063 *
6064 * @remarks No-long-jump zone!!!
6065 */
6066static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6067{
6068 int rc = VINF_SUCCESS;
6069 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6070 {
6071 uint32_t u32Val = 0;
6072 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6073 pMixedCtx->SysEnter.cs = u32Val;
6074 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6075 }
6076
6077 uint64_t u64Val = 0;
6078 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6079 {
6080 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6081 pMixedCtx->SysEnter.eip = u64Val;
6082 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6083 }
6084 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6085 {
6086 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6087 pMixedCtx->SysEnter.esp = u64Val;
6088 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6089 }
6090 return rc;
6091}
6092
6093
6094/**
6095 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6096 * the CPU back into the guest-CPU context.
6097 *
6098 * @returns VBox status code.
6099 * @param pVCpu The cross context virtual CPU structure.
6100 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6101 * out-of-sync. Make sure to update the required fields
6102 * before using them.
6103 *
6104 * @remarks No-long-jump zone!!!
6105 */
6106static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6107{
6108#if HC_ARCH_BITS == 64
6109 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
6110 {
6111 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6112 VMMRZCallRing3Disable(pVCpu);
6113 HM_DISABLE_PREEMPT();
6114
6115 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6116 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6117 {
6118 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6119 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6120 }
6121
6122 HM_RESTORE_PREEMPT();
6123 VMMRZCallRing3Enable(pVCpu);
6124 }
6125 else
6126 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6127#else
6128 NOREF(pMixedCtx);
6129 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6130#endif
6131
6132 return VINF_SUCCESS;
6133}
6134
6135
6136/**
6137 * Saves the auto load/store'd guest MSRs from the current VMCS into
6138 * the guest-CPU context.
6139 *
6140 * @returns VBox status code.
6141 * @param pVCpu The cross context virtual CPU structure.
6142 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6143 * out-of-sync. Make sure to update the required fields
6144 * before using them.
6145 *
6146 * @remarks No-long-jump zone!!!
6147 */
6148static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6149{
6150 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6151 return VINF_SUCCESS;
6152
6153 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6154 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6155 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6156 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6157 {
6158 switch (pMsr->u32Msr)
6159 {
6160 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6161 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6162 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6163 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6164 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6165 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6166 break;
6167
6168 default:
6169 {
6170 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6171 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6172 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6173 }
6174 }
6175 }
6176
6177 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6178 return VINF_SUCCESS;
6179}
6180
6181
6182/**
6183 * Saves the guest control registers from the current VMCS into the guest-CPU
6184 * context.
6185 *
6186 * @returns VBox status code.
6187 * @param pVCpu The cross context virtual CPU structure.
6188 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6189 * out-of-sync. Make sure to update the required fields
6190 * before using them.
6191 *
6192 * @remarks No-long-jump zone!!!
6193 */
6194static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6195{
6196 /* Guest CR0. Guest FPU. */
6197 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6198 AssertRCReturn(rc, rc);
6199
6200 /* Guest CR4. */
6201 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6202 AssertRCReturn(rc, rc);
6203
6204 /* Guest CR2 - updated always during the world-switch or in #PF. */
6205 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6206 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6207 {
6208 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6209 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6210
6211 PVM pVM = pVCpu->CTX_SUFF(pVM);
6212 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6213 || ( pVM->hm.s.fNestedPaging
6214 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6215 {
6216 uint64_t u64Val = 0;
6217 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6218 if (pMixedCtx->cr3 != u64Val)
6219 {
6220 CPUMSetGuestCR3(pVCpu, u64Val);
6221 if (VMMRZCallRing3IsEnabled(pVCpu))
6222 {
6223 PGMUpdateCR3(pVCpu, u64Val);
6224 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6225 }
6226 else
6227 {
6228 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6229 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6230 }
6231 }
6232
6233 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6234 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6235 {
6236 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6237 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6238 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6239 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6240 AssertRCReturn(rc, rc);
6241
6242 if (VMMRZCallRing3IsEnabled(pVCpu))
6243 {
6244 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6245 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6246 }
6247 else
6248 {
6249 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6250 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6251 }
6252 }
6253 }
6254
6255 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6256 }
6257
6258 /*
6259 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6260 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6261 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6262 *
6263 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6264 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6265 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6266 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6267 *
6268 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6269 */
6270 if (VMMRZCallRing3IsEnabled(pVCpu))
6271 {
6272 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6273 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6274
6275 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6276 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6277
6278 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6279 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6280 }
6281
6282 return rc;
6283}
6284
6285
6286/**
6287 * Reads a guest segment register from the current VMCS into the guest-CPU
6288 * context.
6289 *
6290 * @returns VBox status code.
6291 * @param pVCpu The cross context virtual CPU structure.
6292 * @param idxSel Index of the selector in the VMCS.
6293 * @param idxLimit Index of the segment limit in the VMCS.
6294 * @param idxBase Index of the segment base in the VMCS.
6295 * @param idxAccess Index of the access rights of the segment in the VMCS.
6296 * @param pSelReg Pointer to the segment selector.
6297 *
6298 * @remarks No-long-jump zone!!!
6299 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6300 * macro as that takes care of whether to read from the VMCS cache or
6301 * not.
6302 */
6303DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6304 PCPUMSELREG pSelReg)
6305{
6306 NOREF(pVCpu);
6307
6308 uint32_t u32Val = 0;
6309 int rc = VMXReadVmcs32(idxSel, &u32Val);
6310 AssertRCReturn(rc, rc);
6311 pSelReg->Sel = (uint16_t)u32Val;
6312 pSelReg->ValidSel = (uint16_t)u32Val;
6313 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6314
6315 rc = VMXReadVmcs32(idxLimit, &u32Val);
6316 AssertRCReturn(rc, rc);
6317 pSelReg->u32Limit = u32Val;
6318
6319 uint64_t u64Val = 0;
6320 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6321 AssertRCReturn(rc, rc);
6322 pSelReg->u64Base = u64Val;
6323
6324 rc = VMXReadVmcs32(idxAccess, &u32Val);
6325 AssertRCReturn(rc, rc);
6326 pSelReg->Attr.u = u32Val;
6327
6328 /*
6329 * If VT-x marks the segment as unusable, most other bits remain undefined:
6330 * - For CS the L, D and G bits have meaning.
6331 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6332 * - For the remaining data segments no bits are defined.
6333 *
6334 * The present bit and the unusable bit has been observed to be set at the
6335 * same time (the selector was supposed to be invalid as we started executing
6336 * a V8086 interrupt in ring-0).
6337 *
6338 * What should be important for the rest of the VBox code, is that the P bit is
6339 * cleared. Some of the other VBox code recognizes the unusable bit, but
6340 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6341 * safe side here, we'll strip off P and other bits we don't care about. If
6342 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6343 *
6344 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6345 */
6346 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6347 {
6348 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6349
6350 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6351 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6352 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6353
6354 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6355#ifdef DEBUG_bird
6356 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6357 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6358 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6359#endif
6360 }
6361 return VINF_SUCCESS;
6362}
6363
6364
6365#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6366# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6367 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6368 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6369#else
6370# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6371 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6372 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6373#endif
6374
6375
6376/**
6377 * Saves the guest segment registers from the current VMCS into the guest-CPU
6378 * context.
6379 *
6380 * @returns VBox status code.
6381 * @param pVCpu The cross context virtual CPU structure.
6382 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6383 * out-of-sync. Make sure to update the required fields
6384 * before using them.
6385 *
6386 * @remarks No-long-jump zone!!!
6387 */
6388static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6389{
6390 /* Guest segment registers. */
6391 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6392 {
6393 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6394 AssertRCReturn(rc, rc);
6395
6396 rc = VMXLOCAL_READ_SEG(CS, cs);
6397 rc |= VMXLOCAL_READ_SEG(SS, ss);
6398 rc |= VMXLOCAL_READ_SEG(DS, ds);
6399 rc |= VMXLOCAL_READ_SEG(ES, es);
6400 rc |= VMXLOCAL_READ_SEG(FS, fs);
6401 rc |= VMXLOCAL_READ_SEG(GS, gs);
6402 AssertRCReturn(rc, rc);
6403
6404 /* Restore segment attributes for real-on-v86 mode hack. */
6405 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6406 {
6407 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6408 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6409 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6410 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6411 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6412 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6413 }
6414 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6415 }
6416
6417 return VINF_SUCCESS;
6418}
6419
6420
6421/**
6422 * Saves the guest descriptor table registers and task register from the current
6423 * VMCS into the guest-CPU context.
6424 *
6425 * @returns VBox status code.
6426 * @param pVCpu The cross context virtual CPU structure.
6427 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6428 * out-of-sync. Make sure to update the required fields
6429 * before using them.
6430 *
6431 * @remarks No-long-jump zone!!!
6432 */
6433static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6434{
6435 int rc = VINF_SUCCESS;
6436
6437 /* Guest LDTR. */
6438 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6439 {
6440 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6441 AssertRCReturn(rc, rc);
6442 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6443 }
6444
6445 /* Guest GDTR. */
6446 uint64_t u64Val = 0;
6447 uint32_t u32Val = 0;
6448 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6449 {
6450 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6451 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6452 pMixedCtx->gdtr.pGdt = u64Val;
6453 pMixedCtx->gdtr.cbGdt = u32Val;
6454 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6455 }
6456
6457 /* Guest IDTR. */
6458 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6459 {
6460 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6461 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6462 pMixedCtx->idtr.pIdt = u64Val;
6463 pMixedCtx->idtr.cbIdt = u32Val;
6464 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6465 }
6466
6467 /* Guest TR. */
6468 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6469 {
6470 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6471 AssertRCReturn(rc, rc);
6472
6473 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6474 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6475 {
6476 rc = VMXLOCAL_READ_SEG(TR, tr);
6477 AssertRCReturn(rc, rc);
6478 }
6479 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6480 }
6481 return rc;
6482}
6483
6484#undef VMXLOCAL_READ_SEG
6485
6486
6487/**
6488 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6489 * context.
6490 *
6491 * @returns VBox status code.
6492 * @param pVCpu The cross context virtual CPU structure.
6493 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6494 * out-of-sync. Make sure to update the required fields
6495 * before using them.
6496 *
6497 * @remarks No-long-jump zone!!!
6498 */
6499static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6500{
6501 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6502 {
6503 if (!pVCpu->hm.s.fUsingHyperDR7)
6504 {
6505 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6506 uint32_t u32Val;
6507 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6508 pMixedCtx->dr[7] = u32Val;
6509 }
6510
6511 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6512 }
6513 return VINF_SUCCESS;
6514}
6515
6516
6517/**
6518 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6519 *
6520 * @returns VBox status code.
6521 * @param pVCpu The cross context virtual CPU structure.
6522 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6523 * out-of-sync. Make sure to update the required fields
6524 * before using them.
6525 *
6526 * @remarks No-long-jump zone!!!
6527 */
6528static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6529{
6530 NOREF(pMixedCtx);
6531
6532 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6533 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6534 return VINF_SUCCESS;
6535}
6536
6537
6538/**
6539 * Saves the entire guest state from the currently active VMCS into the
6540 * guest-CPU context.
6541 *
6542 * This essentially VMREADs all guest-data.
6543 *
6544 * @returns VBox status code.
6545 * @param pVCpu The cross context virtual CPU structure.
6546 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6547 * out-of-sync. Make sure to update the required fields
6548 * before using them.
6549 */
6550static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6551{
6552 Assert(pVCpu);
6553 Assert(pMixedCtx);
6554
6555 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6556 return VINF_SUCCESS;
6557
6558 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6559 again on the ring-3 callback path, there is no real need to. */
6560 if (VMMRZCallRing3IsEnabled(pVCpu))
6561 VMMR0LogFlushDisable(pVCpu);
6562 else
6563 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6564 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6565
6566 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6567 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6568
6569 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6570 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6571
6572 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6573 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6574
6575 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6576 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6577
6578 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6579 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6580
6581 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6582 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6583
6584 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6585 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6586
6587 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6588 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6589
6590 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6591 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6592
6593 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6594 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6595
6596 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6597 ("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
6598 HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
6599
6600 if (VMMRZCallRing3IsEnabled(pVCpu))
6601 VMMR0LogFlushEnable(pVCpu);
6602
6603 return VINF_SUCCESS;
6604}
6605
6606
6607/**
6608 * Saves basic guest registers needed for IEM instruction execution.
6609 *
6610 * @returns VBox status code (OR-able).
6611 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6612 * @param pMixedCtx Pointer to the CPU context of the guest.
6613 * @param fMemory Whether the instruction being executed operates on
6614 * memory or not. Only CR0 is synced up if clear.
6615 * @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
6616 */
6617static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
6618{
6619 /*
6620 * We assume all general purpose registers other than RSP are available.
6621 *
6622 * RIP is a must, as it will be incremented or otherwise changed.
6623 *
6624 * RFLAGS are always required to figure the CPL.
6625 *
6626 * RSP isn't always required, however it's a GPR, so frequently required.
6627 *
6628 * SS and CS are the only segment register needed if IEM doesn't do memory
6629 * access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
6630 *
6631 * CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
6632 * be required for memory accesses.
6633 *
6634 * Note! Before IEM dispatches an exception, it will call us to sync in everything.
6635 */
6636 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6637 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6638 if (fNeedRsp)
6639 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6640 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6641 if (!fMemory)
6642 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6643 else
6644 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6645 AssertRCReturn(rc, rc);
6646 return rc;
6647}
6648
6649
6650/**
6651 * Ensures that we've got a complete basic guest-context.
6652 *
6653 * This excludes the FPU, SSE, AVX, and similar extended state. The interface
6654 * is for the interpreter.
6655 *
6656 * @returns VBox status code.
6657 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6658 * @param pMixedCtx Pointer to the guest-CPU context which may have data
6659 * needing to be synced in.
6660 * @thread EMT(pVCpu)
6661 */
6662VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6663{
6664 /* Note! Since this is only applicable to VT-x, the implementation is placed
6665 in the VT-x part of the sources instead of the generic stuff. */
6666 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
6667 return hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6668 return VINF_SUCCESS;
6669}
6670
6671
6672/**
6673 * Check per-VM and per-VCPU force flag actions that require us to go back to
6674 * ring-3 for one reason or another.
6675 *
6676 * @returns Strict VBox status code (i.e. informational status codes too)
6677 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6678 * ring-3.
6679 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6680 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6681 * interrupts)
6682 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6683 * all EMTs to be in ring-3.
6684 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6685 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6686 * to the EM loop.
6687 *
6688 * @param pVM The cross context VM structure.
6689 * @param pVCpu The cross context virtual CPU structure.
6690 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6691 * out-of-sync. Make sure to update the required fields
6692 * before using them.
6693 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6694 */
6695static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6696{
6697 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6698
6699 /*
6700 * Anything pending? Should be more likely than not if we're doing a good job.
6701 */
6702 if ( !fStepping
6703 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6704 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6705 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6706 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6707 return VINF_SUCCESS;
6708
6709 /* We need the control registers now, make sure the guest-CPU context is updated. */
6710 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6711 AssertRCReturn(rc3, rc3);
6712
6713 /* Pending HM CR3 sync. */
6714 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6715 {
6716 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6717 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6718 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6719 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6720 }
6721
6722 /* Pending HM PAE PDPEs. */
6723 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6724 {
6725 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6726 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6727 }
6728
6729 /* Pending PGM C3 sync. */
6730 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6731 {
6732 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6733 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6734 if (rcStrict2 != VINF_SUCCESS)
6735 {
6736 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6737 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6738 return rcStrict2;
6739 }
6740 }
6741
6742 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6743 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6744 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6745 {
6746 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6747 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6748 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6749 return rc2;
6750 }
6751
6752 /* Pending VM request packets, such as hardware interrupts. */
6753 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6754 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6755 {
6756 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6757 return VINF_EM_PENDING_REQUEST;
6758 }
6759
6760 /* Pending PGM pool flushes. */
6761 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6762 {
6763 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6764 return VINF_PGM_POOL_FLUSH_PENDING;
6765 }
6766
6767 /* Pending DMA requests. */
6768 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6769 {
6770 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6771 return VINF_EM_RAW_TO_R3;
6772 }
6773
6774 return VINF_SUCCESS;
6775}
6776
6777
6778/**
6779 * Converts any TRPM trap into a pending HM event. This is typically used when
6780 * entering from ring-3 (not longjmp returns).
6781 *
6782 * @param pVCpu The cross context virtual CPU structure.
6783 */
6784static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6785{
6786 Assert(TRPMHasTrap(pVCpu));
6787 Assert(!pVCpu->hm.s.Event.fPending);
6788
6789 uint8_t uVector;
6790 TRPMEVENT enmTrpmEvent;
6791 RTGCUINT uErrCode;
6792 RTGCUINTPTR GCPtrFaultAddress;
6793 uint8_t cbInstr;
6794
6795 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6796 AssertRC(rc);
6797
6798 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6799 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6800 if (enmTrpmEvent == TRPM_TRAP)
6801 {
6802 switch (uVector)
6803 {
6804 case X86_XCPT_NMI:
6805 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6806 break;
6807
6808 case X86_XCPT_BP:
6809 case X86_XCPT_OF:
6810 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6811 break;
6812
6813 case X86_XCPT_PF:
6814 case X86_XCPT_DF:
6815 case X86_XCPT_TS:
6816 case X86_XCPT_NP:
6817 case X86_XCPT_SS:
6818 case X86_XCPT_GP:
6819 case X86_XCPT_AC:
6820 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6821 /* no break! */
6822 default:
6823 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6824 break;
6825 }
6826 }
6827 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6828 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6829 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6830 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6831 else
6832 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6833
6834 rc = TRPMResetTrap(pVCpu);
6835 AssertRC(rc);
6836 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6837 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6838
6839 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6840 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6841}
6842
6843
6844/**
6845 * Converts the pending HM event into a TRPM trap.
6846 *
6847 * @param pVCpu The cross context virtual CPU structure.
6848 */
6849static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6850{
6851 Assert(pVCpu->hm.s.Event.fPending);
6852
6853 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6854 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6855 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6856 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6857
6858 /* If a trap was already pending, we did something wrong! */
6859 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6860
6861 TRPMEVENT enmTrapType;
6862 switch (uVectorType)
6863 {
6864 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6865 enmTrapType = TRPM_HARDWARE_INT;
6866 break;
6867
6868 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6869 enmTrapType = TRPM_SOFTWARE_INT;
6870 break;
6871
6872 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6873 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6874 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6875 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6876 enmTrapType = TRPM_TRAP;
6877 break;
6878
6879 default:
6880 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6881 enmTrapType = TRPM_32BIT_HACK;
6882 break;
6883 }
6884
6885 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6886
6887 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6888 AssertRC(rc);
6889
6890 if (fErrorCodeValid)
6891 TRPMSetErrorCode(pVCpu, uErrorCode);
6892
6893 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6894 && uVector == X86_XCPT_PF)
6895 {
6896 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6897 }
6898 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6899 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6900 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6901 {
6902 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6903 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6904 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6905 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6906 }
6907
6908 /* Clear any pending events from the VMCS. */
6909 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
6910 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
6911
6912 /* We're now done converting the pending event. */
6913 pVCpu->hm.s.Event.fPending = false;
6914}
6915
6916
6917/**
6918 * Does the necessary state syncing before returning to ring-3 for any reason
6919 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6920 *
6921 * @returns VBox status code.
6922 * @param pVM The cross context VM structure.
6923 * @param pVCpu The cross context virtual CPU structure.
6924 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6925 * be out-of-sync. Make sure to update the required
6926 * fields before using them.
6927 * @param fSaveGuestState Whether to save the guest state or not.
6928 *
6929 * @remarks No-long-jmp zone!!!
6930 */
6931static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6932{
6933 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6934 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6935
6936 RTCPUID idCpu = RTMpCpuId();
6937 Log4Func(("HostCpuId=%u\n", idCpu));
6938
6939 /*
6940 * !!! IMPORTANT !!!
6941 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
6942 */
6943
6944 /* Save the guest state if necessary. */
6945 if ( fSaveGuestState
6946 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6947 {
6948 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6949 AssertRCReturn(rc, rc);
6950 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6951 }
6952
6953 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6954 if (CPUMIsGuestFPUStateActive(pVCpu))
6955 {
6956 /* We shouldn't reload CR0 without saving it first. */
6957 if (!fSaveGuestState)
6958 {
6959 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6960 AssertRCReturn(rc, rc);
6961 }
6962 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6963 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6964 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6965 }
6966
6967 /* Restore host debug registers if necessary and resync on next R0 reentry. */
6968#ifdef VBOX_STRICT
6969 if (CPUMIsHyperDebugStateActive(pVCpu))
6970 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6971#endif
6972 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6973 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6974 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6975 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6976
6977#if HC_ARCH_BITS == 64
6978 /* Restore host-state bits that VT-x only restores partially. */
6979 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6980 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6981 {
6982 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6983 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6984 }
6985 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6986#endif
6987
6988#if HC_ARCH_BITS == 64
6989 /* Restore the lazy host MSRs as we're leaving VT-x context. */
6990 if ( pVM->hm.s.fAllow64BitGuests
6991 && pVCpu->hm.s.vmx.fLazyMsrs)
6992 {
6993 /* We shouldn't reload the guest MSRs without saving it first. */
6994 if (!fSaveGuestState)
6995 {
6996 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6997 AssertRCReturn(rc, rc);
6998 }
6999 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7000 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7001 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7002 }
7003#endif
7004
7005 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7006 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7007
7008 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7009 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7010 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7011 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7012 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7013 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7014 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7015 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7016
7017 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7018
7019 /** @todo This partially defeats the purpose of having preemption hooks.
7020 * The problem is, deregistering the hooks should be moved to a place that
7021 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7022 * context.
7023 */
7024 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7025 {
7026 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7027 AssertRCReturn(rc, rc);
7028
7029 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7030 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7031 }
7032 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7033 NOREF(idCpu);
7034
7035 return VINF_SUCCESS;
7036}
7037
7038
7039/**
7040 * Leaves the VT-x session.
7041 *
7042 * @returns VBox status code.
7043 * @param pVM The cross context VM structure.
7044 * @param pVCpu The cross context virtual CPU structure.
7045 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7046 * out-of-sync. Make sure to update the required fields
7047 * before using them.
7048 *
7049 * @remarks No-long-jmp zone!!!
7050 */
7051DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7052{
7053 HM_DISABLE_PREEMPT();
7054 HMVMX_ASSERT_CPU_SAFE();
7055 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7056 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7057
7058 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7059 and done this from the VMXR0ThreadCtxCallback(). */
7060 if (!pVCpu->hm.s.fLeaveDone)
7061 {
7062 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7063 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7064 pVCpu->hm.s.fLeaveDone = true;
7065 }
7066 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7067
7068 /*
7069 * !!! IMPORTANT !!!
7070 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7071 */
7072
7073 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7074 /** @todo Deregistering here means we need to VMCLEAR always
7075 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7076 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7077 VMMR0ThreadCtxHookDisable(pVCpu);
7078
7079 /* Leave HM context. This takes care of local init (term). */
7080 int rc = HMR0LeaveCpu(pVCpu);
7081
7082 HM_RESTORE_PREEMPT();
7083 return rc;
7084}
7085
7086
7087/**
7088 * Does the necessary state syncing before doing a longjmp to ring-3.
7089 *
7090 * @returns VBox status code.
7091 * @param pVM The cross context VM structure.
7092 * @param pVCpu The cross context virtual CPU structure.
7093 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7094 * out-of-sync. Make sure to update the required fields
7095 * before using them.
7096 *
7097 * @remarks No-long-jmp zone!!!
7098 */
7099DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7100{
7101 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7102}
7103
7104
7105/**
7106 * Take necessary actions before going back to ring-3.
7107 *
7108 * An action requires us to go back to ring-3. This function does the necessary
7109 * steps before we can safely return to ring-3. This is not the same as longjmps
7110 * to ring-3, this is voluntary and prepares the guest so it may continue
7111 * executing outside HM (recompiler/IEM).
7112 *
7113 * @returns VBox status code.
7114 * @param pVM The cross context VM structure.
7115 * @param pVCpu The cross context virtual CPU structure.
7116 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7117 * out-of-sync. Make sure to update the required fields
7118 * before using them.
7119 * @param rcExit The reason for exiting to ring-3. Can be
7120 * VINF_VMM_UNKNOWN_RING3_CALL.
7121 */
7122static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7123{
7124 Assert(pVM);
7125 Assert(pVCpu);
7126 Assert(pMixedCtx);
7127 HMVMX_ASSERT_PREEMPT_SAFE();
7128
7129 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7130 {
7131 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7132 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7133 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7134 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7135 }
7136
7137 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7138 VMMRZCallRing3Disable(pVCpu);
7139 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7140
7141 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7142 if (pVCpu->hm.s.Event.fPending)
7143 {
7144 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7145 Assert(!pVCpu->hm.s.Event.fPending);
7146 }
7147
7148 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7149 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7150
7151 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7152 and if we're injecting an event we should have a TRPM trap pending. */
7153 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7154#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7155 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7156#endif
7157
7158 /* Save guest state and restore host state bits. */
7159 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7160 AssertRCReturn(rc, rc);
7161 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7162 /* Thread-context hooks are unregistered at this point!!! */
7163
7164 /* Sync recompiler state. */
7165 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7166 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7167 | CPUM_CHANGED_LDTR
7168 | CPUM_CHANGED_GDTR
7169 | CPUM_CHANGED_IDTR
7170 | CPUM_CHANGED_TR
7171 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7172 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7173 if ( pVM->hm.s.fNestedPaging
7174 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7175 {
7176 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7177 }
7178
7179 Assert(!pVCpu->hm.s.fClearTrapFlag);
7180
7181 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7182 if (rcExit != VINF_EM_RAW_INTERRUPT)
7183 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7184
7185 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7186
7187 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7188 VMMRZCallRing3RemoveNotification(pVCpu);
7189 VMMRZCallRing3Enable(pVCpu);
7190
7191 return rc;
7192}
7193
7194
7195/**
7196 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7197 * longjump to ring-3 and possibly get preempted.
7198 *
7199 * @returns VBox status code.
7200 * @param pVCpu The cross context virtual CPU structure.
7201 * @param enmOperation The operation causing the ring-3 longjump.
7202 * @param pvUser Opaque pointer to the guest-CPU context. The data
7203 * may be out-of-sync. Make sure to update the required
7204 * fields before using them.
7205 */
7206static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7207{
7208 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7209 {
7210 /*
7211 * !!! IMPORTANT !!!
7212 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7213 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7214 */
7215 VMMRZCallRing3RemoveNotification(pVCpu);
7216 VMMRZCallRing3Disable(pVCpu);
7217 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7218 RTThreadPreemptDisable(&PreemptState);
7219
7220 PVM pVM = pVCpu->CTX_SUFF(pVM);
7221 if (CPUMIsGuestFPUStateActive(pVCpu))
7222 CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
7223
7224 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7225
7226#if HC_ARCH_BITS == 64
7227 /* Restore host-state bits that VT-x only restores partially. */
7228 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7229 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7230 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7231 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7232
7233 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7234 if ( pVM->hm.s.fAllow64BitGuests
7235 && pVCpu->hm.s.vmx.fLazyMsrs)
7236 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7237#endif
7238 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7239 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7240 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7241 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7242 {
7243 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7244 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7245 }
7246
7247 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7248 VMMR0ThreadCtxHookDisable(pVCpu);
7249 HMR0LeaveCpu(pVCpu);
7250 RTThreadPreemptRestore(&PreemptState);
7251 return VINF_SUCCESS;
7252 }
7253
7254 Assert(pVCpu);
7255 Assert(pvUser);
7256 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7257 HMVMX_ASSERT_PREEMPT_SAFE();
7258
7259 VMMRZCallRing3Disable(pVCpu);
7260 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7261
7262 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7263 enmOperation));
7264
7265 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7266 AssertRCReturn(rc, rc);
7267
7268 VMMRZCallRing3Enable(pVCpu);
7269 return VINF_SUCCESS;
7270}
7271
7272
7273/**
7274 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7275 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7276 *
7277 * @param pVCpu The cross context virtual CPU structure.
7278 */
7279DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7280{
7281 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7282 {
7283 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7284 {
7285 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7286 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7287 AssertRC(rc);
7288 Log4(("Setup interrupt-window exiting\n"));
7289 }
7290 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7291}
7292
7293
7294/**
7295 * Clears the interrupt-window exiting control in the VMCS.
7296 *
7297 * @param pVCpu The cross context virtual CPU structure.
7298 */
7299DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7300{
7301 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7302 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7303 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7304 AssertRC(rc);
7305 Log4(("Cleared interrupt-window exiting\n"));
7306}
7307
7308
7309/**
7310 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7311 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7312 *
7313 * @param pVCpu The cross context virtual CPU structure.
7314 */
7315DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7316{
7317 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7318 {
7319 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7320 {
7321 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7322 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7323 AssertRC(rc);
7324 Log4(("Setup NMI-window exiting\n"));
7325 }
7326 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7327}
7328
7329
7330/**
7331 * Clears the NMI-window exiting control in the VMCS.
7332 *
7333 * @param pVCpu The cross context virtual CPU structure.
7334 */
7335DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7336{
7337 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7338 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7339 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7340 AssertRC(rc);
7341 Log4(("Cleared NMI-window exiting\n"));
7342}
7343
7344
7345/**
7346 * Evaluates the event to be delivered to the guest and sets it as the pending
7347 * event.
7348 *
7349 * @param pVCpu The cross context virtual CPU structure.
7350 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7351 * out-of-sync. Make sure to update the required fields
7352 * before using them.
7353 */
7354static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7355{
7356 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7357 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7358 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7359 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7360 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7361
7362 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7363 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7364 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7365 Assert(!TRPMHasTrap(pVCpu));
7366
7367 /*
7368 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7369 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7370 */
7371 /** @todo SMI. SMIs take priority over NMIs. */
7372 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7373 {
7374 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7375 if ( !pVCpu->hm.s.Event.fPending
7376 && !fBlockNmi
7377 && !fBlockSti
7378 && !fBlockMovSS)
7379 {
7380 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7381 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7382 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7383
7384 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7385 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7386 }
7387 else
7388 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7389 }
7390 /*
7391 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7392 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt.
7393 */
7394 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7395 && !pVCpu->hm.s.fSingleInstruction)
7396 {
7397 Assert(!DBGFIsStepping(pVCpu));
7398 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7399 AssertRC(rc);
7400 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7401 if ( !pVCpu->hm.s.Event.fPending
7402 && !fBlockInt
7403 && !fBlockSti
7404 && !fBlockMovSS)
7405 {
7406 uint8_t u8Interrupt;
7407 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7408 if (RT_SUCCESS(rc))
7409 {
7410 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7411 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7412 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7413
7414 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7415 }
7416 else
7417 {
7418 /** @todo Does this actually happen? If not turn it into an assertion. */
7419 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
7420 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7421 }
7422 }
7423 else
7424 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7425 }
7426}
7427
7428
7429/**
7430 * Sets a pending-debug exception to be delivered to the guest if the guest is
7431 * single-stepping in the VMCS.
7432 *
7433 * @param pVCpu The cross context virtual CPU structure.
7434 */
7435DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7436{
7437 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7438 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7439 AssertRC(rc);
7440}
7441
7442
7443/**
7444 * Injects any pending events into the guest if the guest is in a state to
7445 * receive them.
7446 *
7447 * @returns Strict VBox status code (i.e. informational status codes too).
7448 * @param pVCpu The cross context virtual CPU structure.
7449 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7450 * out-of-sync. Make sure to update the required fields
7451 * before using them.
7452 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7453 * return VINF_EM_DBG_STEPPED if the event was
7454 * dispatched directly.
7455 */
7456static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
7457{
7458 HMVMX_ASSERT_PREEMPT_SAFE();
7459 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7460
7461 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7462 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7463 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7464 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7465
7466 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7467 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7468 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7469 Assert(!TRPMHasTrap(pVCpu));
7470
7471 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7472 if (pVCpu->hm.s.Event.fPending)
7473 {
7474 /*
7475 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7476 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7477 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7478 *
7479 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7480 */
7481 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7482#ifdef VBOX_STRICT
7483 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7484 {
7485 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7486 Assert(!fBlockInt);
7487 Assert(!fBlockSti);
7488 Assert(!fBlockMovSS);
7489 }
7490 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7491 {
7492 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7493 Assert(!fBlockSti);
7494 Assert(!fBlockMovSS);
7495 Assert(!fBlockNmi);
7496 }
7497#endif
7498 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7499 (uint8_t)uIntType));
7500 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7501 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7502 fStepping, &uIntrState);
7503 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7504
7505 /* Update the interruptibility-state as it could have been changed by
7506 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7507 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7508 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7509
7510 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7511 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7512 else
7513 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7514 }
7515
7516 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7517 if ( fBlockSti
7518 || fBlockMovSS)
7519 {
7520 if (!pVCpu->hm.s.fSingleInstruction)
7521 {
7522 /*
7523 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7524 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7525 * See Intel spec. 27.3.4 "Saving Non-Register State".
7526 */
7527 Assert(!DBGFIsStepping(pVCpu));
7528 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7529 AssertRCReturn(rc2, rc2);
7530 if (pMixedCtx->eflags.Bits.u1TF)
7531 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7532 }
7533 else if (pMixedCtx->eflags.Bits.u1TF)
7534 {
7535 /*
7536 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7537 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7538 */
7539 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7540 uIntrState = 0;
7541 }
7542 }
7543
7544 /*
7545 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7546 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7547 */
7548 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7549 AssertRC(rc2);
7550
7551 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7552 NOREF(fBlockMovSS); NOREF(fBlockSti);
7553 return rcStrict;
7554}
7555
7556
7557/**
7558 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7559 *
7560 * @param pVCpu The cross context virtual CPU structure.
7561 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7562 * out-of-sync. Make sure to update the required fields
7563 * before using them.
7564 */
7565DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7566{
7567 NOREF(pMixedCtx);
7568 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7569 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7570}
7571
7572
7573/**
7574 * Injects a double-fault (\#DF) exception into the VM.
7575 *
7576 * @returns Strict VBox status code (i.e. informational status codes too).
7577 * @param pVCpu The cross context virtual CPU structure.
7578 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7579 * out-of-sync. Make sure to update the required fields
7580 * before using them.
7581 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7582 * and should return VINF_EM_DBG_STEPPED if the event
7583 * is injected directly (register modified by us, not
7584 * by hardware on VM-entry).
7585 * @param puIntrState Pointer to the current guest interruptibility-state.
7586 * This interruptibility-state will be updated if
7587 * necessary. This cannot not be NULL.
7588 */
7589DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7590{
7591 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7592 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7593 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7594 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7595 fStepping, puIntrState);
7596}
7597
7598
7599/**
7600 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7601 *
7602 * @param pVCpu The cross context virtual CPU structure.
7603 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7604 * out-of-sync. Make sure to update the required fields
7605 * before using them.
7606 */
7607DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7608{
7609 NOREF(pMixedCtx);
7610 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7611 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7612 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7613}
7614
7615
7616/**
7617 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7618 *
7619 * @param pVCpu The cross context virtual CPU structure.
7620 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7621 * out-of-sync. Make sure to update the required fields
7622 * before using them.
7623 * @param cbInstr The value of RIP that is to be pushed on the guest
7624 * stack.
7625 */
7626DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7627{
7628 NOREF(pMixedCtx);
7629 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7630 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7631 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7632}
7633
7634
7635/**
7636 * Injects a general-protection (\#GP) fault into the VM.
7637 *
7638 * @returns Strict VBox status code (i.e. informational status codes too).
7639 * @param pVCpu The cross context virtual CPU structure.
7640 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7641 * out-of-sync. Make sure to update the required fields
7642 * before using them.
7643 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7644 * mode, i.e. in real-mode it's not valid).
7645 * @param u32ErrorCode The error code associated with the \#GP.
7646 * @param fStepping Whether we're running in
7647 * hmR0VmxRunGuestCodeStep() and should return
7648 * VINF_EM_DBG_STEPPED if the event is injected
7649 * directly (register modified by us, not by
7650 * hardware on VM-entry).
7651 * @param puIntrState Pointer to the current guest interruptibility-state.
7652 * This interruptibility-state will be updated if
7653 * necessary. This cannot not be NULL.
7654 */
7655DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7656 bool fStepping, uint32_t *puIntrState)
7657{
7658 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7659 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7660 if (fErrorCodeValid)
7661 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7662 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7663 fStepping, puIntrState);
7664}
7665
7666
7667/**
7668 * Sets a general-protection (\#GP) exception as pending-for-injection into the
7669 * VM.
7670 *
7671 * @param pVCpu The cross context virtual CPU structure.
7672 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7673 * out-of-sync. Make sure to update the required fields
7674 * before using them.
7675 * @param u32ErrorCode The error code associated with the \#GP.
7676 */
7677DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7678{
7679 NOREF(pMixedCtx);
7680 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7681 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7682 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7683 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7684}
7685
7686
7687/**
7688 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7689 *
7690 * @param pVCpu The cross context virtual CPU structure.
7691 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7692 * out-of-sync. Make sure to update the required fields
7693 * before using them.
7694 * @param uVector The software interrupt vector number.
7695 * @param cbInstr The value of RIP that is to be pushed on the guest
7696 * stack.
7697 */
7698DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7699{
7700 NOREF(pMixedCtx);
7701 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7702 if ( uVector == X86_XCPT_BP
7703 || uVector == X86_XCPT_OF)
7704 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7705 else
7706 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7707 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7708}
7709
7710
7711/**
7712 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7713 * stack.
7714 *
7715 * @returns Strict VBox status code (i.e. informational status codes too).
7716 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7717 * @param pVM The cross context VM structure.
7718 * @param pMixedCtx Pointer to the guest-CPU context.
7719 * @param uValue The value to push to the guest stack.
7720 */
7721DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7722{
7723 /*
7724 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7725 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7726 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7727 */
7728 if (pMixedCtx->sp == 1)
7729 return VINF_EM_RESET;
7730 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7731 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7732 AssertRC(rc);
7733 return rc;
7734}
7735
7736
7737/**
7738 * Injects an event into the guest upon VM-entry by updating the relevant fields
7739 * in the VM-entry area in the VMCS.
7740 *
7741 * @returns Strict VBox status code (i.e. informational status codes too).
7742 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7743 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7744 *
7745 * @param pVCpu The cross context virtual CPU structure.
7746 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7747 * be out-of-sync. Make sure to update the required
7748 * fields before using them.
7749 * @param u64IntInfo The VM-entry interruption-information field.
7750 * @param cbInstr The VM-entry instruction length in bytes (for
7751 * software interrupts, exceptions and privileged
7752 * software exceptions).
7753 * @param u32ErrCode The VM-entry exception error code.
7754 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7755 * @param puIntrState Pointer to the current guest interruptibility-state.
7756 * This interruptibility-state will be updated if
7757 * necessary. This cannot not be NULL.
7758 * @param fStepping Whether we're running in
7759 * hmR0VmxRunGuestCodeStep() and should return
7760 * VINF_EM_DBG_STEPPED if the event is injected
7761 * directly (register modified by us, not by
7762 * hardware on VM-entry).
7763 *
7764 * @remarks Requires CR0!
7765 * @remarks No-long-jump zone!!!
7766 */
7767static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7768 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
7769 uint32_t *puIntrState)
7770{
7771 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7772 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7773 Assert(puIntrState);
7774 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7775
7776 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7777 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7778
7779#ifdef VBOX_STRICT
7780 /* Validate the error-code-valid bit for hardware exceptions. */
7781 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7782 {
7783 switch (uVector)
7784 {
7785 case X86_XCPT_PF:
7786 case X86_XCPT_DF:
7787 case X86_XCPT_TS:
7788 case X86_XCPT_NP:
7789 case X86_XCPT_SS:
7790 case X86_XCPT_GP:
7791 case X86_XCPT_AC:
7792 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7793 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7794 /* fallthru */
7795 default:
7796 break;
7797 }
7798 }
7799#endif
7800
7801 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7802 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7803 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7804
7805 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7806
7807 /* We require CR0 to check if the guest is in real-mode. */
7808 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7809 AssertRCReturn(rc, rc);
7810
7811 /*
7812 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7813 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7814 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7815 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7816 */
7817 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7818 {
7819 PVM pVM = pVCpu->CTX_SUFF(pVM);
7820 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7821 {
7822 Assert(PDMVmmDevHeapIsEnabled(pVM));
7823 Assert(pVM->hm.s.vmx.pRealModeTSS);
7824
7825 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7826 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7827 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7828 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7829 AssertRCReturn(rc, rc);
7830 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7831
7832 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7833 size_t const cbIdtEntry = sizeof(X86IDTR16);
7834 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7835 {
7836 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7837 if (uVector == X86_XCPT_DF)
7838 return VINF_EM_RESET;
7839
7840 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7841 if (uVector == X86_XCPT_GP)
7842 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
7843
7844 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7845 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7846 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
7847 fStepping, puIntrState);
7848 }
7849
7850 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7851 uint16_t uGuestIp = pMixedCtx->ip;
7852 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7853 {
7854 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7855 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7856 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7857 }
7858 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7859 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7860
7861 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7862 X86IDTR16 IdtEntry;
7863 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7864 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7865 AssertRCReturn(rc, rc);
7866
7867 /* Construct the stack frame for the interrupt/exception handler. */
7868 VBOXSTRICTRC rcStrict;
7869 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7870 if (rcStrict == VINF_SUCCESS)
7871 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7872 if (rcStrict == VINF_SUCCESS)
7873 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7874
7875 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7876 if (rcStrict == VINF_SUCCESS)
7877 {
7878 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7879 pMixedCtx->rip = IdtEntry.offSel;
7880 pMixedCtx->cs.Sel = IdtEntry.uSel;
7881 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7882 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7883 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7884 && uVector == X86_XCPT_PF)
7885 pMixedCtx->cr2 = GCPtrFaultAddress;
7886
7887 /* If any other guest-state bits are changed here, make sure to update
7888 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7889 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7890 | HM_CHANGED_GUEST_RIP
7891 | HM_CHANGED_GUEST_RFLAGS
7892 | HM_CHANGED_GUEST_RSP);
7893
7894 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7895 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7896 {
7897 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7898 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7899 Log4(("Clearing inhibition due to STI.\n"));
7900 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7901 }
7902 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7903 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
7904
7905 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7906 it, if we are returning to ring-3 before executing guest code. */
7907 pVCpu->hm.s.Event.fPending = false;
7908
7909 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
7910 if (fStepping)
7911 rcStrict = VINF_EM_DBG_STEPPED;
7912 }
7913 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7914 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7915 return rcStrict;
7916 }
7917
7918 /*
7919 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7920 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7921 */
7922 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7923 }
7924
7925 /* Validate. */
7926 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7927 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7928 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7929
7930 /* Inject. */
7931 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7932 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7933 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7934 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7935
7936 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7937 && uVector == X86_XCPT_PF)
7938 pMixedCtx->cr2 = GCPtrFaultAddress;
7939
7940 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7941 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7942
7943 AssertRCReturn(rc, rc);
7944 return VINF_SUCCESS;
7945}
7946
7947
7948/**
7949 * Clears the interrupt-window exiting control in the VMCS and if necessary
7950 * clears the current event in the VMCS as well.
7951 *
7952 * @returns VBox status code.
7953 * @param pVCpu The cross context virtual CPU structure.
7954 *
7955 * @remarks Use this function only to clear events that have not yet been
7956 * delivered to the guest but are injected in the VMCS!
7957 * @remarks No-long-jump zone!!!
7958 */
7959static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
7960{
7961 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7962
7963 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7964 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7965
7966 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
7967 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
7968}
7969
7970
7971/**
7972 * Enters the VT-x session.
7973 *
7974 * @returns VBox status code.
7975 * @param pVM The cross context VM structure.
7976 * @param pVCpu The cross context virtual CPU structure.
7977 * @param pCpu Pointer to the CPU info struct.
7978 */
7979VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7980{
7981 AssertPtr(pVM);
7982 AssertPtr(pVCpu);
7983 Assert(pVM->hm.s.vmx.fSupported);
7984 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7985 NOREF(pCpu); NOREF(pVM);
7986
7987 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7988 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7989
7990#ifdef VBOX_STRICT
7991 /* Make sure we're in VMX root mode. */
7992 RTCCUINTREG u32HostCR4 = ASMGetCR4();
7993 if (!(u32HostCR4 & X86_CR4_VMXE))
7994 {
7995 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7996 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7997 }
7998#endif
7999
8000 /*
8001 * Load the VCPU's VMCS as the current (and active) one.
8002 */
8003 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8004 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8005 if (RT_FAILURE(rc))
8006 return rc;
8007
8008 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8009 pVCpu->hm.s.fLeaveDone = false;
8010 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8011
8012 return VINF_SUCCESS;
8013}
8014
8015
8016/**
8017 * The thread-context callback (only on platforms which support it).
8018 *
8019 * @param enmEvent The thread-context event.
8020 * @param pVCpu The cross context virtual CPU structure.
8021 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8022 * @thread EMT(pVCpu)
8023 */
8024VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8025{
8026 NOREF(fGlobalInit);
8027
8028 switch (enmEvent)
8029 {
8030 case RTTHREADCTXEVENT_OUT:
8031 {
8032 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8033 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8034 VMCPU_ASSERT_EMT(pVCpu);
8035
8036 PVM pVM = pVCpu->CTX_SUFF(pVM);
8037 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8038
8039 /* No longjmps (logger flushes, locks) in this fragile context. */
8040 VMMRZCallRing3Disable(pVCpu);
8041 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8042
8043 /*
8044 * Restore host-state (FPU, debug etc.)
8045 */
8046 if (!pVCpu->hm.s.fLeaveDone)
8047 {
8048 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8049 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8050 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
8051 pVCpu->hm.s.fLeaveDone = true;
8052 }
8053
8054 /* Leave HM context, takes care of local init (term). */
8055 int rc = HMR0LeaveCpu(pVCpu);
8056 AssertRC(rc); NOREF(rc);
8057
8058 /* Restore longjmp state. */
8059 VMMRZCallRing3Enable(pVCpu);
8060 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8061 break;
8062 }
8063
8064 case RTTHREADCTXEVENT_IN:
8065 {
8066 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8067 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8068 VMCPU_ASSERT_EMT(pVCpu);
8069
8070 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8071 VMMRZCallRing3Disable(pVCpu);
8072 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8073
8074 /* Initialize the bare minimum state required for HM. This takes care of
8075 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8076 int rc = HMR0EnterCpu(pVCpu);
8077 AssertRC(rc);
8078 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8079
8080 /* Load the active VMCS as the current one. */
8081 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8082 {
8083 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8084 AssertRC(rc); NOREF(rc);
8085 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8086 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8087 }
8088 pVCpu->hm.s.fLeaveDone = false;
8089
8090 /* Restore longjmp state. */
8091 VMMRZCallRing3Enable(pVCpu);
8092 break;
8093 }
8094
8095 default:
8096 break;
8097 }
8098}
8099
8100
8101/**
8102 * Saves the host state in the VMCS host-state.
8103 * Sets up the VM-exit MSR-load area.
8104 *
8105 * The CPU state will be loaded from these fields on every successful VM-exit.
8106 *
8107 * @returns VBox status code.
8108 * @param pVM The cross context VM structure.
8109 * @param pVCpu The cross context virtual CPU structure.
8110 *
8111 * @remarks No-long-jump zone!!!
8112 */
8113static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8114{
8115 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8116
8117 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8118 return VINF_SUCCESS;
8119
8120 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8121 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8122
8123 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8124 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8125
8126 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8127 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8128
8129 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8130 return rc;
8131}
8132
8133
8134/**
8135 * Saves the host state in the VMCS host-state.
8136 *
8137 * @returns VBox status code.
8138 * @param pVM The cross context VM structure.
8139 * @param pVCpu The cross context virtual CPU structure.
8140 *
8141 * @remarks No-long-jump zone!!!
8142 */
8143VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8144{
8145 AssertPtr(pVM);
8146 AssertPtr(pVCpu);
8147
8148 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8149
8150 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8151 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8152 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8153 return hmR0VmxSaveHostState(pVM, pVCpu);
8154}
8155
8156
8157/**
8158 * Loads the guest state into the VMCS guest-state area.
8159 *
8160 * The will typically be done before VM-entry when the guest-CPU state and the
8161 * VMCS state may potentially be out of sync.
8162 *
8163 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8164 * VM-entry controls.
8165 * Sets up the appropriate VMX non-root function to execute guest code based on
8166 * the guest CPU mode.
8167 *
8168 * @returns VBox status code.
8169 * @param pVM The cross context VM structure.
8170 * @param pVCpu The cross context virtual CPU structure.
8171 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8172 * out-of-sync. Make sure to update the required fields
8173 * before using them.
8174 *
8175 * @remarks No-long-jump zone!!!
8176 */
8177static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8178{
8179 AssertPtr(pVM);
8180 AssertPtr(pVCpu);
8181 AssertPtr(pMixedCtx);
8182 HMVMX_ASSERT_PREEMPT_SAFE();
8183
8184 VMMRZCallRing3Disable(pVCpu);
8185 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8186
8187 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8188
8189 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8190
8191 /* Determine real-on-v86 mode. */
8192 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8193 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8194 && CPUMIsGuestInRealModeEx(pMixedCtx))
8195 {
8196 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8197 }
8198
8199 /*
8200 * Load the guest-state into the VMCS.
8201 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8202 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8203 */
8204 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8205 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8206
8207 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8208 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8209 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8210
8211 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8212 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8213 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8214
8215 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8216 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8217
8218 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8219 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8220
8221 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8222 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8223 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8224
8225 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8226 determine we don't have to swap EFER after all. */
8227 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8228 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8229
8230 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8231 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8232
8233 rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8234 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8235
8236 /*
8237 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8238 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8239 */
8240 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8241 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8242
8243 /* Clear any unused and reserved bits. */
8244 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8245
8246 VMMRZCallRing3Enable(pVCpu);
8247
8248 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8249 return rc;
8250}
8251
8252
8253/**
8254 * Loads the state shared between the host and guest into the VMCS.
8255 *
8256 * @param pVM The cross context VM structure.
8257 * @param pVCpu The cross context virtual CPU structure.
8258 * @param pCtx Pointer to the guest-CPU context.
8259 *
8260 * @remarks No-long-jump zone!!!
8261 */
8262static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8263{
8264 NOREF(pVM);
8265
8266 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8267 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8268
8269 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8270 {
8271 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8272 AssertRC(rc);
8273 }
8274
8275 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8276 {
8277 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8278 AssertRC(rc);
8279
8280 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8281 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8282 {
8283 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8284 AssertRC(rc);
8285 }
8286 }
8287
8288 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8289 {
8290#if HC_ARCH_BITS == 64
8291 if (pVM->hm.s.fAllow64BitGuests)
8292 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8293#endif
8294 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8295 }
8296
8297 /* Loading CR0, debug state might have changed intercepts, update VMCS. */
8298 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8299 {
8300 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8301 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8302 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8303 AssertRC(rc);
8304 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8305 }
8306
8307 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8308 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8309}
8310
8311
8312/**
8313 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8314 *
8315 * @returns Strict VBox status code (i.e. informational status codes too).
8316 * @param pVM The cross context VM structure.
8317 * @param pVCpu The cross context virtual CPU structure.
8318 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8319 * out-of-sync. Make sure to update the required fields
8320 * before using them.
8321 */
8322static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8323{
8324 HMVMX_ASSERT_PREEMPT_SAFE();
8325
8326 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8327#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8328 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8329#endif
8330
8331 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8332 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8333 {
8334 rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8335 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8336 { /* likely */}
8337 else
8338 {
8339 AssertLogRelMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8340 VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8341 }
8342 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8343 }
8344 else if (HMCPU_CF_VALUE(pVCpu))
8345 {
8346 rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8347 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8348 { /* likely */}
8349 else
8350 {
8351 AssertLogRelMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n",
8352 VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8353 }
8354 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8355 }
8356
8357 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8358 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8359 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8360 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8361 return rcStrict;
8362}
8363
8364
8365/**
8366 * Does the preparations before executing guest code in VT-x.
8367 *
8368 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8369 * recompiler/IEM. We must be cautious what we do here regarding committing
8370 * guest-state information into the VMCS assuming we assuredly execute the
8371 * guest in VT-x mode.
8372 *
8373 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8374 * the common-state (TRPM/forceflags), we must undo those changes so that the
8375 * recompiler/IEM can (and should) use them when it resumes guest execution.
8376 * Otherwise such operations must be done when we can no longer exit to ring-3.
8377 *
8378 * @returns Strict VBox status code (i.e. informational status codes too).
8379 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8380 * have been disabled.
8381 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8382 * double-fault into the guest.
8383 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8384 * dispatched directly.
8385 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8386 *
8387 * @param pVM The cross context VM structure.
8388 * @param pVCpu The cross context virtual CPU structure.
8389 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8390 * out-of-sync. Make sure to update the required fields
8391 * before using them.
8392 * @param pVmxTransient Pointer to the VMX transient structure.
8393 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8394 * us ignore some of the reasons for returning to
8395 * ring-3, and return VINF_EM_DBG_STEPPED if event
8396 * dispatching took place.
8397 */
8398static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8399{
8400 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8401
8402#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8403 PGMRZDynMapFlushAutoSet(pVCpu);
8404#endif
8405
8406 /* Check force flag actions that might require us to go back to ring-3. */
8407 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8408 if (rcStrict == VINF_SUCCESS)
8409 { /* FFs doesn't get set all the time. */ }
8410 else
8411 return rcStrict;
8412
8413#ifndef IEM_VERIFICATION_MODE_FULL
8414 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8415 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8416 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8417 {
8418 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8419 RTGCPHYS GCPhysApicBase;
8420 GCPhysApicBase = pMixedCtx->msrApicBase;
8421 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8422
8423 /* Unalias any existing mapping. */
8424 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8425 AssertRCReturn(rc, rc);
8426
8427 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8428 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
8429 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8430 AssertRCReturn(rc, rc);
8431
8432 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8433 }
8434#endif /* !IEM_VERIFICATION_MODE_FULL */
8435
8436 if (TRPMHasTrap(pVCpu))
8437 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8438 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8439
8440 /*
8441 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8442 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8443 */
8444 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, fStepping);
8445 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8446 { /* likely */ }
8447 else
8448 {
8449 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8450 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8451 return rcStrict;
8452 }
8453
8454 /*
8455 * Load the guest state bits, we can handle longjmps/getting preempted here.
8456 *
8457 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8458 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8459 * Hence, this needs to be done -after- injection of events.
8460 */
8461 rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8462 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8463 { /* likely */ }
8464 else
8465 return rcStrict;
8466
8467 /*
8468 * No longjmps to ring-3 from this point on!!!
8469 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8470 * This also disables flushing of the R0-logger instance (if any).
8471 */
8472 VMMRZCallRing3Disable(pVCpu);
8473
8474 /*
8475 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8476 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8477 *
8478 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8479 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8480 *
8481 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8482 * executing guest code.
8483 */
8484 pVmxTransient->fEFlags = ASMIntDisableFlags();
8485
8486 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8487 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8488 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8489 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8490 {
8491 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8492 {
8493 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8494 pVCpu->hm.s.Event.fPending = false;
8495
8496 return VINF_SUCCESS;
8497 }
8498
8499 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8500 rcStrict = VINF_EM_RAW_INTERRUPT;
8501 }
8502 else
8503 {
8504 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8505 rcStrict = VINF_EM_RAW_TO_R3;
8506 }
8507
8508 ASMSetFlags(pVmxTransient->fEFlags);
8509 VMMRZCallRing3Enable(pVCpu);
8510
8511 return rcStrict;
8512}
8513
8514
8515/**
8516 * Prepares to run guest code in VT-x and we've committed to doing so. This
8517 * means there is no backing out to ring-3 or anywhere else at this
8518 * point.
8519 *
8520 * @param pVM The cross context VM structure.
8521 * @param pVCpu The cross context virtual CPU structure.
8522 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8523 * out-of-sync. Make sure to update the required fields
8524 * before using them.
8525 * @param pVmxTransient Pointer to the VMX transient structure.
8526 *
8527 * @remarks Called with preemption disabled.
8528 * @remarks No-long-jump zone!!!
8529 */
8530static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8531{
8532 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8533 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8534 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8535
8536 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8537 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8538
8539#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8540 if (!CPUMIsGuestFPUStateActive(pVCpu))
8541 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8542 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8543#endif
8544
8545 if ( pVCpu->hm.s.fPreloadGuestFpu
8546 && !CPUMIsGuestFPUStateActive(pVCpu))
8547 {
8548 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8549 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8550 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8551 }
8552
8553 /*
8554 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8555 */
8556 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8557 && pVCpu->hm.s.vmx.cMsrs > 0)
8558 {
8559 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8560 }
8561
8562 /*
8563 * Load the host state bits as we may've been preempted (only happens when
8564 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8565 */
8566 /** @todo Why should hmR0VmxSetupVMRunHandler() changing pfnStartVM have
8567 * any effect to the host state needing to be saved? */
8568 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8569 {
8570 /* This ASSUMES that pfnStartVM has been set up already. */
8571 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8572 AssertRC(rc);
8573 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
8574 }
8575 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8576
8577 /*
8578 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8579 */
8580 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8581 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8582 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8583
8584 /* Store status of the shared guest-host state at the time of VM-entry. */
8585#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8586 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8587 {
8588 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8589 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8590 }
8591 else
8592#endif
8593 {
8594 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8595 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8596 }
8597 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8598
8599 /*
8600 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8601 */
8602 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8603 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8604
8605 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8606 RTCPUID idCurrentCpu = pCpu->idCpu;
8607 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8608 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8609 {
8610 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8611 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8612 }
8613
8614 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8615 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8616 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8617 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8618
8619 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8620
8621 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8622 to start executing. */
8623
8624 /*
8625 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8626 */
8627 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8628 {
8629 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8630 {
8631 bool fMsrUpdated;
8632 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8633 AssertRC(rc2);
8634 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8635
8636 rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8637 &fMsrUpdated);
8638 AssertRC(rc2);
8639 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8640
8641 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8642 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8643 }
8644 else
8645 {
8646 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8647 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8648 }
8649 }
8650
8651#ifdef VBOX_STRICT
8652 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8653 hmR0VmxCheckHostEferMsr(pVCpu);
8654 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8655#endif
8656#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8657 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8658 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8659 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8660#endif
8661}
8662
8663
8664/**
8665 * Performs some essential restoration of state after running guest code in
8666 * VT-x.
8667 *
8668 * @param pVM The cross context VM structure.
8669 * @param pVCpu The cross context virtual CPU structure.
8670 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8671 * out-of-sync. Make sure to update the required fields
8672 * before using them.
8673 * @param pVmxTransient Pointer to the VMX transient structure.
8674 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8675 *
8676 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8677 *
8678 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8679 * unconditionally when it is safe to do so.
8680 */
8681static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8682{
8683 NOREF(pVM);
8684
8685 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8686
8687 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8688 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8689 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8690 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8691 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8692 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8693
8694 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8695 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
8696
8697 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8698 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8699 Assert(!ASMIntAreEnabled());
8700 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8701
8702#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8703 if (CPUMIsGuestFPUStateActive(pVCpu))
8704 {
8705 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8706 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8707 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8708 }
8709#endif
8710
8711#if HC_ARCH_BITS == 64
8712 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8713#endif
8714 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8715#ifdef VBOX_STRICT
8716 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8717#endif
8718 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8719 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8720
8721 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8722 uint32_t uExitReason;
8723 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8724 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8725 AssertRC(rc);
8726 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8727 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8728
8729 /* Update the VM-exit history array. */
8730 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8731
8732 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8733 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8734 {
8735 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8736 pVmxTransient->fVMEntryFailed));
8737 return;
8738 }
8739
8740 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8741 {
8742 /** @todo We can optimize this by only syncing with our force-flags when
8743 * really needed and keeping the VMCS state as it is for most
8744 * VM-exits. */
8745 /* Update the guest interruptibility-state from the VMCS. */
8746 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8747
8748#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8749 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8750 AssertRC(rc);
8751#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8752 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8753 AssertRC(rc);
8754#endif
8755
8756 /*
8757 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8758 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8759 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8760 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8761 */
8762 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8763 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8764 {
8765 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8766 AssertRC(rc);
8767 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8768 }
8769 }
8770}
8771
8772
8773/**
8774 * Runs the guest code using VT-x the normal way.
8775 *
8776 * @returns VBox status code.
8777 * @param pVM The cross context VM structure.
8778 * @param pVCpu The cross context virtual CPU structure.
8779 * @param pCtx Pointer to the guest-CPU context.
8780 *
8781 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8782 */
8783static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8784{
8785 VMXTRANSIENT VmxTransient;
8786 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8787 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8788 uint32_t cLoops = 0;
8789
8790 for (;; cLoops++)
8791 {
8792 Assert(!HMR0SuspendPending());
8793 HMVMX_ASSERT_CPU_SAFE();
8794
8795 /* Preparatory work for running guest code, this may force us to return
8796 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8797 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8798 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8799 if (rcStrict != VINF_SUCCESS)
8800 break;
8801
8802 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8803 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8804 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8805
8806 /* Restore any residual host-state and save any bits shared between host
8807 and guest into the guest-CPU state. Re-enables interrupts! */
8808 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
8809
8810 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8811 if (RT_SUCCESS(rcRun))
8812 { /* very likely */ }
8813 else
8814 {
8815 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8816 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
8817 return rcRun;
8818 }
8819
8820 /* Profile the VM-exit. */
8821 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8822 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8823 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8824 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8825 HMVMX_START_EXIT_DISPATCH_PROF();
8826
8827 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8828
8829 /* Handle the VM-exit. */
8830#ifdef HMVMX_USE_FUNCTION_TABLE
8831 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8832#else
8833 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8834#endif
8835 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8836 if (rcStrict == VINF_SUCCESS)
8837 {
8838 if (cLoops <= pVM->hm.s.cMaxResumeLoops)
8839 continue; /* likely */
8840 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8841 rcStrict = VINF_EM_RAW_INTERRUPT;
8842 }
8843 break;
8844 }
8845
8846 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8847 return rcStrict;
8848}
8849
8850
8851
8852/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8853 * probes.
8854 *
8855 * The following few functions and associated structure contains the bloat
8856 * necessary for providing detailed debug events and dtrace probes as well as
8857 * reliable host side single stepping. This works on the principle of
8858 * "subclassing" the normal execution loop and workers. We replace the loop
8859 * method completely and override selected helpers to add necessary adjustments
8860 * to their core operation.
8861 *
8862 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8863 * any performance for debug and analysis features.
8864 *
8865 * @{
8866 */
8867
8868typedef struct VMXRUNDBGSTATE
8869{
8870 /** The RIP we started executing at. This is for detecting that we stepped. */
8871 uint64_t uRipStart;
8872 /** The CS we started executing with. */
8873 uint16_t uCsStart;
8874
8875 /** Whether we've actually modified the 1st execution control field. */
8876 bool fModifiedProcCtls : 1;
8877 /** Whether we've actually modified the 2nd execution control field. */
8878 bool fModifiedProcCtls2 : 1;
8879 /** Whether we've actually modified the exception bitmap. */
8880 bool fModifiedXcptBitmap : 1;
8881
8882 /** We desire the modified the CR0 mask to be cleared. */
8883 bool fClearCr0Mask : 1;
8884 /** We desire the modified the CR4 mask to be cleared. */
8885 bool fClearCr4Mask : 1;
8886 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
8887 uint32_t fCpe1Extra;
8888 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
8889 uint32_t fCpe1Unwanted;
8890 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
8891 uint32_t fCpe2Extra;
8892 /** Extra stuff we need in */
8893 uint32_t bmXcptExtra;
8894 /** The sequence number of the Dtrace provider settings the state was
8895 * configured against. */
8896 uint32_t uDtraceSettingsSeqNo;
8897 /** Exits to check (one bit per exit). */
8898 uint32_t bmExitsToCheck[3];
8899
8900 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
8901 uint32_t fProcCtlsInitial;
8902 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
8903 uint32_t fProcCtls2Initial;
8904 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
8905 uint32_t bmXcptInitial;
8906
8907} VMXRUNDBGSTATE;
8908AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
8909typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
8910
8911
8912/**
8913 * Initializes the VMXRUNDBGSTATE structure.
8914 *
8915 * @param pVCpu The cross context virtual CPU structure of the
8916 * calling EMT.
8917 * @param pCtx The CPU register context to go with @a pVCpu.
8918 * @param pDbgState The structure to initialize.
8919 */
8920DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
8921{
8922 pDbgState->uRipStart = pCtx->rip;
8923 pDbgState->uCsStart = pCtx->cs.Sel;
8924
8925 pDbgState->fModifiedProcCtls = false;
8926 pDbgState->fModifiedProcCtls2 = false;
8927 pDbgState->fModifiedXcptBitmap = false;
8928 pDbgState->fClearCr0Mask = false;
8929 pDbgState->fClearCr4Mask = false;
8930 pDbgState->fCpe1Extra = 0;
8931 pDbgState->fCpe1Unwanted = 0;
8932 pDbgState->fCpe2Extra = 0;
8933 pDbgState->bmXcptExtra = 0;
8934 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
8935 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
8936 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
8937}
8938
8939
8940/**
8941 * Updates the VMSC fields with changes requested by @a pDbgState.
8942 *
8943 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
8944 * immediately before executing guest code, i.e. when interrupts are disabled.
8945 * We don't check status codes here as we cannot easily assert or return in the
8946 * latter case.
8947 *
8948 * @param pVCpu The cross context virtual CPU structure.
8949 * @param pDbgState The debug state.
8950 */
8951DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
8952{
8953 /*
8954 * Ensure desired flags in VMCS control fields are set.
8955 * (Ignoring write failure here, as we're committed and it's just debug extras.)
8956 *
8957 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
8958 * there should be no stale data in pCtx at this point.
8959 */
8960 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
8961 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
8962 {
8963 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
8964 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
8965 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8966 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
8967 pDbgState->fModifiedProcCtls = true;
8968 }
8969
8970 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
8971 {
8972 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
8973 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
8974 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
8975 pDbgState->fModifiedProcCtls2 = true;
8976 }
8977
8978 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
8979 {
8980 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
8981 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8982 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
8983 pDbgState->fModifiedXcptBitmap = true;
8984 }
8985
8986 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
8987 {
8988 pVCpu->hm.s.vmx.u32CR0Mask = 0;
8989 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
8990 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
8991 }
8992
8993 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
8994 {
8995 pVCpu->hm.s.vmx.u32CR4Mask = 0;
8996 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
8997 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
8998 }
8999}
9000
9001
9002DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9003{
9004 /*
9005 * Restore exit control settings as we may not reenter this function the
9006 * next time around.
9007 */
9008 /* We reload the initial value, trigger what we can of recalculations the
9009 next time around. From the looks of things, that's all that's required atm. */
9010 if (pDbgState->fModifiedProcCtls)
9011 {
9012 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9013 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9014 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9015 AssertRCReturn(rc2, rc2);
9016 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9017 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0 | HM_CHANGED_GUEST_DEBUG);
9018 }
9019
9020 /* We're currently the only ones messing with this one, so just restore the
9021 cached value and reload the field. */
9022 if ( pDbgState->fModifiedProcCtls2
9023 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9024 {
9025 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9026 AssertRCReturn(rc2, rc2);
9027 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9028 }
9029
9030 /* If we've modified the exception bitmap, we restore it and trigger
9031 reloading and partial recalculation the next time around. */
9032 if (pDbgState->fModifiedXcptBitmap)
9033 {
9034 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9035 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS | HM_CHANGED_GUEST_CR0);
9036 }
9037
9038 /* We assume hmR0VmxLoadSharedCR0 will recalculate and load the CR0 mask. */
9039 if (pDbgState->fClearCr0Mask)
9040 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9041
9042 /* We assume hmR0VmxLoadGuestCR3AndCR4 will recalculate and load the CR4 mask. */
9043 if (pDbgState->fClearCr4Mask)
9044 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9045
9046 return rcStrict;
9047}
9048
9049
9050/**
9051 * Configures VM-exit controls for current DBGF and DTrace settings.
9052 *
9053 * This updates @a pDbgState and the VMCS execution control fields to reflect
9054 * the necessary exits demanded by DBGF and DTrace.
9055 *
9056 * @param pVM The cross context VM structure.
9057 * @param pVCpu The cross context virtual CPU structure.
9058 * @param pCtx Pointer to the guest-CPU context.
9059 * @param pDbgState The debug state.
9060 * @param pVmxTransient Pointer to the VMX transient structure. May update
9061 * fUpdateTscOffsettingAndPreemptTimer.
9062 */
9063static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9064 PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9065{
9066 /*
9067 * Take down the dtrace serial number so we can spot changes.
9068 */
9069 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9070 ASMCompilerBarrier();
9071
9072 /*
9073 * We'll rebuild most of the middle block of data members (holding the
9074 * current settings) as we go along here, so start by clearing it all.
9075 */
9076 pDbgState->bmXcptExtra = 0;
9077 pDbgState->fCpe1Extra = 0;
9078 pDbgState->fCpe1Unwanted = 0;
9079 pDbgState->fCpe2Extra = 0;
9080 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9081 pDbgState->bmExitsToCheck[i] = 0;
9082
9083 /*
9084 * Software interrupts (INT XXh) - no idea how to trigger these...
9085 */
9086 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9087 || VBOXVMM_INT_SOFTWARE_ENABLED())
9088 {
9089 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9090 }
9091
9092 /*
9093 * Exception bitmap and XCPT events+probes.
9094 */
9095 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9096 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9097 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9098
9099 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9100 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9101 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9102 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9103 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9104 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9105 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9106 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9107 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9108 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9109 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9110 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9111 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9112 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9113 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9114 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9115 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9116 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9117
9118 if (pDbgState->bmXcptExtra)
9119 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9120
9121 /*
9122 * Process events and probes for VM exits, making sure we get the wanted exits.
9123 *
9124 * Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9125 * So, when adding/changing/removing please don't forget to update it.
9126 *
9127 * Some of the macros are picking up local variables to save horizontal space,
9128 * (being able to see it in a table is the lesser evil here).
9129 */
9130#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9131 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9132 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9133#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9134 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9135 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9136 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9137 } else do { } while (0)
9138#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9139 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9140 { \
9141 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9142 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9143 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9144 } else do { } while (0)
9145#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9146 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9147 { \
9148 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9149 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9150 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9151 } else do { } while (0)
9152#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9153 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9154 { \
9155 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9156 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9157 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9158 } else do { } while (0)
9159
9160 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9161 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9162 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9163 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9164 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9165
9166 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9167 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9168 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9169 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9170 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9171 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9172 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9173 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9174 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9175 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9176 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9177 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9178 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9179 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9180 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9181 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9182 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9183 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9184 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9185 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9186 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9187 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9188 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9189 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9190 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9191 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9192 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9193 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9194 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9195 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9196 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9197 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9198 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9199 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9200 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9201 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9202
9203 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9204 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9205 {
9206 int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9207 rc2 |= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9208 rc2 |= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9209 AssertRC(rc2);
9210
9211#if 0 /** @todo fix me */
9212 pDbgState->fClearCr0Mask = true;
9213 pDbgState->fClearCr4Mask = true;
9214#endif
9215 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9216 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9217 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9218 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9219 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9220 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9221 require clearing here and in the loop if we start using it. */
9222 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9223 }
9224 else
9225 {
9226 if (pDbgState->fClearCr0Mask)
9227 {
9228 pDbgState->fClearCr0Mask = false;
9229 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9230 }
9231 if (pDbgState->fClearCr4Mask)
9232 {
9233 pDbgState->fClearCr4Mask = false;
9234 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9235 }
9236 }
9237 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9238 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9239
9240 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9241 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9242 {
9243 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9244 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9245 }
9246 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9247 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9248
9249 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9250 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9251 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9252 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9253 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9254 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9255 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9256 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9257#if 0 /** @todo too slow, fix handler. */
9258 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9259#endif
9260 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9261
9262 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9263 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9264 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9265 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9266 {
9267 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9268 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9269 }
9270 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9271 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9272 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9273 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9274
9275 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9276 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9277 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9278 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9279 {
9280 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9281 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9282 }
9283 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9284 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9285 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9286 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9287
9288 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9289 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9290 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9291 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9292 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9293 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9294 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9295 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9296 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9297 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9298 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9299 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9300 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9301 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9302 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9303 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9304 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9305 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9306 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9307 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9308 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9309 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9310
9311#undef IS_EITHER_ENABLED
9312#undef SET_ONLY_XBM_IF_EITHER_EN
9313#undef SET_CPE1_XBM_IF_EITHER_EN
9314#undef SET_CPEU_XBM_IF_EITHER_EN
9315#undef SET_CPE2_XBM_IF_EITHER_EN
9316
9317 /*
9318 * Sanitize the control stuff.
9319 */
9320 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9321 if (pDbgState->fCpe2Extra)
9322 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9323 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9324 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9325 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9326 {
9327 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9328 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9329 }
9330
9331 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9332 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9333 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9334 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9335}
9336
9337
9338/**
9339 * Fires off DBGF events and dtrace probes for an exit, when it's appropriate.
9340 *
9341 * The caller has checked exit against the VMXRUNDBGSTATE::bmExitsToCheck
9342 * bitmap. The caller has checked for NMIs already, so we don't have to do that
9343 * either.
9344 *
9345 * @returns Strict VBox status code (i.e. informational status codes too).
9346 * @param pVM The cross context VM structure.
9347 * @param pVCpu The cross context virtual CPU structure.
9348 * @param pMixedCtx Pointer to the guest-CPU context.
9349 * @param pVmxTransient Pointer to the VMX-transient structure.
9350 * @param uExitReason The VM-exit reason.
9351 *
9352 * @remarks The name of this function is displayed by dtrace, so keep it short
9353 * and to the point. No longer than 33 chars long, please.
9354 */
9355static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9356 PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9357{
9358 /*
9359 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9360 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9361 *
9362 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9363 * does. Must add/change/remove both places. Same ordering, please.
9364 *
9365 * Added/removed events must also be reflected in the next section
9366 * where we dispatch dtrace events.
9367 */
9368 bool fDtrace1 = false;
9369 bool fDtrace2 = false;
9370 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9371 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9372 uint32_t uEventArg = 0;
9373#define SET_EXIT(a_EventSubName) \
9374 do { \
9375 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9376 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9377 } while (0)
9378#define SET_BOTH(a_EventSubName) \
9379 do { \
9380 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9381 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9382 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9383 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9384 } while (0)
9385 switch (uExitReason)
9386 {
9387 case VMX_EXIT_MTF:
9388 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9389
9390 case VMX_EXIT_XCPT_OR_NMI:
9391 {
9392 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9393 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9394 {
9395 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9396 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9397 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9398 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9399 {
9400 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9401 {
9402 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9403 uEventArg = pVmxTransient->uExitIntErrorCode;
9404 }
9405 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9406 switch (enmEvent1)
9407 {
9408 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9409 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9410 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9411 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9412 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9413 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9414 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9415 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9416 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9417 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9418 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9419 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9420 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9421 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9422 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9423 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9424 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9425 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9426 default: break;
9427 }
9428 }
9429 else
9430 AssertFailed();
9431 break;
9432
9433 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9434 uEventArg = idxVector;
9435 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9436 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9437 break;
9438 }
9439 break;
9440 }
9441
9442 case VMX_EXIT_TRIPLE_FAULT:
9443 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9444 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9445 break;
9446 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9447 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9448 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9449 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9450 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9451
9452 /* Instruction specific VM-exits: */
9453 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9454 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9455 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9456 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9457 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9458 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9459 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9460 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9461 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9462 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9463 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9464 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9465 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9466 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9467 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9468 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9469 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9470 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9471 case VMX_EXIT_MOV_CRX:
9472 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9473/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
9474* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
9475 if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
9476 == VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
9477 SET_BOTH(CRX_READ);
9478 else
9479 SET_BOTH(CRX_WRITE);
9480 uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
9481 break;
9482 case VMX_EXIT_MOV_DRX:
9483 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9484 if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
9485 == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
9486 SET_BOTH(DRX_READ);
9487 else
9488 SET_BOTH(DRX_WRITE);
9489 uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
9490 break;
9491 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9492 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9493 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9494 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9495 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9496 case VMX_EXIT_XDTR_ACCESS:
9497 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9498 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9499 {
9500 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9501 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9502 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9503 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9504 }
9505 break;
9506
9507 case VMX_EXIT_TR_ACCESS:
9508 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9509 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9510 {
9511 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9512 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9513 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9514 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9515 }
9516 break;
9517
9518 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9519 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9520 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9521 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9522 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9523 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9524 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9525 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9526 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9527 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9528 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9529
9530 /* Events that aren't relevant at this point. */
9531 case VMX_EXIT_EXT_INT:
9532 case VMX_EXIT_INT_WINDOW:
9533 case VMX_EXIT_NMI_WINDOW:
9534 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9535 case VMX_EXIT_PREEMPT_TIMER:
9536 case VMX_EXIT_IO_INSTR:
9537 break;
9538
9539 /* Errors and unexpected events. */
9540 case VMX_EXIT_INIT_SIGNAL:
9541 case VMX_EXIT_SIPI:
9542 case VMX_EXIT_IO_SMI:
9543 case VMX_EXIT_SMI:
9544 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9545 case VMX_EXIT_ERR_MSR_LOAD:
9546 case VMX_EXIT_ERR_MACHINE_CHECK:
9547 break;
9548
9549 default:
9550 AssertMsgFailed(("Unexpected exit=%#x\n", uExitReason));
9551 break;
9552 }
9553#undef SET_BOTH
9554#undef SET_EXIT
9555
9556 /*
9557 * Dtrace tracepoints go first. We do them here at once so we don't
9558 * have to copy the guest state saving and stuff a few dozen times.
9559 * Down side is that we've got to repeat the switch, though this time
9560 * we use enmEvent since the probes are a subset of what DBGF does.
9561 */
9562 if (fDtrace1 || fDtrace2)
9563 {
9564 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9565 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9566 switch (enmEvent1)
9567 {
9568 /** @todo consider which extra parameters would be helpful for each probe. */
9569 case DBGFEVENT_END: break;
9570 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9571 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9572 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9573 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9574 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9575 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9576 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9577 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9578 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9579 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9580 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9581 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9582 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9583 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9584 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9585 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9586 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9587 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9588 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9589 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9590 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9591 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9592 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9593 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9594 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9595 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9596 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9597 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9598 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9599 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9600 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9601 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9602 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9603 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9604 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9605 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9606 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9607 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9608 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9609 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9610 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9611 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9612 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9613 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9614 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9615 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9616 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9617 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9618 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9619 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9620 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9621 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9622 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9623 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9624 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9625 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9626 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9627 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9628 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9629 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9630 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9631 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9632 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9633 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9634 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9635 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9636 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9637 }
9638 switch (enmEvent2)
9639 {
9640 /** @todo consider which extra parameters would be helpful for each probe. */
9641 case DBGFEVENT_END: break;
9642 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9643 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9644 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9645 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9646 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9647 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9648 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9649 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9650 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9651 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9652 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9653 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9654 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9655 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9656 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9657 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9658 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9659 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9660 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9661 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9662 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9663 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9664 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9665 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9666 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9667 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9668 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9669 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9670 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9671 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9672 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9673 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9674 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9675 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9676 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9677 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9678 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9679 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9680 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9681 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9682 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9683 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9684 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9685 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9686 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9687 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9688 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9689 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9690 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9691 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9692 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9693 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9694 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9695 }
9696 }
9697
9698 /*
9699 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9700 * the DBGF call will do a full check).
9701 *
9702 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9703 * Note! If we have to events, we prioritize the first, i.e. the instruction
9704 * one, in order to avoid event nesting.
9705 */
9706 if ( enmEvent1 != DBGFEVENT_END
9707 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9708 {
9709 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9710 if (rcStrict != VINF_SUCCESS)
9711 return rcStrict;
9712 }
9713 else if ( enmEvent2 != DBGFEVENT_END
9714 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9715 {
9716 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9717 if (rcStrict != VINF_SUCCESS)
9718 return rcStrict;
9719 }
9720
9721 return VINF_SUCCESS;
9722}
9723
9724
9725/**
9726 * Single-stepping VM-exit filtering.
9727 *
9728 * This is preprocessing the exits and deciding whether we've gotten far enough
9729 * to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit handling is
9730 * performed.
9731 *
9732 * @returns Strict VBox status code (i.e. informational status codes too).
9733 * @param pVM The cross context VM structure.
9734 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9735 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9736 * out-of-sync. Make sure to update the required
9737 * fields before using them.
9738 * @param pVmxTransient Pointer to the VMX-transient structure.
9739 * @param uExitReason The VM-exit reason.
9740 * @param pDbgState The debug state.
9741 */
9742DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9743 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9744{
9745 /*
9746 * Expensive (saves context) generic dtrace exit probe.
9747 */
9748 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9749 { /* more likely */ }
9750 else
9751 {
9752 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9753 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9754 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9755 }
9756
9757 /*
9758 * Check for host NMI, just to get that out of the way.
9759 */
9760 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9761 { /* normally likely */ }
9762 else
9763 {
9764 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9765 AssertRCReturn(rc2, rc2);
9766 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9767 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9768 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9769 }
9770
9771 /*
9772 * Check for single stepping event if we're stepping.
9773 */
9774 if (pVCpu->hm.s.fSingleInstruction)
9775 {
9776 switch (uExitReason)
9777 {
9778 case VMX_EXIT_MTF:
9779 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9780
9781 /* Various events: */
9782 case VMX_EXIT_XCPT_OR_NMI:
9783 case VMX_EXIT_EXT_INT:
9784 case VMX_EXIT_TRIPLE_FAULT:
9785 case VMX_EXIT_INT_WINDOW:
9786 case VMX_EXIT_NMI_WINDOW:
9787 case VMX_EXIT_TASK_SWITCH:
9788 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9789 case VMX_EXIT_APIC_ACCESS:
9790 case VMX_EXIT_EPT_VIOLATION:
9791 case VMX_EXIT_EPT_MISCONFIG:
9792 case VMX_EXIT_PREEMPT_TIMER:
9793
9794 /* Instruction specific VM-exits: */
9795 case VMX_EXIT_CPUID:
9796 case VMX_EXIT_GETSEC:
9797 case VMX_EXIT_HLT:
9798 case VMX_EXIT_INVD:
9799 case VMX_EXIT_INVLPG:
9800 case VMX_EXIT_RDPMC:
9801 case VMX_EXIT_RDTSC:
9802 case VMX_EXIT_RSM:
9803 case VMX_EXIT_VMCALL:
9804 case VMX_EXIT_VMCLEAR:
9805 case VMX_EXIT_VMLAUNCH:
9806 case VMX_EXIT_VMPTRLD:
9807 case VMX_EXIT_VMPTRST:
9808 case VMX_EXIT_VMREAD:
9809 case VMX_EXIT_VMRESUME:
9810 case VMX_EXIT_VMWRITE:
9811 case VMX_EXIT_VMXOFF:
9812 case VMX_EXIT_VMXON:
9813 case VMX_EXIT_MOV_CRX:
9814 case VMX_EXIT_MOV_DRX:
9815 case VMX_EXIT_IO_INSTR:
9816 case VMX_EXIT_RDMSR:
9817 case VMX_EXIT_WRMSR:
9818 case VMX_EXIT_MWAIT:
9819 case VMX_EXIT_MONITOR:
9820 case VMX_EXIT_PAUSE:
9821 case VMX_EXIT_XDTR_ACCESS:
9822 case VMX_EXIT_TR_ACCESS:
9823 case VMX_EXIT_INVEPT:
9824 case VMX_EXIT_RDTSCP:
9825 case VMX_EXIT_INVVPID:
9826 case VMX_EXIT_WBINVD:
9827 case VMX_EXIT_XSETBV:
9828 case VMX_EXIT_RDRAND:
9829 case VMX_EXIT_INVPCID:
9830 case VMX_EXIT_VMFUNC:
9831 case VMX_EXIT_RDSEED:
9832 case VMX_EXIT_XSAVES:
9833 case VMX_EXIT_XRSTORS:
9834 {
9835 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9836 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9837 AssertRCReturn(rc2, rc2);
9838 if ( pMixedCtx->rip != pDbgState->uRipStart
9839 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
9840 return VINF_EM_DBG_STEPPED;
9841 break;
9842 }
9843
9844 /* Errors and unexpected events: */
9845 case VMX_EXIT_INIT_SIGNAL:
9846 case VMX_EXIT_SIPI:
9847 case VMX_EXIT_IO_SMI:
9848 case VMX_EXIT_SMI:
9849 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9850 case VMX_EXIT_ERR_MSR_LOAD:
9851 case VMX_EXIT_ERR_MACHINE_CHECK:
9852 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9853 break;
9854
9855 default:
9856 AssertMsgFailed(("Unexpected exit=%#x\n", uExitReason));
9857 break;
9858 }
9859 }
9860
9861 /*
9862 * Check for debugger event breakpoints and dtrace probes.
9863 */
9864 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9865 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9866 {
9867 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9868 if (rcStrict != VINF_SUCCESS)
9869 return rcStrict;
9870 }
9871
9872 /*
9873 * Normal processing.
9874 */
9875#ifdef HMVMX_USE_FUNCTION_TABLE
9876 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9877#else
9878 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9879#endif
9880}
9881
9882
9883/**
9884 * Single steps guest code using VT-x.
9885 *
9886 * @returns Strict VBox status code (i.e. informational status codes too).
9887 * @param pVM The cross context VM structure.
9888 * @param pVCpu The cross context virtual CPU structure.
9889 * @param pCtx Pointer to the guest-CPU context.
9890 *
9891 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
9892 */
9893static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9894{
9895 VMXTRANSIENT VmxTransient;
9896 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9897
9898 /* Set HMCPU indicators. */
9899 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
9900 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
9901 pVCpu->hm.s.fDebugWantRdTscExit = false;
9902 pVCpu->hm.s.fUsingDebugLoop = true;
9903
9904 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
9905 VMXRUNDBGSTATE DbgState;
9906 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
9907 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
9908
9909 /*
9910 * The loop.
9911 */
9912 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9913 for (uint32_t cLoops = 0; ; cLoops++)
9914 {
9915 Assert(!HMR0SuspendPending());
9916 HMVMX_ASSERT_CPU_SAFE();
9917 bool fStepping = pVCpu->hm.s.fSingleInstruction;
9918
9919 /*
9920 * Preparatory work for running guest code, this may force us to return
9921 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
9922 */
9923 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9924 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
9925 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
9926 if (rcStrict != VINF_SUCCESS)
9927 break;
9928
9929 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9930 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
9931
9932 /*
9933 * Now we can run the guest code.
9934 */
9935 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9936
9937 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9938
9939 /*
9940 * Restore any residual host-state and save any bits shared between host
9941 * and guest into the guest-CPU state. Re-enables interrupts!
9942 */
9943 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
9944
9945 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9946 if (RT_SUCCESS(rcRun))
9947 { /* very likely */ }
9948 else
9949 {
9950 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9951 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9952 return rcRun;
9953 }
9954
9955 /* Profile the VM-exit. */
9956 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9957 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9958 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9959 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9960 HMVMX_START_EXIT_DISPATCH_PROF();
9961
9962 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9963
9964 /*
9965 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
9966 */
9967 rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
9968 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9969 if (rcStrict != VINF_SUCCESS)
9970 break;
9971 if (cLoops > pVM->hm.s.cMaxResumeLoops)
9972 {
9973 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9974 rcStrict = VINF_EM_RAW_INTERRUPT;
9975 break;
9976 }
9977
9978 /*
9979 * Stepping: Did the RIP change, if so, consider it a single step.
9980 * Otherwise, make sure one of the TFs gets set.
9981 */
9982 if (fStepping)
9983 {
9984 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
9985 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
9986 AssertRCReturn(rc2, rc2);
9987 if ( pCtx->rip != DbgState.uRipStart
9988 || pCtx->cs.Sel != DbgState.uCsStart)
9989 {
9990 rcStrict = VINF_EM_DBG_STEPPED;
9991 break;
9992 }
9993 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
9994 }
9995
9996 /*
9997 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
9998 */
9999 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10000 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10001 }
10002
10003 /*
10004 * Clear the X86_EFL_TF if necessary.
10005 */
10006 if (pVCpu->hm.s.fClearTrapFlag)
10007 {
10008 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10009 AssertRCReturn(rc2, rc2);
10010 pVCpu->hm.s.fClearTrapFlag = false;
10011 pCtx->eflags.Bits.u1TF = 0;
10012 }
10013 /** @todo there seems to be issues with the resume flag when the monitor trap
10014 * flag is pending without being used. Seen early in bios init when
10015 * accessing APIC page in protected mode. */
10016
10017 /*
10018 * Restore VM-exit control settings as we may not reenter this function the
10019 * next time around.
10020 */
10021 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10022
10023 /* Restore HMCPU indicators. */
10024 pVCpu->hm.s.fUsingDebugLoop = false;
10025 pVCpu->hm.s.fDebugWantRdTscExit = false;
10026 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10027
10028 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10029 return rcStrict;
10030}
10031
10032
10033/** @} */
10034
10035
10036/**
10037 * Checks if any expensive dtrace probes are enabled and we should go to the
10038 * debug loop.
10039 *
10040 * @returns true if we should use debug loop, false if not.
10041 */
10042static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10043{
10044 /* It's probably faster to OR the raw 32-bit counter variables together.
10045 Since the variables are in an array and the probes are next to one
10046 another (more or less), we have good locality. So, better read
10047 eight-nine cache lines ever time and only have one conditional, than
10048 128+ conditionals, right? */
10049 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10050 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10051 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10052 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10053 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10054 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10055 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10056 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10057 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10058 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10059 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10060 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10061 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10062 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10063 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10064 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10065 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10066 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10067 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10068 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10069 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10070 ) != 0
10071 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10072 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10073 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10074 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10075 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10076 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10077 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10078 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10079 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10080 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10081 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10082 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10083 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10084 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10085 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10086 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10087 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10088 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10089 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10090 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10091 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10092 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10093 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10094 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10095 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10096 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10097 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10098 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10099 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10100 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10101 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10102 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10103 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10104 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10105 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10106 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10107 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10108 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10109 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10110 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10111 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10112 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10113 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10114 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10115 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10116 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10117 ) != 0
10118 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10119 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10120 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10121 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10122 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10123 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10124 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10125 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10126 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10127 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10128 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10129 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10130 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10131 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10132 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10133 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10134 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10135 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10136 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10137 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10138 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10139 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10140 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10141 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10142 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10143 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10144 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10145 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10146 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10147 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10148 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10149 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10150 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10151 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10152 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10153 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10154 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10155 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10156 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10157 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10158 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10159 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10160 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10161 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10162 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10163 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10164 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10165 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10166 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10167 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10168 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10169 ) != 0;
10170}
10171
10172
10173/**
10174 * Runs the guest code using VT-x.
10175 *
10176 * @returns Strict VBox status code (i.e. informational status codes too).
10177 * @param pVM The cross context VM structure.
10178 * @param pVCpu The cross context virtual CPU structure.
10179 * @param pCtx Pointer to the guest-CPU context.
10180 */
10181VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10182{
10183 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10184 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10185 HMVMX_ASSERT_PREEMPT_SAFE();
10186
10187 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10188
10189 VBOXSTRICTRC rcStrict;
10190 if ( !pVCpu->hm.s.fUseDebugLoop
10191 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10192 && !DBGFIsStepping(pVCpu) )
10193 rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10194 else
10195 rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10196
10197 if (rcStrict == VERR_EM_INTERPRETER)
10198 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10199 else if (rcStrict == VINF_EM_RESET)
10200 rcStrict = VINF_EM_TRIPLE_FAULT;
10201
10202 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10203 if (RT_FAILURE(rc2))
10204 {
10205 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10206 rcStrict = rc2;
10207 }
10208 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10209 return rcStrict;
10210}
10211
10212
10213#ifndef HMVMX_USE_FUNCTION_TABLE
10214DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10215{
10216# ifdef DEBUG_ramshankar
10217# define RETURN_EXIT_CALL(a_CallExpr) \
10218 do { \
10219 /* int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); */ \
10220 VBOXSTRICTRC rcStrict = a_CallExpr; \
10221 /* HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); */ \
10222 return rcStrict; \
10223 } while (0)
10224# else
10225# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10226# endif
10227 switch (rcReason)
10228 {
10229 case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10230 case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10231 case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10232 case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10233 case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10234 case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10235 case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10236 case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10237 case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10238 case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10239 case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10240 case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10241 case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10242 case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10243 case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10244 case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10245 case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10246 case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10247 case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10248 case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10249 case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10250 case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10251 case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10252 case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10253 case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10254 case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10255 case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10256 case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10257 case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10258 case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10259 case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10260 case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10261 case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10262 case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10263
10264 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10265 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10266 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10267 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10268 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10269 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10270 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10271 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10272 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10273
10274 case VMX_EXIT_VMCLEAR:
10275 case VMX_EXIT_VMLAUNCH:
10276 case VMX_EXIT_VMPTRLD:
10277 case VMX_EXIT_VMPTRST:
10278 case VMX_EXIT_VMREAD:
10279 case VMX_EXIT_VMRESUME:
10280 case VMX_EXIT_VMWRITE:
10281 case VMX_EXIT_VMXOFF:
10282 case VMX_EXIT_VMXON:
10283 case VMX_EXIT_INVEPT:
10284 case VMX_EXIT_INVVPID:
10285 case VMX_EXIT_VMFUNC:
10286 case VMX_EXIT_XSAVES:
10287 case VMX_EXIT_XRSTORS:
10288 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10289 case VMX_EXIT_RESERVED_60:
10290 case VMX_EXIT_RDSEED: /* only spurious exits, so undefined */
10291 case VMX_EXIT_RESERVED_62:
10292 default:
10293 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10294 }
10295}
10296#endif /* !HMVMX_USE_FUNCTION_TABLE */
10297
10298
10299#ifdef VBOX_STRICT
10300/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10301# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10302 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10303
10304# define HMVMX_ASSERT_PREEMPT_CPUID() \
10305 do { \
10306 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10307 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10308 } while (0)
10309
10310# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10311 do { \
10312 AssertPtr(pVCpu); \
10313 AssertPtr(pMixedCtx); \
10314 AssertPtr(pVmxTransient); \
10315 Assert(pVmxTransient->fVMEntryFailed == false); \
10316 Assert(ASMIntAreEnabled()); \
10317 HMVMX_ASSERT_PREEMPT_SAFE(); \
10318 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10319 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10320 HMVMX_ASSERT_PREEMPT_SAFE(); \
10321 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10322 HMVMX_ASSERT_PREEMPT_CPUID(); \
10323 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10324 } while (0)
10325
10326# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10327 do { \
10328 Log4Func(("\n")); \
10329 } while (0)
10330#else /* nonstrict builds: */
10331# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10332 do { \
10333 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10334 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10335 } while (0)
10336# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10337#endif
10338
10339
10340/**
10341 * Advances the guest RIP after reading it from the VMCS.
10342 *
10343 * @returns VBox status code, no informational status codes.
10344 * @param pVCpu The cross context virtual CPU structure.
10345 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10346 * out-of-sync. Make sure to update the required fields
10347 * before using them.
10348 * @param pVmxTransient Pointer to the VMX transient structure.
10349 *
10350 * @remarks No-long-jump zone!!!
10351 */
10352DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10353{
10354 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10355 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10356 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10357 AssertRCReturn(rc, rc);
10358
10359 pMixedCtx->rip += pVmxTransient->cbInstr;
10360 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10361
10362 /*
10363 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10364 * pending debug exception field as it takes care of priority of events.
10365 *
10366 * See Intel spec. 32.2.1 "Debug Exceptions".
10367 */
10368 if ( !pVCpu->hm.s.fSingleInstruction
10369 && pMixedCtx->eflags.Bits.u1TF)
10370 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10371
10372 return VINF_SUCCESS;
10373}
10374
10375
10376/**
10377 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10378 * and update error record fields accordingly.
10379 *
10380 * @return VMX_IGS_* return codes.
10381 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10382 * wrong with the guest state.
10383 *
10384 * @param pVM The cross context VM structure.
10385 * @param pVCpu The cross context virtual CPU structure.
10386 * @param pCtx Pointer to the guest-CPU state.
10387 *
10388 * @remarks This function assumes our cache of the VMCS controls
10389 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10390 */
10391static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10392{
10393#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10394#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10395 uError = (err); \
10396 break; \
10397 } else do { } while (0)
10398
10399 int rc;
10400 uint32_t uError = VMX_IGS_ERROR;
10401 uint32_t u32Val;
10402 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10403
10404 do
10405 {
10406 /*
10407 * CR0.
10408 */
10409 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10410 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10411 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10412 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10413 if (fUnrestrictedGuest)
10414 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
10415
10416 uint32_t u32GuestCR0;
10417 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
10418 AssertRCBreak(rc);
10419 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
10420 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
10421 if ( !fUnrestrictedGuest
10422 && (u32GuestCR0 & X86_CR0_PG)
10423 && !(u32GuestCR0 & X86_CR0_PE))
10424 {
10425 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10426 }
10427
10428 /*
10429 * CR4.
10430 */
10431 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10432 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10433
10434 uint32_t u32GuestCR4;
10435 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
10436 AssertRCBreak(rc);
10437 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
10438 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
10439
10440 /*
10441 * IA32_DEBUGCTL MSR.
10442 */
10443 uint64_t u64Val;
10444 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10445 AssertRCBreak(rc);
10446 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10447 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10448 {
10449 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10450 }
10451 uint64_t u64DebugCtlMsr = u64Val;
10452
10453#ifdef VBOX_STRICT
10454 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10455 AssertRCBreak(rc);
10456 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10457#endif
10458 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10459
10460 /*
10461 * RIP and RFLAGS.
10462 */
10463 uint32_t u32Eflags;
10464#if HC_ARCH_BITS == 64
10465 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10466 AssertRCBreak(rc);
10467 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10468 if ( !fLongModeGuest
10469 || !pCtx->cs.Attr.n.u1Long)
10470 {
10471 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10472 }
10473 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10474 * must be identical if the "IA-32e mode guest" VM-entry
10475 * control is 1 and CS.L is 1. No check applies if the
10476 * CPU supports 64 linear-address bits. */
10477
10478 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10479 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10480 AssertRCBreak(rc);
10481 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10482 VMX_IGS_RFLAGS_RESERVED);
10483 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10484 u32Eflags = u64Val;
10485#else
10486 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10487 AssertRCBreak(rc);
10488 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10489 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10490#endif
10491
10492 if ( fLongModeGuest
10493 || ( fUnrestrictedGuest
10494 && !(u32GuestCR0 & X86_CR0_PE)))
10495 {
10496 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10497 }
10498
10499 uint32_t u32EntryInfo;
10500 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10501 AssertRCBreak(rc);
10502 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10503 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10504 {
10505 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10506 }
10507
10508 /*
10509 * 64-bit checks.
10510 */
10511#if HC_ARCH_BITS == 64
10512 if (fLongModeGuest)
10513 {
10514 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10515 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10516 }
10517
10518 if ( !fLongModeGuest
10519 && (u32GuestCR4 & X86_CR4_PCIDE))
10520 {
10521 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10522 }
10523
10524 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10525 * 51:32 beyond the processor's physical-address width are 0. */
10526
10527 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10528 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10529 {
10530 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10531 }
10532
10533 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10534 AssertRCBreak(rc);
10535 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10536
10537 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10538 AssertRCBreak(rc);
10539 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10540#endif
10541
10542 /*
10543 * PERF_GLOBAL MSR.
10544 */
10545 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10546 {
10547 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10548 AssertRCBreak(rc);
10549 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10550 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10551 }
10552
10553 /*
10554 * PAT MSR.
10555 */
10556 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10557 {
10558 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10559 AssertRCBreak(rc);
10560 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10561 for (unsigned i = 0; i < 8; i++)
10562 {
10563 uint8_t u8Val = (u64Val & 0xff);
10564 if ( u8Val != 0 /* UC */
10565 && u8Val != 1 /* WC */
10566 && u8Val != 4 /* WT */
10567 && u8Val != 5 /* WP */
10568 && u8Val != 6 /* WB */
10569 && u8Val != 7 /* UC- */)
10570 {
10571 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10572 }
10573 u64Val >>= 8;
10574 }
10575 }
10576
10577 /*
10578 * EFER MSR.
10579 */
10580 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10581 {
10582 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10583 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10584 AssertRCBreak(rc);
10585 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10586 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10587 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10588 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10589 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10590 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10591 || !(u32GuestCR0 & X86_CR0_PG)
10592 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10593 VMX_IGS_EFER_LMA_LME_MISMATCH);
10594 }
10595
10596 /*
10597 * Segment registers.
10598 */
10599 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10600 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10601 if (!(u32Eflags & X86_EFL_VM))
10602 {
10603 /* CS */
10604 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10605 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10606 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10607 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10608 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10609 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10610 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10611 /* CS cannot be loaded with NULL in protected mode. */
10612 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10613 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10614 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10615 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10616 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10617 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10618 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10619 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10620 else
10621 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10622
10623 /* SS */
10624 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10625 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10626 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10627 if ( !(pCtx->cr0 & X86_CR0_PE)
10628 || pCtx->cs.Attr.n.u4Type == 3)
10629 {
10630 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10631 }
10632 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10633 {
10634 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10635 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10636 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10637 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10638 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10639 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10640 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10641 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10642 }
10643
10644 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
10645 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10646 {
10647 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10648 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10649 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10650 || pCtx->ds.Attr.n.u4Type > 11
10651 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10652 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10653 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10654 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10655 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10656 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10657 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10658 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10659 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10660 }
10661 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10662 {
10663 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10664 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10665 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10666 || pCtx->es.Attr.n.u4Type > 11
10667 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10668 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10669 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10670 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10671 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10672 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10673 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10674 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10675 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10676 }
10677 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10678 {
10679 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10680 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10681 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10682 || pCtx->fs.Attr.n.u4Type > 11
10683 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10684 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10685 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10686 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10687 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10688 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10689 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10690 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10691 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10692 }
10693 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10694 {
10695 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10696 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10697 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10698 || pCtx->gs.Attr.n.u4Type > 11
10699 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10700 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10701 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10702 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10703 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10704 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10705 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10706 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10707 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10708 }
10709 /* 64-bit capable CPUs. */
10710#if HC_ARCH_BITS == 64
10711 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10712 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10713 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10714 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10715 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10716 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10717 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10718 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10719 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10720 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10721 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10722#endif
10723 }
10724 else
10725 {
10726 /* V86 mode checks. */
10727 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10728 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10729 {
10730 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10731 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10732 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10733 }
10734 else
10735 {
10736 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10737 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10738 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10739 }
10740
10741 /* CS */
10742 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10743 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10744 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10745 /* SS */
10746 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10747 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10748 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10749 /* DS */
10750 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10751 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10752 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10753 /* ES */
10754 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10755 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10756 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10757 /* FS */
10758 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10759 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10760 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10761 /* GS */
10762 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10763 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10764 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10765 /* 64-bit capable CPUs. */
10766#if HC_ARCH_BITS == 64
10767 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10768 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10769 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10770 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10771 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10772 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10773 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10774 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10775 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10776 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10777 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10778#endif
10779 }
10780
10781 /*
10782 * TR.
10783 */
10784 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10785 /* 64-bit capable CPUs. */
10786#if HC_ARCH_BITS == 64
10787 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10788#endif
10789 if (fLongModeGuest)
10790 {
10791 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10792 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10793 }
10794 else
10795 {
10796 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10797 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10798 VMX_IGS_TR_ATTR_TYPE_INVALID);
10799 }
10800 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10801 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10802 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10803 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10804 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10805 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10806 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10807 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10808
10809 /*
10810 * GDTR and IDTR.
10811 */
10812#if HC_ARCH_BITS == 64
10813 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10814 AssertRCBreak(rc);
10815 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10816
10817 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10818 AssertRCBreak(rc);
10819 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10820#endif
10821
10822 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10823 AssertRCBreak(rc);
10824 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10825
10826 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10827 AssertRCBreak(rc);
10828 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10829
10830 /*
10831 * Guest Non-Register State.
10832 */
10833 /* Activity State. */
10834 uint32_t u32ActivityState;
10835 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10836 AssertRCBreak(rc);
10837 HMVMX_CHECK_BREAK( !u32ActivityState
10838 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
10839 VMX_IGS_ACTIVITY_STATE_INVALID);
10840 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10841 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10842 uint32_t u32IntrState;
10843 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
10844 AssertRCBreak(rc);
10845 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
10846 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10847 {
10848 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10849 }
10850
10851 /** @todo Activity state and injecting interrupts. Left as a todo since we
10852 * currently don't use activity states but ACTIVE. */
10853
10854 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10855 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10856
10857 /* Guest interruptibility-state. */
10858 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10859 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10860 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
10861 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10862 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10863 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10864 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10865 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10866 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10867 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
10868 {
10869 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10870 {
10871 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10872 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10873 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10874 }
10875 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10876 {
10877 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10878 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10879 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10880 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10881 }
10882 }
10883 /** @todo Assumes the processor is not in SMM. */
10884 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10885 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10886 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10887 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10888 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10889 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
10890 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10891 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10892 {
10893 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
10894 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10895 }
10896
10897 /* Pending debug exceptions. */
10898#if HC_ARCH_BITS == 64
10899 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
10900 AssertRCBreak(rc);
10901 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10902 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10903 u32Val = u64Val; /* For pending debug exceptions checks below. */
10904#else
10905 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
10906 AssertRCBreak(rc);
10907 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10908 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10909#endif
10910
10911 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10912 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
10913 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10914 {
10915 if ( (u32Eflags & X86_EFL_TF)
10916 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10917 {
10918 /* Bit 14 is PendingDebug.BS. */
10919 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10920 }
10921 if ( !(u32Eflags & X86_EFL_TF)
10922 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10923 {
10924 /* Bit 14 is PendingDebug.BS. */
10925 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10926 }
10927 }
10928
10929 /* VMCS link pointer. */
10930 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10931 AssertRCBreak(rc);
10932 if (u64Val != UINT64_C(0xffffffffffffffff))
10933 {
10934 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10935 /** @todo Bits beyond the processor's physical-address width MBZ. */
10936 /** @todo 32-bit located in memory referenced by value of this field (as a
10937 * physical address) must contain the processor's VMCS revision ID. */
10938 /** @todo SMM checks. */
10939 }
10940
10941 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10942 * not using Nested Paging? */
10943 if ( pVM->hm.s.fNestedPaging
10944 && !fLongModeGuest
10945 && CPUMIsGuestInPAEModeEx(pCtx))
10946 {
10947 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10948 AssertRCBreak(rc);
10949 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10950
10951 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10952 AssertRCBreak(rc);
10953 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10954
10955 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10956 AssertRCBreak(rc);
10957 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10958
10959 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10960 AssertRCBreak(rc);
10961 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10962 }
10963
10964 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10965 if (uError == VMX_IGS_ERROR)
10966 uError = VMX_IGS_REASON_NOT_FOUND;
10967 } while (0);
10968
10969 pVCpu->hm.s.u32HMError = uError;
10970 return uError;
10971
10972#undef HMVMX_ERROR_BREAK
10973#undef HMVMX_CHECK_BREAK
10974}
10975
10976/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10977/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
10978/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10979
10980/** @name VM-exit handlers.
10981 * @{
10982 */
10983
10984/**
10985 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
10986 */
10987HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10988{
10989 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10990 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
10991 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
10992 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
10993 return VINF_SUCCESS;
10994 return VINF_EM_RAW_INTERRUPT;
10995}
10996
10997
10998/**
10999 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11000 */
11001HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11002{
11003 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11004 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11005
11006 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11007 AssertRCReturn(rc, rc);
11008
11009 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11010 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11011 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11012 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11013
11014 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11015 {
11016 /*
11017 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11018 * anything we inject is not going to cause a VM-exit directly for the event being injected.
11019 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11020 *
11021 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11022 */
11023 VMXDispatchHostNmi();
11024 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11025 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11026 return VINF_SUCCESS;
11027 }
11028
11029 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11030 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11031 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11032 { /* likely */ }
11033 else
11034 {
11035 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11036 rcStrictRc1 = VINF_SUCCESS;
11037 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11038 return rcStrictRc1;
11039 }
11040
11041 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11042 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11043 switch (uIntType)
11044 {
11045 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11046 Assert(uVector == X86_XCPT_DB);
11047 /* no break */
11048 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11049 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11050 /* no break */
11051 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11052 {
11053 switch (uVector)
11054 {
11055 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11056 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11057 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11058 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11059 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11060 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11061 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11062
11063 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11064 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11065 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11066 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11067 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11068 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11069 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11070 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11071 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11072 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11073 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11074 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11075 default:
11076 {
11077 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11078 AssertRCReturn(rc, rc);
11079
11080 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11081 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11082 {
11083 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11084 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11085 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11086
11087 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11088 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11089 AssertRCReturn(rc, rc);
11090 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11091 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11092 0 /* GCPtrFaultAddress */);
11093 AssertRCReturn(rc, rc);
11094 }
11095 else
11096 {
11097 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11098 pVCpu->hm.s.u32HMError = uVector;
11099 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11100 }
11101 break;
11102 }
11103 }
11104 break;
11105 }
11106
11107 default:
11108 {
11109 pVCpu->hm.s.u32HMError = uExitIntInfo;
11110 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11111 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11112 break;
11113 }
11114 }
11115 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11116 return rc;
11117}
11118
11119
11120/**
11121 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11122 */
11123HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11124{
11125 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11126
11127 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11128 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11129
11130 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11131 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11132 return VINF_SUCCESS;
11133}
11134
11135
11136/**
11137 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11138 */
11139HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11140{
11141 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11142 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11143 {
11144 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11145 HMVMX_RETURN_UNEXPECTED_EXIT();
11146 }
11147
11148 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11149
11150 /*
11151 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11152 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11153 */
11154 uint32_t uIntrState = 0;
11155 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11156 AssertRCReturn(rc, rc);
11157
11158 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11159 if ( fBlockSti
11160 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11161 {
11162 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11163 }
11164
11165 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11166 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11167
11168 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11169 return VINF_SUCCESS;
11170}
11171
11172
11173/**
11174 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11175 */
11176HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11177{
11178 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11179 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11180 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11181}
11182
11183
11184/**
11185 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11186 */
11187HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11188{
11189 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11190 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11191 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11192}
11193
11194
11195/**
11196 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11197 */
11198HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11199{
11200 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11201 PVM pVM = pVCpu->CTX_SUFF(pVM);
11202 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11203 if (RT_LIKELY(rc == VINF_SUCCESS))
11204 {
11205 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11206 Assert(pVmxTransient->cbInstr == 2);
11207 }
11208 else
11209 {
11210 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11211 rc = VERR_EM_INTERPRETER;
11212 }
11213 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11214 return rc;
11215}
11216
11217
11218/**
11219 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11220 */
11221HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11222{
11223 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11224 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11225 AssertRCReturn(rc, rc);
11226
11227 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11228 return VINF_EM_RAW_EMULATE_INSTR;
11229
11230 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11231 HMVMX_RETURN_UNEXPECTED_EXIT();
11232}
11233
11234
11235/**
11236 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11237 */
11238HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11239{
11240 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11241 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
11242 AssertRCReturn(rc, rc);
11243
11244 PVM pVM = pVCpu->CTX_SUFF(pVM);
11245 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11246 if (RT_LIKELY(rc == VINF_SUCCESS))
11247 {
11248 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11249 Assert(pVmxTransient->cbInstr == 2);
11250 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11251 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11252 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11253 }
11254 else
11255 rc = VERR_EM_INTERPRETER;
11256 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11257 return rc;
11258}
11259
11260
11261/**
11262 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11263 */
11264HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11265{
11266 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11267 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
11268 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11269 AssertRCReturn(rc, rc);
11270
11271 PVM pVM = pVCpu->CTX_SUFF(pVM);
11272 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11273 if (RT_SUCCESS(rc))
11274 {
11275 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11276 Assert(pVmxTransient->cbInstr == 3);
11277 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11278 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11279 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11280 }
11281 else
11282 {
11283 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11284 rc = VERR_EM_INTERPRETER;
11285 }
11286 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11287 return rc;
11288}
11289
11290
11291/**
11292 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11293 */
11294HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11295{
11296 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11297 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
11298 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
11299 AssertRCReturn(rc, rc);
11300
11301 PVM pVM = pVCpu->CTX_SUFF(pVM);
11302 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11303 if (RT_LIKELY(rc == VINF_SUCCESS))
11304 {
11305 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11306 Assert(pVmxTransient->cbInstr == 2);
11307 }
11308 else
11309 {
11310 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11311 rc = VERR_EM_INTERPRETER;
11312 }
11313 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11314 return rc;
11315}
11316
11317
11318/**
11319 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11320 */
11321HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11322{
11323 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11324 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11325
11326 if (pVCpu->hm.s.fHypercallsEnabled)
11327 {
11328#if 0
11329 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11330#else
11331 /* Aggressive state sync. for now. */
11332 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11333 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11334#endif
11335 rc |= hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11336 AssertRCReturn(rc, rc);
11337
11338 /** @todo pre-increment RIP before hypercall will break when we have to implement
11339 * continuing hypercalls (e.g. Hyper-V). */
11340 /** @todo r=bird: GIMHypercall will probably have to be able to return
11341 * informational status codes, so it should be made VBOXSTRICTRC. Not
11342 * doing that now because the status code handling isn't clean (i.e.
11343 * if you use RT_SUCCESS(rc) on the result of something, you don't
11344 * return rc in the success case, you return VINF_SUCCESS). */
11345 rc = GIMHypercall(pVCpu, pMixedCtx);
11346 /* If the hypercall changes anything other than guest general-purpose registers,
11347 we would need to reload the guest changed bits here before VM-entry. */
11348 return rc;
11349 }
11350
11351 Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11352 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11353 return VINF_SUCCESS;
11354}
11355
11356
11357/**
11358 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11359 */
11360HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11361{
11362 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11363 PVM pVM = pVCpu->CTX_SUFF(pVM);
11364 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11365
11366 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11367 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11368 AssertRCReturn(rc, rc);
11369
11370 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11371 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11372 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11373 else
11374 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11375 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11376 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
11377 return rcStrict;
11378}
11379
11380
11381/**
11382 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11383 */
11384HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11385{
11386 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11387 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11388 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11389 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11390 AssertRCReturn(rc, rc);
11391
11392 PVM pVM = pVCpu->CTX_SUFF(pVM);
11393 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11394 if (RT_LIKELY(rc == VINF_SUCCESS))
11395 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11396 else
11397 {
11398 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11399 rc = VERR_EM_INTERPRETER;
11400 }
11401 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11402 return rc;
11403}
11404
11405
11406/**
11407 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11408 */
11409HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11410{
11411 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11412 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11413 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11414 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11415 AssertRCReturn(rc, rc);
11416
11417 PVM pVM = pVCpu->CTX_SUFF(pVM);
11418 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11419 rc = VBOXSTRICTRC_VAL(rc2);
11420 if (RT_LIKELY( rc == VINF_SUCCESS
11421 || rc == VINF_EM_HALT))
11422 {
11423 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11424 AssertRCReturn(rc3, rc3);
11425
11426 if ( rc == VINF_EM_HALT
11427 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11428 {
11429 rc = VINF_SUCCESS;
11430 }
11431 }
11432 else
11433 {
11434 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11435 rc = VERR_EM_INTERPRETER;
11436 }
11437 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11438 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11439 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11440 return rc;
11441}
11442
11443
11444/**
11445 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11446 */
11447HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11448{
11449 /*
11450 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
11451 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
11452 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
11453 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
11454 */
11455 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11456 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11457 HMVMX_RETURN_UNEXPECTED_EXIT();
11458}
11459
11460
11461/**
11462 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11463 */
11464HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11465{
11466 /*
11467 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
11468 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
11469 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
11470 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
11471 */
11472 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11473 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11474 HMVMX_RETURN_UNEXPECTED_EXIT();
11475}
11476
11477
11478/**
11479 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11480 */
11481HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11482{
11483 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11484 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11485 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11486 HMVMX_RETURN_UNEXPECTED_EXIT();
11487}
11488
11489
11490/**
11491 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11492 */
11493HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11494{
11495 /*
11496 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
11497 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
11498 * See Intel spec. 25.3 "Other Causes of VM-exits".
11499 */
11500 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11501 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11502 HMVMX_RETURN_UNEXPECTED_EXIT();
11503}
11504
11505
11506/**
11507 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11508 * VM-exit.
11509 */
11510HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11511{
11512 /*
11513 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11514 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11515 *
11516 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11517 * See Intel spec. "23.8 Restrictions on VMX operation".
11518 */
11519 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11520 return VINF_SUCCESS;
11521}
11522
11523
11524/**
11525 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11526 * VM-exit.
11527 */
11528HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11529{
11530 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11531 return VINF_EM_RESET;
11532}
11533
11534
11535/**
11536 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11537 */
11538HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11539{
11540 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11541 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11542 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11543 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11544 AssertRCReturn(rc, rc);
11545
11546 pMixedCtx->rip++;
11547 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11548 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11549 rc = VINF_SUCCESS;
11550 else
11551 rc = VINF_EM_HALT;
11552
11553 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11554 if (rc != VINF_SUCCESS)
11555 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11556 return rc;
11557}
11558
11559
11560/**
11561 * VM-exit handler for instructions that result in a \#UD exception delivered to
11562 * the guest.
11563 */
11564HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11565{
11566 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11567 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11568 return VINF_SUCCESS;
11569}
11570
11571
11572/**
11573 * VM-exit handler for expiry of the VMX preemption timer.
11574 */
11575HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11576{
11577 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11578
11579 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11580 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11581
11582 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11583 PVM pVM = pVCpu->CTX_SUFF(pVM);
11584 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11585 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11586 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11587}
11588
11589
11590/**
11591 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11592 */
11593HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11594{
11595 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11596
11597 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11598 rc |= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, false /*fNeedRsp*/);
11599 rc |= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11600 AssertRCReturn(rc, rc);
11601
11602 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11603 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
11604
11605 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11606
11607 return rcStrict;
11608}
11609
11610
11611/**
11612 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11613 */
11614HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11615{
11616 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11617
11618 /* The guest should not invalidate the host CPU's TLBs, fallback to interpreter. */
11619 /** @todo implement EMInterpretInvpcid() */
11620 return VERR_EM_INTERPRETER;
11621}
11622
11623
11624/**
11625 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11626 * Error VM-exit.
11627 */
11628HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11629{
11630 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11631 AssertRCReturn(rc, rc);
11632
11633 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11634 AssertRCReturn(rc, rc);
11635
11636 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11637 NOREF(uInvalidReason);
11638
11639#ifdef VBOX_STRICT
11640 uint32_t uIntrState;
11641 RTHCUINTREG uHCReg;
11642 uint64_t u64Val;
11643 uint32_t u32Val;
11644
11645 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11646 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11647 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11648 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11649 AssertRCReturn(rc, rc);
11650
11651 Log4(("uInvalidReason %u\n", uInvalidReason));
11652 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11653 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11654 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11655 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
11656
11657 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11658 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11659 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11660 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11661 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11662 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11663 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11664 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11665 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11666 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11667 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11668 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11669#else
11670 NOREF(pVmxTransient);
11671#endif
11672
11673 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11674 return VERR_VMX_INVALID_GUEST_STATE;
11675}
11676
11677
11678/**
11679 * VM-exit handler for VM-entry failure due to an MSR-load
11680 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11681 */
11682HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11683{
11684 NOREF(pVmxTransient);
11685 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11686 HMVMX_RETURN_UNEXPECTED_EXIT();
11687}
11688
11689
11690/**
11691 * VM-exit handler for VM-entry failure due to a machine-check event
11692 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11693 */
11694HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11695{
11696 NOREF(pVmxTransient);
11697 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11698 HMVMX_RETURN_UNEXPECTED_EXIT();
11699}
11700
11701
11702/**
11703 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11704 * theory.
11705 */
11706HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11707{
11708 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
11709 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
11710 return VERR_VMX_UNDEFINED_EXIT_CODE;
11711}
11712
11713
11714/**
11715 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11716 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11717 * Conditional VM-exit.
11718 */
11719HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11720{
11721 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11722
11723 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
11724 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11725 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
11726 return VERR_EM_INTERPRETER;
11727 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11728 HMVMX_RETURN_UNEXPECTED_EXIT();
11729}
11730
11731
11732/**
11733 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11734 */
11735HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11736{
11737 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11738
11739 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
11740 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
11741 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
11742 return VERR_EM_INTERPRETER;
11743 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11744 HMVMX_RETURN_UNEXPECTED_EXIT();
11745}
11746
11747
11748/**
11749 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11750 */
11751HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11752{
11753 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11754
11755 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
11756 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11757 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11758 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11759 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11760 {
11761 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
11762 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
11763 }
11764 AssertRCReturn(rc, rc);
11765 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
11766
11767#ifdef VBOX_STRICT
11768 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
11769 {
11770 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
11771 && pMixedCtx->ecx != MSR_K6_EFER)
11772 {
11773 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11774 pMixedCtx->ecx));
11775 HMVMX_RETURN_UNEXPECTED_EXIT();
11776 }
11777# if HC_ARCH_BITS == 64
11778 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
11779 && hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11780 {
11781 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11782 HMVMX_RETURN_UNEXPECTED_EXIT();
11783 }
11784# endif
11785 }
11786#endif
11787
11788 PVM pVM = pVCpu->CTX_SUFF(pVM);
11789 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11790 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
11791 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
11792 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11793 if (RT_SUCCESS(rc))
11794 {
11795 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11796 Assert(pVmxTransient->cbInstr == 2);
11797 }
11798 return rc;
11799}
11800
11801
11802/**
11803 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11804 */
11805HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11806{
11807 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11808 PVM pVM = pVCpu->CTX_SUFF(pVM);
11809 int rc = VINF_SUCCESS;
11810
11811 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
11812 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11813 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11814 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11815 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11816 {
11817 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
11818 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
11819 }
11820 AssertRCReturn(rc, rc);
11821 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
11822
11823 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11824 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
11825 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
11826
11827 if (RT_SUCCESS(rc))
11828 {
11829 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11830
11831 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
11832 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
11833 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
11834 {
11835 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
11836 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
11837 EMInterpretWrmsr() changes it. */
11838 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
11839 }
11840 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
11841 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11842 else if (pMixedCtx->ecx == MSR_K6_EFER)
11843 {
11844 /*
11845 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
11846 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
11847 * the other bits as well, SCE and NXE. See @bugref{7368}.
11848 */
11849 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
11850 }
11851
11852 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
11853 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11854 {
11855 switch (pMixedCtx->ecx)
11856 {
11857 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
11858 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
11859 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
11860 case MSR_K8_FS_BASE: /* no break */
11861 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
11862 case MSR_K6_EFER: /* already handled above */ break;
11863 default:
11864 {
11865 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11866 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
11867#if HC_ARCH_BITS == 64
11868 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11869 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
11870#endif
11871 break;
11872 }
11873 }
11874 }
11875#ifdef VBOX_STRICT
11876 else
11877 {
11878 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
11879 switch (pMixedCtx->ecx)
11880 {
11881 case MSR_IA32_SYSENTER_CS:
11882 case MSR_IA32_SYSENTER_EIP:
11883 case MSR_IA32_SYSENTER_ESP:
11884 case MSR_K8_FS_BASE:
11885 case MSR_K8_GS_BASE:
11886 {
11887 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
11888 HMVMX_RETURN_UNEXPECTED_EXIT();
11889 }
11890
11891 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
11892 default:
11893 {
11894 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11895 {
11896 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
11897 if (pMixedCtx->ecx != MSR_K6_EFER)
11898 {
11899 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11900 pMixedCtx->ecx));
11901 HMVMX_RETURN_UNEXPECTED_EXIT();
11902 }
11903 }
11904
11905#if HC_ARCH_BITS == 64
11906 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11907 {
11908 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11909 HMVMX_RETURN_UNEXPECTED_EXIT();
11910 }
11911#endif
11912 break;
11913 }
11914 }
11915 }
11916#endif /* VBOX_STRICT */
11917 }
11918 return rc;
11919}
11920
11921
11922/**
11923 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
11924 */
11925HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11926{
11927 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11928
11929 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
11930 return VINF_EM_RAW_INTERRUPT;
11931}
11932
11933
11934/**
11935 * VM-exit handler for when the TPR value is lowered below the specified
11936 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
11937 */
11938HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11939{
11940 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11941 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
11942
11943 /*
11944 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
11945 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
11946 * resume guest execution.
11947 */
11948 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
11949 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
11950 return VINF_SUCCESS;
11951}
11952
11953
11954/**
11955 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
11956 * VM-exit.
11957 *
11958 * @retval VINF_SUCCESS when guest execution can continue.
11959 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
11960 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
11961 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
11962 * interpreter.
11963 */
11964HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11965{
11966 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11967 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
11968 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11969 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11970 AssertRCReturn(rc, rc);
11971
11972 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
11973 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
11974 PVM pVM = pVCpu->CTX_SUFF(pVM);
11975 VBOXSTRICTRC rcStrict;
11976 rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, true /*fNeedRsp*/);
11977 switch (uAccessType)
11978 {
11979 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
11980 {
11981 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11982 AssertRCReturn(rc, rc);
11983
11984 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
11985 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
11986 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
11987 AssertMsg( rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE
11988 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11989 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
11990 {
11991 case 0: /* CR0 */
11992 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11993 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
11994 break;
11995 case 2: /* CR2 */
11996 /* Nothing to do here, CR2 it's not part of the VMCS. */
11997 break;
11998 case 3: /* CR3 */
11999 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
12000 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12001 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12002 break;
12003 case 4: /* CR4 */
12004 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12005 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12006 VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12007 break;
12008 case 8: /* CR8 */
12009 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12010 /* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12011 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12012 break;
12013 default:
12014 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12015 break;
12016 }
12017
12018 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12019 break;
12020 }
12021
12022 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12023 {
12024 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12025 AssertRCReturn(rc, rc);
12026
12027 Assert( !pVM->hm.s.fNestedPaging
12028 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12029 || pVCpu->hm.s.fUsingDebugLoop
12030 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12031
12032 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12033 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12034 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12035
12036 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12037 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12038 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12039 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12040 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12041 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12042 VBOXSTRICTRC_VAL(rcStrict)));
12043 break;
12044 }
12045
12046 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12047 {
12048 AssertRCReturn(rc, rc);
12049 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12050 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12051 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12052 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12053 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12054 break;
12055 }
12056
12057 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12058 {
12059 AssertRCReturn(rc, rc);
12060 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12061 VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12062 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE,
12063 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12064 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12065 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12066 break;
12067 }
12068
12069 default:
12070 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12071 VERR_VMX_UNEXPECTED_EXCEPTION);
12072 }
12073
12074 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12075 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12076 NOREF(pVM);
12077 return rcStrict;
12078}
12079
12080
12081/**
12082 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12083 * VM-exit.
12084 */
12085HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12086{
12087 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12088 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12089
12090 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12091 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12092 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12093 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12094 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12095 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12096 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12097 AssertRCReturn(rc2, rc2);
12098
12099 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12100 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12101 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12102 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12103 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12104 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12105 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12106 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12107 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12108
12109 /* I/O operation lookup arrays. */
12110 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12111 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12112
12113 VBOXSTRICTRC rcStrict;
12114 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12115 uint32_t const cbInstr = pVmxTransient->cbInstr;
12116 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12117 PVM pVM = pVCpu->CTX_SUFF(pVM);
12118 if (fIOString)
12119 {
12120#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12121 See @bugref{5752#c158}. Should work now. */
12122 /*
12123 * INS/OUTS - I/O String instruction.
12124 *
12125 * Use instruction-information if available, otherwise fall back on
12126 * interpreting the instruction.
12127 */
12128 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12129 fIOWrite ? 'w' : 'r'));
12130 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12131 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12132 {
12133 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12134 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12135 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12136 AssertRCReturn(rc2, rc2);
12137 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12138 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12139 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12140 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12141 if (fIOWrite)
12142 {
12143 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12144 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
12145 }
12146 else
12147 {
12148 /*
12149 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12150 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12151 * See Intel Instruction spec. for "INS".
12152 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12153 */
12154 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
12155 }
12156 }
12157 else
12158 {
12159 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12160 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12161 AssertRCReturn(rc2, rc2);
12162 rcStrict = IEMExecOne(pVCpu);
12163 }
12164 /** @todo IEM needs to be setting these flags somehow. */
12165 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12166 fUpdateRipAlready = true;
12167#else
12168 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12169 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12170 if (RT_SUCCESS(rcStrict))
12171 {
12172 if (fIOWrite)
12173 {
12174 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12175 (DISCPUMODE)pDis->uAddrMode, cbValue);
12176 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12177 }
12178 else
12179 {
12180 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12181 (DISCPUMODE)pDis->uAddrMode, cbValue);
12182 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12183 }
12184 }
12185 else
12186 {
12187 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12188 pMixedCtx->rip));
12189 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12190 }
12191#endif
12192 }
12193 else
12194 {
12195 /*
12196 * IN/OUT - I/O instruction.
12197 */
12198 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12199 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12200 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12201 if (fIOWrite)
12202 {
12203 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12204 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
12205 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12206 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12207 }
12208 else
12209 {
12210 uint32_t u32Result = 0;
12211 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12212 if (IOM_SUCCESS(rcStrict))
12213 {
12214 /* Save result of I/O IN instr. in AL/AX/EAX. */
12215 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12216 }
12217 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12218 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12219 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12220 }
12221 }
12222
12223 if (IOM_SUCCESS(rcStrict))
12224 {
12225 if (!fUpdateRipAlready)
12226 {
12227 pMixedCtx->rip += cbInstr;
12228 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12229 }
12230
12231 /*
12232 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12233 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12234 */
12235 if (fIOString)
12236 {
12237 /** @todo Single-step for INS/OUTS with REP prefix? */
12238 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12239 }
12240 else if ( !fDbgStepping
12241 && fGstStepping)
12242 {
12243 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12244 }
12245
12246 /*
12247 * If any I/O breakpoints are armed, we need to check if one triggered
12248 * and take appropriate action.
12249 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12250 */
12251 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12252 AssertRCReturn(rc2, rc2);
12253
12254 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12255 * execution engines about whether hyper BPs and such are pending. */
12256 uint32_t const uDr7 = pMixedCtx->dr[7];
12257 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12258 && X86_DR7_ANY_RW_IO(uDr7)
12259 && (pMixedCtx->cr4 & X86_CR4_DE))
12260 || DBGFBpIsHwIoArmed(pVM)))
12261 {
12262 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12263
12264 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12265 VMMRZCallRing3Disable(pVCpu);
12266 HM_DISABLE_PREEMPT();
12267
12268 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12269
12270 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12271 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12272 {
12273 /* Raise #DB. */
12274 if (fIsGuestDbgActive)
12275 ASMSetDR6(pMixedCtx->dr[6]);
12276 if (pMixedCtx->dr[7] != uDr7)
12277 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12278
12279 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12280 }
12281 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
12282 else if ( rcStrict2 != VINF_SUCCESS
12283 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12284 rcStrict = rcStrict2;
12285
12286 HM_RESTORE_PREEMPT();
12287 VMMRZCallRing3Enable(pVCpu);
12288 }
12289 }
12290
12291#ifdef VBOX_STRICT
12292 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12293 Assert(!fIOWrite);
12294 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
12295 Assert(fIOWrite);
12296 else
12297 {
12298#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12299 * statuses, that the VMM device and some others may return. See
12300 * IOM_SUCCESS() for guidance. */
12301 AssertMsg( RT_FAILURE(rcStrict)
12302 || rcStrict == VINF_SUCCESS
12303 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12304 || rcStrict == VINF_EM_DBG_BREAKPOINT
12305 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12306 || rcStrict == VINF_EM_RAW_TO_R3
12307 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12308#endif
12309 }
12310#endif
12311
12312 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12313 return rcStrict;
12314}
12315
12316
12317/**
12318 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12319 * VM-exit.
12320 */
12321HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12322{
12323 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12324
12325 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12326 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12327 AssertRCReturn(rc, rc);
12328 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12329 {
12330 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12331 AssertRCReturn(rc, rc);
12332 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12333 {
12334 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12335
12336 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12337 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12338
12339 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
12340 Assert(!pVCpu->hm.s.Event.fPending);
12341 pVCpu->hm.s.Event.fPending = true;
12342 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
12343 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12344 AssertRCReturn(rc, rc);
12345 if (fErrorCodeValid)
12346 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12347 else
12348 pVCpu->hm.s.Event.u32ErrCode = 0;
12349 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12350 && uVector == X86_XCPT_PF)
12351 {
12352 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
12353 }
12354
12355 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12356 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12357 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12358 }
12359 }
12360
12361 /** @todo Emulate task switch someday, currently just going back to ring-3 for
12362 * emulation. */
12363 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12364 return VERR_EM_INTERPRETER;
12365}
12366
12367
12368/**
12369 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12370 */
12371HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12372{
12373 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12374 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12375 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12376 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12377 AssertRCReturn(rc, rc);
12378 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12379 return VINF_EM_DBG_STEPPED;
12380}
12381
12382
12383/**
12384 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12385 */
12386HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12387{
12388 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12389
12390 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12391 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12392 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12393 { /* likely */ }
12394 else
12395 {
12396 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12397 rcStrict1 = VINF_SUCCESS;
12398 return rcStrict1;
12399 }
12400
12401#if 0
12402 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
12403 * just sync the whole thing. */
12404 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12405#else
12406 /* Aggressive state sync. for now. */
12407 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12408 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12409 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12410#endif
12411 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12412 AssertRCReturn(rc, rc);
12413
12414 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12415 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12416 VBOXSTRICTRC rcStrict2;
12417 switch (uAccessType)
12418 {
12419 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12420 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12421 {
12422 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12423 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != 0x80,
12424 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12425
12426 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
12427 GCPhys &= PAGE_BASE_GC_MASK;
12428 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12429 PVM pVM = pVCpu->CTX_SUFF(pVM);
12430 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
12431 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12432
12433 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12434 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12435 CPUMCTX2CORE(pMixedCtx), GCPhys);
12436 Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
12437 if ( rcStrict2 == VINF_SUCCESS
12438 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12439 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12440 {
12441 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12442 | HM_CHANGED_GUEST_RSP
12443 | HM_CHANGED_GUEST_RFLAGS
12444 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12445 rcStrict2 = VINF_SUCCESS;
12446 }
12447 break;
12448 }
12449
12450 default:
12451 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
12452 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12453 break;
12454 }
12455
12456 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12457 if (rcStrict2 != VINF_SUCCESS)
12458 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12459 return rcStrict2;
12460}
12461
12462
12463/**
12464 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12465 * VM-exit.
12466 */
12467HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12468{
12469 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12470
12471 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12472 if (pVmxTransient->fWasGuestDebugStateActive)
12473 {
12474 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12475 HMVMX_RETURN_UNEXPECTED_EXIT();
12476 }
12477
12478 if ( !pVCpu->hm.s.fSingleInstruction
12479 && !pVmxTransient->fWasHyperDebugStateActive)
12480 {
12481 Assert(!DBGFIsStepping(pVCpu));
12482 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12483
12484 /* Don't intercept MOV DRx any more. */
12485 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12486 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12487 AssertRCReturn(rc, rc);
12488
12489 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12490 VMMRZCallRing3Disable(pVCpu);
12491 HM_DISABLE_PREEMPT();
12492
12493 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12494 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12495 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12496
12497 HM_RESTORE_PREEMPT();
12498 VMMRZCallRing3Enable(pVCpu);
12499
12500#ifdef VBOX_WITH_STATISTICS
12501 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12502 AssertRCReturn(rc, rc);
12503 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12504 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12505 else
12506 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12507#endif
12508 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12509 return VINF_SUCCESS;
12510 }
12511
12512 /*
12513 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12514 * Update the segment registers and DR7 from the CPU.
12515 */
12516 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12517 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12518 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12519 AssertRCReturn(rc, rc);
12520 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12521
12522 PVM pVM = pVCpu->CTX_SUFF(pVM);
12523 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12524 {
12525 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12526 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
12527 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
12528 if (RT_SUCCESS(rc))
12529 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12530 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12531 }
12532 else
12533 {
12534 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12535 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
12536 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
12537 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12538 }
12539
12540 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12541 if (RT_SUCCESS(rc))
12542 {
12543 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12544 AssertRCReturn(rc2, rc2);
12545 return VINF_SUCCESS;
12546 }
12547 return rc;
12548}
12549
12550
12551/**
12552 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12553 * Conditional VM-exit.
12554 */
12555HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12556{
12557 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12558 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12559
12560 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12561 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12562 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12563 { /* likely */ }
12564 else
12565 {
12566 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12567 rcStrict1 = VINF_SUCCESS;
12568 return rcStrict1;
12569 }
12570
12571 RTGCPHYS GCPhys = 0;
12572 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12573
12574#if 0
12575 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12576#else
12577 /* Aggressive state sync. for now. */
12578 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12579 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12580 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12581#endif
12582 AssertRCReturn(rc, rc);
12583
12584 /*
12585 * If we succeed, resume guest execution.
12586 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12587 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12588 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12589 * weird case. See @bugref{6043}.
12590 */
12591 PVM pVM = pVCpu->CTX_SUFF(pVM);
12592 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12593 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
12594 if ( rcStrict2 == VINF_SUCCESS
12595 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12596 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12597 {
12598 /* Successfully handled MMIO operation. */
12599 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12600 | HM_CHANGED_GUEST_RSP
12601 | HM_CHANGED_GUEST_RFLAGS
12602 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12603 return VINF_SUCCESS;
12604 }
12605 return rcStrict2;
12606}
12607
12608
12609/**
12610 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12611 * VM-exit.
12612 */
12613HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12614{
12615 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12616 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12617
12618 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12619 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12620 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12621 { /* likely */ }
12622 else
12623 {
12624 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12625 rcStrict1 = VINF_SUCCESS;
12626 return rcStrict1;
12627 }
12628
12629 RTGCPHYS GCPhys = 0;
12630 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12631 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12632#if 0
12633 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12634#else
12635 /* Aggressive state sync. for now. */
12636 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12637 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12638 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12639#endif
12640 AssertRCReturn(rc, rc);
12641
12642 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12643 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12644
12645 RTGCUINT uErrorCode = 0;
12646 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
12647 uErrorCode |= X86_TRAP_PF_ID;
12648 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
12649 uErrorCode |= X86_TRAP_PF_RW;
12650 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
12651 uErrorCode |= X86_TRAP_PF_P;
12652
12653 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12654
12655 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12656 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12657
12658 /* Handle the pagefault trap for the nested shadow table. */
12659 PVM pVM = pVCpu->CTX_SUFF(pVM);
12660 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12661 TRPMResetTrap(pVCpu);
12662
12663 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12664 if ( rcStrict2 == VINF_SUCCESS
12665 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12666 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12667 {
12668 /* Successfully synced our nested page tables. */
12669 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12670 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12671 | HM_CHANGED_GUEST_RSP
12672 | HM_CHANGED_GUEST_RFLAGS);
12673 return VINF_SUCCESS;
12674 }
12675
12676 Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12677 return rcStrict2;
12678}
12679
12680/** @} */
12681
12682/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12683/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
12684/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12685
12686/** @name VM-exit exception handlers.
12687 * @{
12688 */
12689
12690/**
12691 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12692 */
12693static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12694{
12695 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12696 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12697
12698 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12699 AssertRCReturn(rc, rc);
12700
12701 if (!(pMixedCtx->cr0 & X86_CR0_NE))
12702 {
12703 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12704 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12705
12706 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12707 * provides VM-exit instruction length. If this causes problem later,
12708 * disassemble the instruction like it's done on AMD-V. */
12709 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12710 AssertRCReturn(rc2, rc2);
12711 return rc;
12712 }
12713
12714 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12715 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12716 return rc;
12717}
12718
12719
12720/**
12721 * VM-exit exception handler for \#BP (Breakpoint exception).
12722 */
12723static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12724{
12725 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12726 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12727
12728 /** @todo Try optimize this by not saving the entire guest state unless
12729 * really needed. */
12730 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12731 AssertRCReturn(rc, rc);
12732
12733 PVM pVM = pVCpu->CTX_SUFF(pVM);
12734 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12735 if (rc == VINF_EM_RAW_GUEST_TRAP)
12736 {
12737 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12738 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12739 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12740 AssertRCReturn(rc, rc);
12741
12742 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12743 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12744 }
12745
12746 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
12747 return rc;
12748}
12749
12750
12751/**
12752 * VM-exit exception handler for \#AC (alignment check exception).
12753 */
12754static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12755{
12756 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12757
12758 /*
12759 * Re-inject it. We'll detect any nesting before getting here.
12760 */
12761 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12762 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12763 AssertRCReturn(rc, rc);
12764 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
12765
12766 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12767 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12768 return VINF_SUCCESS;
12769}
12770
12771
12772/**
12773 * VM-exit exception handler for \#DB (Debug exception).
12774 */
12775static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12776{
12777 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12778 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
12779 Log6(("XcptDB\n"));
12780
12781 /*
12782 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
12783 * for processing.
12784 */
12785 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12786 AssertRCReturn(rc, rc);
12787
12788 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
12789 uint64_t uDR6 = X86_DR6_INIT_VAL;
12790 uDR6 |= ( pVmxTransient->uExitQualification
12791 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
12792
12793 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
12794 if (rc == VINF_EM_RAW_GUEST_TRAP)
12795 {
12796 /*
12797 * The exception was for the guest. Update DR6, DR7.GD and
12798 * IA32_DEBUGCTL.LBR before forwarding it.
12799 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
12800 */
12801 VMMRZCallRing3Disable(pVCpu);
12802 HM_DISABLE_PREEMPT();
12803
12804 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
12805 pMixedCtx->dr[6] |= uDR6;
12806 if (CPUMIsGuestDebugStateActive(pVCpu))
12807 ASMSetDR6(pMixedCtx->dr[6]);
12808
12809 HM_RESTORE_PREEMPT();
12810 VMMRZCallRing3Enable(pVCpu);
12811
12812 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12813 AssertRCReturn(rc, rc);
12814
12815 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
12816 pMixedCtx->dr[7] &= ~X86_DR7_GD;
12817
12818 /* Paranoia. */
12819 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
12820 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
12821
12822 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
12823 AssertRCReturn(rc, rc);
12824
12825 /*
12826 * Raise #DB in the guest.
12827 *
12828 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
12829 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
12830 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
12831 *
12832 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
12833 */
12834 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12835 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12836 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12837 AssertRCReturn(rc, rc);
12838 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12839 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12840 return VINF_SUCCESS;
12841 }
12842
12843 /*
12844 * Not a guest trap, must be a hypervisor related debug event then.
12845 * Update DR6 in case someone is interested in it.
12846 */
12847 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
12848 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
12849 CPUMSetHyperDR6(pVCpu, uDR6);
12850
12851 return rc;
12852}
12853
12854
12855/**
12856 * VM-exit exception handler for \#NM (Device-not-available exception: floating
12857 * point exception).
12858 */
12859static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12860{
12861 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12862
12863 /* We require CR0 and EFER. EFER is always up-to-date. */
12864 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12865 AssertRCReturn(rc, rc);
12866
12867 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
12868 VMMRZCallRing3Disable(pVCpu);
12869 HM_DISABLE_PREEMPT();
12870
12871 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
12872 if (pVmxTransient->fWasGuestFPUStateActive)
12873 {
12874 rc = VINF_EM_RAW_GUEST_TRAP;
12875 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
12876 }
12877 else
12878 {
12879#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12880 Assert(!pVmxTransient->fWasGuestFPUStateActive || pVCpu->hm.s.fUsingDebugLoop);
12881#endif
12882 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12883 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
12884 }
12885
12886 HM_RESTORE_PREEMPT();
12887 VMMRZCallRing3Enable(pVCpu);
12888
12889 if (rc == VINF_SUCCESS)
12890 {
12891 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
12892 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12893 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
12894 pVCpu->hm.s.fPreloadGuestFpu = true;
12895 }
12896 else
12897 {
12898 /* Forward #NM to the guest. */
12899 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
12900 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12901 AssertRCReturn(rc, rc);
12902 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12903 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
12904 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
12905 }
12906
12907 return VINF_SUCCESS;
12908}
12909
12910
12911/**
12912 * VM-exit exception handler for \#GP (General-protection exception).
12913 *
12914 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
12915 */
12916static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12917{
12918 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12919 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
12920
12921 int rc;
12922 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
12923 { /* likely */ }
12924 else
12925 {
12926#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12927 Assert(pVCpu->hm.s.fUsingDebugLoop);
12928#endif
12929 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
12930 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12931 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12932 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12933 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12934 AssertRCReturn(rc, rc);
12935 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
12936 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
12937 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12938 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12939 return rc;
12940 }
12941
12942 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
12943 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
12944
12945 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
12946 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12947 AssertRCReturn(rc, rc);
12948
12949 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12950 uint32_t cbOp = 0;
12951 PVM pVM = pVCpu->CTX_SUFF(pVM);
12952 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12953 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
12954 if (RT_SUCCESS(rc))
12955 {
12956 rc = VINF_SUCCESS;
12957 Assert(cbOp == pDis->cbInstr);
12958 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12959 switch (pDis->pCurInstr->uOpcode)
12960 {
12961 case OP_CLI:
12962 {
12963 pMixedCtx->eflags.Bits.u1IF = 0;
12964 pMixedCtx->eflags.Bits.u1RF = 0;
12965 pMixedCtx->rip += pDis->cbInstr;
12966 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12967 if ( !fDbgStepping
12968 && pMixedCtx->eflags.Bits.u1TF)
12969 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12970 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
12971 break;
12972 }
12973
12974 case OP_STI:
12975 {
12976 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
12977 pMixedCtx->eflags.Bits.u1IF = 1;
12978 pMixedCtx->eflags.Bits.u1RF = 0;
12979 pMixedCtx->rip += pDis->cbInstr;
12980 if (!fOldIF)
12981 {
12982 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
12983 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
12984 }
12985 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12986 if ( !fDbgStepping
12987 && pMixedCtx->eflags.Bits.u1TF)
12988 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12989 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
12990 break;
12991 }
12992
12993 case OP_HLT:
12994 {
12995 rc = VINF_EM_HALT;
12996 pMixedCtx->rip += pDis->cbInstr;
12997 pMixedCtx->eflags.Bits.u1RF = 0;
12998 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12999 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13000 break;
13001 }
13002
13003 case OP_POPF:
13004 {
13005 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13006 uint32_t cbParm;
13007 uint32_t uMask;
13008 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13009 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13010 {
13011 cbParm = 4;
13012 uMask = 0xffffffff;
13013 }
13014 else
13015 {
13016 cbParm = 2;
13017 uMask = 0xffff;
13018 }
13019
13020 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13021 RTGCPTR GCPtrStack = 0;
13022 X86EFLAGS Eflags;
13023 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13024 &GCPtrStack);
13025 if (RT_SUCCESS(rc))
13026 {
13027 Assert(sizeof(Eflags.u32) >= cbParm);
13028 Eflags.u32 = 0;
13029 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13030 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13031 }
13032 if (RT_FAILURE(rc))
13033 {
13034 rc = VERR_EM_INTERPRETER;
13035 break;
13036 }
13037 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13038 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13039 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13040 pMixedCtx->esp += cbParm;
13041 pMixedCtx->esp &= uMask;
13042 pMixedCtx->rip += pDis->cbInstr;
13043 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13044 | HM_CHANGED_GUEST_RSP
13045 | HM_CHANGED_GUEST_RFLAGS);
13046 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13047 POPF restores EFLAGS.TF. */
13048 if ( !fDbgStepping
13049 && fGstStepping)
13050 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13051 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13052 break;
13053 }
13054
13055 case OP_PUSHF:
13056 {
13057 uint32_t cbParm;
13058 uint32_t uMask;
13059 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13060 {
13061 cbParm = 4;
13062 uMask = 0xffffffff;
13063 }
13064 else
13065 {
13066 cbParm = 2;
13067 uMask = 0xffff;
13068 }
13069
13070 /* Get the stack pointer & push the contents of eflags onto the stack. */
13071 RTGCPTR GCPtrStack = 0;
13072 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13073 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13074 if (RT_FAILURE(rc))
13075 {
13076 rc = VERR_EM_INTERPRETER;
13077 break;
13078 }
13079 X86EFLAGS Eflags = pMixedCtx->eflags;
13080 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13081 Eflags.Bits.u1RF = 0;
13082 Eflags.Bits.u1VM = 0;
13083
13084 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13085 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13086 {
13087 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13088 rc = VERR_EM_INTERPRETER;
13089 break;
13090 }
13091 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13092 pMixedCtx->esp -= cbParm;
13093 pMixedCtx->esp &= uMask;
13094 pMixedCtx->rip += pDis->cbInstr;
13095 pMixedCtx->eflags.Bits.u1RF = 0;
13096 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13097 | HM_CHANGED_GUEST_RSP
13098 | HM_CHANGED_GUEST_RFLAGS);
13099 if ( !fDbgStepping
13100 && pMixedCtx->eflags.Bits.u1TF)
13101 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13102 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13103 break;
13104 }
13105
13106 case OP_IRET:
13107 {
13108 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13109 * instruction reference. */
13110 RTGCPTR GCPtrStack = 0;
13111 uint32_t uMask = 0xffff;
13112 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13113 uint16_t aIretFrame[3];
13114 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13115 {
13116 rc = VERR_EM_INTERPRETER;
13117 break;
13118 }
13119 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13120 &GCPtrStack);
13121 if (RT_SUCCESS(rc))
13122 {
13123 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13124 PGMACCESSORIGIN_HM));
13125 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13126 }
13127 if (RT_FAILURE(rc))
13128 {
13129 rc = VERR_EM_INTERPRETER;
13130 break;
13131 }
13132 pMixedCtx->eip = 0;
13133 pMixedCtx->ip = aIretFrame[0];
13134 pMixedCtx->cs.Sel = aIretFrame[1];
13135 pMixedCtx->cs.ValidSel = aIretFrame[1];
13136 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13137 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13138 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13139 pMixedCtx->sp += sizeof(aIretFrame);
13140 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13141 | HM_CHANGED_GUEST_SEGMENT_REGS
13142 | HM_CHANGED_GUEST_RSP
13143 | HM_CHANGED_GUEST_RFLAGS);
13144 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13145 if ( !fDbgStepping
13146 && fGstStepping)
13147 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13148 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13149 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13150 break;
13151 }
13152
13153 case OP_INT:
13154 {
13155 uint16_t uVector = pDis->Param1.uValue & 0xff;
13156 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13157 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13158 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13159 break;
13160 }
13161
13162 case OP_INTO:
13163 {
13164 if (pMixedCtx->eflags.Bits.u1OF)
13165 {
13166 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13167 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13168 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13169 }
13170 else
13171 {
13172 pMixedCtx->eflags.Bits.u1RF = 0;
13173 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13174 }
13175 break;
13176 }
13177
13178 default:
13179 {
13180 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13181 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13182 EMCODETYPE_SUPERVISOR);
13183 rc = VBOXSTRICTRC_VAL(rc2);
13184 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13185 /** @todo We have to set pending-debug exceptions here when the guest is
13186 * single-stepping depending on the instruction that was interpreted. */
13187 Log4(("#GP rc=%Rrc\n", rc));
13188 break;
13189 }
13190 }
13191 }
13192 else
13193 rc = VERR_EM_INTERPRETER;
13194
13195 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13196 ("#GP Unexpected rc=%Rrc\n", rc));
13197 return rc;
13198}
13199
13200
13201/**
13202 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13203 * the exception reported in the VMX transient structure back into the VM.
13204 *
13205 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13206 * up-to-date.
13207 */
13208static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13209{
13210 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13211#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13212 Assert(pVCpu->hm.s.fUsingDebugLoop);
13213#endif
13214
13215 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13216 hmR0VmxCheckExitDueToEventDelivery(). */
13217 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13218 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13219 AssertRCReturn(rc, rc);
13220 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13221
13222#ifdef DEBUG_ramshankar
13223 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13224 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13225 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13226#endif
13227
13228 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13229 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13230 return VINF_SUCCESS;
13231}
13232
13233
13234/**
13235 * VM-exit exception handler for \#PF (Page-fault exception).
13236 */
13237static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13238{
13239 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13240 PVM pVM = pVCpu->CTX_SUFF(pVM);
13241 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13242 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13243 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13244 AssertRCReturn(rc, rc);
13245
13246 if (!pVM->hm.s.fNestedPaging)
13247 { /* likely */ }
13248 else
13249 {
13250#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13251 Assert(pVCpu->hm.s.fUsingDebugLoop);
13252#endif
13253 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13254 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13255 {
13256 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13257 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13258 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13259 }
13260 else
13261 {
13262 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13263 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13264 Log4(("Pending #DF due to vectoring #PF. NP\n"));
13265 }
13266 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13267 return rc;
13268 }
13269
13270 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13271 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13272 if (pVmxTransient->fVectoringPF)
13273 {
13274 Assert(pVCpu->hm.s.Event.fPending);
13275 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13276 }
13277
13278 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13279 AssertRCReturn(rc, rc);
13280
13281 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13282 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13283
13284 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13285 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13286 (RTGCPTR)pVmxTransient->uExitQualification);
13287
13288 Log4(("#PF: rc=%Rrc\n", rc));
13289 if (rc == VINF_SUCCESS)
13290 {
13291 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13292 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
13293 * memory? We don't update the whole state here... */
13294 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13295 | HM_CHANGED_GUEST_RSP
13296 | HM_CHANGED_GUEST_RFLAGS
13297 | HM_CHANGED_VMX_GUEST_APIC_STATE);
13298 TRPMResetTrap(pVCpu);
13299 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13300 return rc;
13301 }
13302
13303 if (rc == VINF_EM_RAW_GUEST_TRAP)
13304 {
13305 if (!pVmxTransient->fVectoringDoublePF)
13306 {
13307 /* It's a guest page fault and needs to be reflected to the guest. */
13308 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13309 TRPMResetTrap(pVCpu);
13310 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13311 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13312 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13313 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13314 }
13315 else
13316 {
13317 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13318 TRPMResetTrap(pVCpu);
13319 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13320 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13321 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
13322 }
13323
13324 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13325 return VINF_SUCCESS;
13326 }
13327
13328 TRPMResetTrap(pVCpu);
13329 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13330 return rc;
13331}
13332
13333/** @} */
13334
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette