VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 64831

Last change on this file since 64831 was 64823, checked in by vboxsync, 8 years ago

VMM/HMVMXR0: Comment to clear up confusing code paths.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 584.9 KB
Line 
1/* $Id: HMVMXR0.cpp 64823 2016-12-09 17:05:15Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2016 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#include <iprt/x86.h>
24#include <iprt/asm-amd64-x86.h>
25#include <iprt/thread.h>
26
27#include <VBox/vmm/pdmapi.h>
28#include <VBox/vmm/dbgf.h>
29#include <VBox/vmm/iem.h>
30#include <VBox/vmm/iom.h>
31#include <VBox/vmm/selm.h>
32#include <VBox/vmm/tm.h>
33#include <VBox/vmm/gim.h>
34#include <VBox/vmm/apic.h>
35#ifdef VBOX_WITH_REM
36# include <VBox/vmm/rem.h>
37#endif
38#include "HMInternal.h"
39#include <VBox/vmm/vm.h>
40#include "HMVMXR0.h"
41#include "dtrace/VBoxVMM.h"
42
43#ifdef DEBUG_ramshankar
44# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
45# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
46# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
47# define HMVMX_ALWAYS_CHECK_GUEST_STATE
48# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
49# define HMVMX_ALWAYS_TRAP_PF
50# define HMVMX_ALWAYS_SWAP_FPU_STATE
51# define HMVMX_ALWAYS_FLUSH_TLB
52# define HMVMX_ALWAYS_SWAP_EFER
53#endif
54
55
56/*********************************************************************************************************************************
57* Defined Constants And Macros *
58*********************************************************************************************************************************/
59/** Use the function table. */
60#define HMVMX_USE_FUNCTION_TABLE
61
62/** Determine which tagged-TLB flush handler to use. */
63#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68/** @name Updated-guest-state flags.
69 * @{ */
70#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
71#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
72#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
73#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
74#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
75#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
76#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
77#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
78#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
79#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
80#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
81#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
82#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
83#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
84#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
85#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
86#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
87#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
88#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
89#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
90#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
91 | HMVMX_UPDATED_GUEST_RSP \
92 | HMVMX_UPDATED_GUEST_RFLAGS \
93 | HMVMX_UPDATED_GUEST_CR0 \
94 | HMVMX_UPDATED_GUEST_CR3 \
95 | HMVMX_UPDATED_GUEST_CR4 \
96 | HMVMX_UPDATED_GUEST_GDTR \
97 | HMVMX_UPDATED_GUEST_IDTR \
98 | HMVMX_UPDATED_GUEST_LDTR \
99 | HMVMX_UPDATED_GUEST_TR \
100 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
101 | HMVMX_UPDATED_GUEST_DEBUG \
102 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
103 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
104 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
105 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
106 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
107 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
108 | HMVMX_UPDATED_GUEST_INTR_STATE \
109 | HMVMX_UPDATED_GUEST_APIC_STATE)
110/** @} */
111
112/** @name
113 * Flags to skip redundant reads of some common VMCS fields that are not part of
114 * the guest-CPU state but are in the transient structure.
115 */
116#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
117#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
118#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
119#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
120#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
123/** @} */
124
125/** @name
126 * States of the VMCS.
127 *
128 * This does not reflect all possible VMCS states but currently only those
129 * needed for maintaining the VMCS consistently even when thread-context hooks
130 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
131 */
132#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
133#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
134#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
135/** @} */
136
137/**
138 * Exception bitmap mask for real-mode guests (real-on-v86).
139 *
140 * We need to intercept all exceptions manually except:
141 * - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
142 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
143 * due to bugs in Intel CPUs.
144 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
145 * support.
146 */
147#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
148 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
149 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
150 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
151 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
152 /* RT_BIT(X86_XCPT_MF) always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
153 | RT_BIT(X86_XCPT_XF))
154
155/**
156 * Exception bitmap mask for all contributory exceptions.
157 *
158 * Page fault is deliberately excluded here as it's conditional as to whether
159 * it's contributory or benign. Page faults are handled separately.
160 */
161#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
162 | RT_BIT(X86_XCPT_DE))
163
164/** Maximum VM-instruction error number. */
165#define HMVMX_INSTR_ERROR_MAX 28
166
167/** Profiling macro. */
168#ifdef HM_PROFILE_EXIT_DISPATCH
169# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
170# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
171#else
172# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
173# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
174#endif
175
176/** Assert that preemption is disabled or covered by thread-context hooks. */
177#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
178 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
179
180/** Assert that we haven't migrated CPUs when thread-context hooks are not
181 * used. */
182#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
183 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
184 ("Illegal migration! Entered on CPU %u Current %u\n", \
185 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
186
187/** Helper macro for VM-exit handlers called unexpectedly. */
188#define HMVMX_RETURN_UNEXPECTED_EXIT() \
189 do { \
190 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
191 return VERR_VMX_UNEXPECTED_EXIT; \
192 } while (0)
193
194
195/*********************************************************************************************************************************
196* Structures and Typedefs *
197*********************************************************************************************************************************/
198/**
199 * VMX transient state.
200 *
201 * A state structure for holding miscellaneous information across
202 * VMX non-root operation and restored after the transition.
203 */
204typedef struct VMXTRANSIENT
205{
206 /** The host's rflags/eflags. */
207 RTCCUINTREG fEFlags;
208#if HC_ARCH_BITS == 32
209 uint32_t u32Alignment0;
210#endif
211 /** The guest's TPR value used for TPR shadowing. */
212 uint8_t u8GuestTpr;
213 /** Alignment. */
214 uint8_t abAlignment0[7];
215
216 /** The basic VM-exit reason. */
217 uint16_t uExitReason;
218 /** Alignment. */
219 uint16_t u16Alignment0;
220 /** The VM-exit interruption error code. */
221 uint32_t uExitIntErrorCode;
222 /** The VM-exit exit code qualification. */
223 uint64_t uExitQualification;
224
225 /** The VM-exit interruption-information field. */
226 uint32_t uExitIntInfo;
227 /** The VM-exit instruction-length field. */
228 uint32_t cbInstr;
229 /** The VM-exit instruction-information field. */
230 union
231 {
232 /** Plain unsigned int representation. */
233 uint32_t u;
234 /** INS and OUTS information. */
235 struct
236 {
237 uint32_t u7Reserved0 : 7;
238 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
239 uint32_t u3AddrSize : 3;
240 uint32_t u5Reserved1 : 5;
241 /** The segment register (X86_SREG_XXX). */
242 uint32_t iSegReg : 3;
243 uint32_t uReserved2 : 14;
244 } StrIo;
245 } ExitInstrInfo;
246 /** Whether the VM-entry failed or not. */
247 bool fVMEntryFailed;
248 /** Alignment. */
249 uint8_t abAlignment1[3];
250
251 /** The VM-entry interruption-information field. */
252 uint32_t uEntryIntInfo;
253 /** The VM-entry exception error code field. */
254 uint32_t uEntryXcptErrorCode;
255 /** The VM-entry instruction length field. */
256 uint32_t cbEntryInstr;
257
258 /** IDT-vectoring information field. */
259 uint32_t uIdtVectoringInfo;
260 /** IDT-vectoring error code. */
261 uint32_t uIdtVectoringErrorCode;
262
263 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
264 uint32_t fVmcsFieldsRead;
265
266 /** Whether the guest FPU was active at the time of VM-exit. */
267 bool fWasGuestFPUStateActive;
268 /** Whether the guest debug state was active at the time of VM-exit. */
269 bool fWasGuestDebugStateActive;
270 /** Whether the hyper debug state was active at the time of VM-exit. */
271 bool fWasHyperDebugStateActive;
272 /** Whether TSC-offsetting should be setup before VM-entry. */
273 bool fUpdateTscOffsettingAndPreemptTimer;
274 /** Whether the VM-exit was caused by a page-fault during delivery of a
275 * contributory exception or a page-fault. */
276 bool fVectoringDoublePF;
277 /** Whether the VM-exit was caused by a page-fault during delivery of an
278 * external interrupt or NMI. */
279 bool fVectoringPF;
280} VMXTRANSIENT;
281AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
282AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
283AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
284AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
285AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
286/** Pointer to VMX transient state. */
287typedef VMXTRANSIENT *PVMXTRANSIENT;
288
289
290/**
291 * MSR-bitmap read permissions.
292 */
293typedef enum VMXMSREXITREAD
294{
295 /** Reading this MSR causes a VM-exit. */
296 VMXMSREXIT_INTERCEPT_READ = 0xb,
297 /** Reading this MSR does not cause a VM-exit. */
298 VMXMSREXIT_PASSTHRU_READ
299} VMXMSREXITREAD;
300/** Pointer to MSR-bitmap read permissions. */
301typedef VMXMSREXITREAD* PVMXMSREXITREAD;
302
303/**
304 * MSR-bitmap write permissions.
305 */
306typedef enum VMXMSREXITWRITE
307{
308 /** Writing to this MSR causes a VM-exit. */
309 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
310 /** Writing to this MSR does not cause a VM-exit. */
311 VMXMSREXIT_PASSTHRU_WRITE
312} VMXMSREXITWRITE;
313/** Pointer to MSR-bitmap write permissions. */
314typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
315
316
317/**
318 * VMX VM-exit handler.
319 *
320 * @returns Strict VBox status code (i.e. informational status codes too).
321 * @param pVCpu The cross context virtual CPU structure.
322 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
323 * out-of-sync. Make sure to update the required
324 * fields before using them.
325 * @param pVmxTransient Pointer to the VMX-transient structure.
326 */
327#ifndef HMVMX_USE_FUNCTION_TABLE
328typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
329#else
330typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
331/** Pointer to VM-exit handler. */
332typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
333#endif
334
335/**
336 * VMX VM-exit handler, non-strict status code.
337 *
338 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
339 *
340 * @returns VBox status code, no informational status code returned.
341 * @param pVCpu The cross context virtual CPU structure.
342 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
343 * out-of-sync. Make sure to update the required
344 * fields before using them.
345 * @param pVmxTransient Pointer to the VMX-transient structure.
346 *
347 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
348 * use of that status code will be replaced with VINF_EM_SOMETHING
349 * later when switching over to IEM.
350 */
351#ifndef HMVMX_USE_FUNCTION_TABLE
352typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
353#else
354typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
355#endif
356
357
358/*********************************************************************************************************************************
359* Internal Functions *
360*********************************************************************************************************************************/
361static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
362static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
363static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
364static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
365 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
366 bool fStepping, uint32_t *puIntState);
367#if HC_ARCH_BITS == 32
368static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
369#endif
370#ifndef HMVMX_USE_FUNCTION_TABLE
371DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
372# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
373# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
374#else
375# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
376# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
377#endif
378
379
380/** @name VM-exit handlers.
381 * @{
382 */
383static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
384static FNVMXEXITHANDLER hmR0VmxExitExtInt;
385static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
386static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
387static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
388static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
389static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
390static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
391static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
392static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
393static FNVMXEXITHANDLER hmR0VmxExitCpuid;
394static FNVMXEXITHANDLER hmR0VmxExitGetsec;
395static FNVMXEXITHANDLER hmR0VmxExitHlt;
396static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
397static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
398static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
399static FNVMXEXITHANDLER hmR0VmxExitVmcall;
400static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
401static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
402static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
403static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
404static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
405static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
406static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
407static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
408static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
409static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
410static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
411static FNVMXEXITHANDLER hmR0VmxExitMwait;
412static FNVMXEXITHANDLER hmR0VmxExitMtf;
413static FNVMXEXITHANDLER hmR0VmxExitMonitor;
414static FNVMXEXITHANDLER hmR0VmxExitPause;
415static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
416static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
417static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
418static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
419static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
420static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
421static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
422static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
423static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
424static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
425static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
426static FNVMXEXITHANDLER hmR0VmxExitRdrand;
427static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
428/** @} */
429
430static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
431static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
432static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
433static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
434static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
435static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
436static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
437static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
438static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
439
440
441/*********************************************************************************************************************************
442* Global Variables *
443*********************************************************************************************************************************/
444#ifdef HMVMX_USE_FUNCTION_TABLE
445
446/**
447 * VMX_EXIT dispatch table.
448 */
449static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
450{
451 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
452 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
453 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
454 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
455 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
456 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
457 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
458 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
459 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
460 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
461 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
462 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
463 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
464 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
465 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
466 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
467 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
468 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
469 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
470 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
471 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
472 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
473 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
474 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
475 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
476 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
477 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
478 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
479 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
480 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
481 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
482 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
483 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
484 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
485 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
486 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
487 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
488 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
489 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
490 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
491 /* 40 UNDEFINED */ hmR0VmxExitPause,
492 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
493 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
494 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
495 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
496 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
497 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
498 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
499 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
500 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
501 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
502 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
503 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
504 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
505 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
506 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
507 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
508 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
509 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
510 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
511 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
512 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
513 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
514 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
515 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
516};
517#endif /* HMVMX_USE_FUNCTION_TABLE */
518
519#ifdef VBOX_STRICT
520static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
521{
522 /* 0 */ "(Not Used)",
523 /* 1 */ "VMCALL executed in VMX root operation.",
524 /* 2 */ "VMCLEAR with invalid physical address.",
525 /* 3 */ "VMCLEAR with VMXON pointer.",
526 /* 4 */ "VMLAUNCH with non-clear VMCS.",
527 /* 5 */ "VMRESUME with non-launched VMCS.",
528 /* 6 */ "VMRESUME after VMXOFF",
529 /* 7 */ "VM-entry with invalid control fields.",
530 /* 8 */ "VM-entry with invalid host state fields.",
531 /* 9 */ "VMPTRLD with invalid physical address.",
532 /* 10 */ "VMPTRLD with VMXON pointer.",
533 /* 11 */ "VMPTRLD with incorrect revision identifier.",
534 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
535 /* 13 */ "VMWRITE to read-only VMCS component.",
536 /* 14 */ "(Not Used)",
537 /* 15 */ "VMXON executed in VMX root operation.",
538 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
539 /* 17 */ "VM-entry with non-launched executing VMCS.",
540 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
541 /* 19 */ "VMCALL with non-clear VMCS.",
542 /* 20 */ "VMCALL with invalid VM-exit control fields.",
543 /* 21 */ "(Not Used)",
544 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
545 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
546 /* 24 */ "VMCALL with invalid SMM-monitor features.",
547 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
548 /* 26 */ "VM-entry with events blocked by MOV SS.",
549 /* 27 */ "(Not Used)",
550 /* 28 */ "Invalid operand to INVEPT/INVVPID."
551};
552#endif /* VBOX_STRICT */
553
554
555
556/**
557 * Updates the VM's last error record.
558 *
559 * If there was a VMX instruction error, reads the error data from the VMCS and
560 * updates VCPU's last error record as well.
561 *
562 * @param pVM The cross context VM structure.
563 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
564 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
565 * VERR_VMX_INVALID_VMCS_FIELD.
566 * @param rc The error code.
567 */
568static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
569{
570 AssertPtr(pVM);
571 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
572 || rc == VERR_VMX_UNABLE_TO_START_VM)
573 {
574 AssertPtrReturnVoid(pVCpu);
575 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
576 }
577 pVM->hm.s.lLastError = rc;
578}
579
580
581/**
582 * Reads the VM-entry interruption-information field from the VMCS into the VMX
583 * transient structure.
584 *
585 * @returns VBox status code.
586 * @param pVmxTransient Pointer to the VMX transient structure.
587 *
588 * @remarks No-long-jump zone!!!
589 */
590DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
591{
592 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
593 AssertRCReturn(rc, rc);
594 return VINF_SUCCESS;
595}
596
597
598#ifdef VBOX_STRICT
599/**
600 * Reads the VM-entry exception error code field from the VMCS into
601 * the VMX transient structure.
602 *
603 * @returns VBox status code.
604 * @param pVmxTransient Pointer to the VMX transient structure.
605 *
606 * @remarks No-long-jump zone!!!
607 */
608DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
609{
610 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
611 AssertRCReturn(rc, rc);
612 return VINF_SUCCESS;
613}
614#endif /* VBOX_STRICT */
615
616
617#ifdef VBOX_STRICT
618/**
619 * Reads the VM-entry exception error code field from the VMCS into
620 * the VMX transient structure.
621 *
622 * @returns VBox status code.
623 * @param pVmxTransient Pointer to the VMX transient structure.
624 *
625 * @remarks No-long-jump zone!!!
626 */
627DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
628{
629 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
630 AssertRCReturn(rc, rc);
631 return VINF_SUCCESS;
632}
633#endif /* VBOX_STRICT */
634
635
636/**
637 * Reads the VM-exit interruption-information field from the VMCS into the VMX
638 * transient structure.
639 *
640 * @returns VBox status code.
641 * @param pVmxTransient Pointer to the VMX transient structure.
642 */
643DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
644{
645 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
646 {
647 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
648 AssertRCReturn(rc, rc);
649 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
650 }
651 return VINF_SUCCESS;
652}
653
654
655/**
656 * Reads the VM-exit interruption error code from the VMCS into the VMX
657 * transient structure.
658 *
659 * @returns VBox status code.
660 * @param pVmxTransient Pointer to the VMX transient structure.
661 */
662DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
663{
664 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
665 {
666 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
667 AssertRCReturn(rc, rc);
668 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
669 }
670 return VINF_SUCCESS;
671}
672
673
674/**
675 * Reads the VM-exit instruction length field from the VMCS into the VMX
676 * transient structure.
677 *
678 * @returns VBox status code.
679 * @param pVmxTransient Pointer to the VMX transient structure.
680 */
681DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
682{
683 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
684 {
685 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
686 AssertRCReturn(rc, rc);
687 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
688 }
689 return VINF_SUCCESS;
690}
691
692
693/**
694 * Reads the VM-exit instruction-information field from the VMCS into
695 * the VMX transient structure.
696 *
697 * @returns VBox status code.
698 * @param pVmxTransient Pointer to the VMX transient structure.
699 */
700DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
701{
702 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
703 {
704 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
705 AssertRCReturn(rc, rc);
706 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
707 }
708 return VINF_SUCCESS;
709}
710
711
712/**
713 * Reads the exit code qualification from the VMCS into the VMX transient
714 * structure.
715 *
716 * @returns VBox status code.
717 * @param pVCpu The cross context virtual CPU structure of the
718 * calling EMT. (Required for the VMCS cache case.)
719 * @param pVmxTransient Pointer to the VMX transient structure.
720 */
721DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
722{
723 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
724 {
725 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
726 AssertRCReturn(rc, rc);
727 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
728 }
729 return VINF_SUCCESS;
730}
731
732
733/**
734 * Reads the IDT-vectoring information field from the VMCS into the VMX
735 * transient structure.
736 *
737 * @returns VBox status code.
738 * @param pVmxTransient Pointer to the VMX transient structure.
739 *
740 * @remarks No-long-jump zone!!!
741 */
742DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
743{
744 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
745 {
746 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
747 AssertRCReturn(rc, rc);
748 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
749 }
750 return VINF_SUCCESS;
751}
752
753
754/**
755 * Reads the IDT-vectoring error code from the VMCS into the VMX
756 * transient structure.
757 *
758 * @returns VBox status code.
759 * @param pVmxTransient Pointer to the VMX transient structure.
760 */
761DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
762{
763 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
764 {
765 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
766 AssertRCReturn(rc, rc);
767 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
768 }
769 return VINF_SUCCESS;
770}
771
772
773/**
774 * Enters VMX root mode operation on the current CPU.
775 *
776 * @returns VBox status code.
777 * @param pVM The cross context VM structure. Can be
778 * NULL, after a resume.
779 * @param HCPhysCpuPage Physical address of the VMXON region.
780 * @param pvCpuPage Pointer to the VMXON region.
781 */
782static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
783{
784 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
785 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
786 Assert(pvCpuPage);
787 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
788
789 if (pVM)
790 {
791 /* Write the VMCS revision dword to the VMXON region. */
792 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
793 }
794
795 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
796 RTCCUINTREG fEFlags = ASMIntDisableFlags();
797
798 /* Enable the VMX bit in CR4 if necessary. */
799 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
800
801 /* Enter VMX root mode. */
802 int rc = VMXEnable(HCPhysCpuPage);
803 if (RT_FAILURE(rc))
804 {
805 if (!(uOldCr4 & X86_CR4_VMXE))
806 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
807
808 if (pVM)
809 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
810 }
811
812 /* Restore interrupts. */
813 ASMSetFlags(fEFlags);
814 return rc;
815}
816
817
818/**
819 * Exits VMX root mode operation on the current CPU.
820 *
821 * @returns VBox status code.
822 */
823static int hmR0VmxLeaveRootMode(void)
824{
825 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
826
827 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
828 RTCCUINTREG fEFlags = ASMIntDisableFlags();
829
830 /* If we're for some reason not in VMX root mode, then don't leave it. */
831 RTCCUINTREG uHostCR4 = ASMGetCR4();
832
833 int rc;
834 if (uHostCR4 & X86_CR4_VMXE)
835 {
836 /* Exit VMX root mode and clear the VMX bit in CR4. */
837 VMXDisable();
838 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
839 rc = VINF_SUCCESS;
840 }
841 else
842 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
843
844 /* Restore interrupts. */
845 ASMSetFlags(fEFlags);
846 return rc;
847}
848
849
850/**
851 * Allocates and maps one physically contiguous page. The allocated page is
852 * zero'd out. (Used by various VT-x structures).
853 *
854 * @returns IPRT status code.
855 * @param pMemObj Pointer to the ring-0 memory object.
856 * @param ppVirt Where to store the virtual address of the
857 * allocation.
858 * @param pHCPhys Where to store the physical address of the
859 * allocation.
860 */
861DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
862{
863 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
864 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
865 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
866
867 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
868 if (RT_FAILURE(rc))
869 return rc;
870 *ppVirt = RTR0MemObjAddress(*pMemObj);
871 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
872 ASMMemZero32(*ppVirt, PAGE_SIZE);
873 return VINF_SUCCESS;
874}
875
876
877/**
878 * Frees and unmaps an allocated physical page.
879 *
880 * @param pMemObj Pointer to the ring-0 memory object.
881 * @param ppVirt Where to re-initialize the virtual address of
882 * allocation as 0.
883 * @param pHCPhys Where to re-initialize the physical address of the
884 * allocation as 0.
885 */
886DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
887{
888 AssertPtr(pMemObj);
889 AssertPtr(ppVirt);
890 AssertPtr(pHCPhys);
891 if (*pMemObj != NIL_RTR0MEMOBJ)
892 {
893 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
894 AssertRC(rc);
895 *pMemObj = NIL_RTR0MEMOBJ;
896 *ppVirt = 0;
897 *pHCPhys = 0;
898 }
899}
900
901
902/**
903 * Worker function to free VT-x related structures.
904 *
905 * @returns IPRT status code.
906 * @param pVM The cross context VM structure.
907 */
908static void hmR0VmxStructsFree(PVM pVM)
909{
910 for (VMCPUID i = 0; i < pVM->cCpus; i++)
911 {
912 PVMCPU pVCpu = &pVM->aCpus[i];
913 AssertPtr(pVCpu);
914
915 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
916 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
917
918 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
919 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
920
921 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
922 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
923 }
924
925 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
926#ifdef VBOX_WITH_CRASHDUMP_MAGIC
927 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
928#endif
929}
930
931
932/**
933 * Worker function to allocate VT-x related VM structures.
934 *
935 * @returns IPRT status code.
936 * @param pVM The cross context VM structure.
937 */
938static int hmR0VmxStructsAlloc(PVM pVM)
939{
940 /*
941 * Initialize members up-front so we can cleanup properly on allocation failure.
942 */
943#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
944 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
945 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
946 pVM->hm.s.vmx.HCPhys##a_Name = 0;
947
948#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
949 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
950 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
951 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
952
953#ifdef VBOX_WITH_CRASHDUMP_MAGIC
954 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
955#endif
956 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
957
958 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
959 for (VMCPUID i = 0; i < pVM->cCpus; i++)
960 {
961 PVMCPU pVCpu = &pVM->aCpus[i];
962 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
963 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
964 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
965 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
966 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
967 }
968#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
969#undef VMXLOCAL_INIT_VM_MEMOBJ
970
971 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
972 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
973 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
974 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
975
976 /*
977 * Allocate all the VT-x structures.
978 */
979 int rc = VINF_SUCCESS;
980#ifdef VBOX_WITH_CRASHDUMP_MAGIC
981 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
982 if (RT_FAILURE(rc))
983 goto cleanup;
984 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
985 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
986#endif
987
988 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
989 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
990 {
991 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
992 &pVM->hm.s.vmx.HCPhysApicAccess);
993 if (RT_FAILURE(rc))
994 goto cleanup;
995 }
996
997 /*
998 * Initialize per-VCPU VT-x structures.
999 */
1000 for (VMCPUID i = 0; i < pVM->cCpus; i++)
1001 {
1002 PVMCPU pVCpu = &pVM->aCpus[i];
1003 AssertPtr(pVCpu);
1004
1005 /* Allocate the VM control structure (VMCS). */
1006 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1007 if (RT_FAILURE(rc))
1008 goto cleanup;
1009
1010 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
1011 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1012 {
1013 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1014 &pVCpu->hm.s.vmx.HCPhysVirtApic);
1015 if (RT_FAILURE(rc))
1016 goto cleanup;
1017 }
1018
1019 /*
1020 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1021 * transparent accesses of specific MSRs.
1022 *
1023 * If the condition for enabling MSR bitmaps changes here, don't forget to
1024 * update HMAreMsrBitmapsAvailable().
1025 */
1026 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1027 {
1028 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1029 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1030 if (RT_FAILURE(rc))
1031 goto cleanup;
1032 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1033 }
1034
1035 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1036 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1037 if (RT_FAILURE(rc))
1038 goto cleanup;
1039
1040 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1041 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1042 if (RT_FAILURE(rc))
1043 goto cleanup;
1044 }
1045
1046 return VINF_SUCCESS;
1047
1048cleanup:
1049 hmR0VmxStructsFree(pVM);
1050 return rc;
1051}
1052
1053
1054/**
1055 * Does global VT-x initialization (called during module initialization).
1056 *
1057 * @returns VBox status code.
1058 */
1059VMMR0DECL(int) VMXR0GlobalInit(void)
1060{
1061#ifdef HMVMX_USE_FUNCTION_TABLE
1062 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1063# ifdef VBOX_STRICT
1064 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1065 Assert(g_apfnVMExitHandlers[i]);
1066# endif
1067#endif
1068 return VINF_SUCCESS;
1069}
1070
1071
1072/**
1073 * Does global VT-x termination (called during module termination).
1074 */
1075VMMR0DECL(void) VMXR0GlobalTerm()
1076{
1077 /* Nothing to do currently. */
1078}
1079
1080
1081/**
1082 * Sets up and activates VT-x on the current CPU.
1083 *
1084 * @returns VBox status code.
1085 * @param pCpu Pointer to the global CPU info struct.
1086 * @param pVM The cross context VM structure. Can be
1087 * NULL after a host resume operation.
1088 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1089 * fEnabledByHost is @c true).
1090 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1091 * @a fEnabledByHost is @c true).
1092 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1093 * enable VT-x on the host.
1094 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1095 */
1096VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1097 void *pvMsrs)
1098{
1099 Assert(pCpu);
1100 Assert(pvMsrs);
1101 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1102
1103 /* Enable VT-x if it's not already enabled by the host. */
1104 if (!fEnabledByHost)
1105 {
1106 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1107 if (RT_FAILURE(rc))
1108 return rc;
1109 }
1110
1111 /*
1112 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1113 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1114 */
1115 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1116 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1117 {
1118 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1119 pCpu->fFlushAsidBeforeUse = false;
1120 }
1121 else
1122 pCpu->fFlushAsidBeforeUse = true;
1123
1124 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1125 ++pCpu->cTlbFlushes;
1126
1127 return VINF_SUCCESS;
1128}
1129
1130
1131/**
1132 * Deactivates VT-x on the current CPU.
1133 *
1134 * @returns VBox status code.
1135 * @param pCpu Pointer to the global CPU info struct.
1136 * @param pvCpuPage Pointer to the VMXON region.
1137 * @param HCPhysCpuPage Physical address of the VMXON region.
1138 *
1139 * @remarks This function should never be called when SUPR0EnableVTx() or
1140 * similar was used to enable VT-x on the host.
1141 */
1142VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1143{
1144 NOREF(pCpu);
1145 NOREF(pvCpuPage);
1146 NOREF(HCPhysCpuPage);
1147
1148 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1149 return hmR0VmxLeaveRootMode();
1150}
1151
1152
1153/**
1154 * Sets the permission bits for the specified MSR in the MSR bitmap.
1155 *
1156 * @param pVCpu The cross context virtual CPU structure.
1157 * @param uMsr The MSR value.
1158 * @param enmRead Whether reading this MSR causes a VM-exit.
1159 * @param enmWrite Whether writing this MSR causes a VM-exit.
1160 */
1161static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1162{
1163 int32_t iBit;
1164 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1165
1166 /*
1167 * Layout:
1168 * 0x000 - 0x3ff - Low MSR read bits
1169 * 0x400 - 0x7ff - High MSR read bits
1170 * 0x800 - 0xbff - Low MSR write bits
1171 * 0xc00 - 0xfff - High MSR write bits
1172 */
1173 if (uMsr <= 0x00001FFF)
1174 iBit = uMsr;
1175 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1176 {
1177 iBit = uMsr - UINT32_C(0xC0000000);
1178 pbMsrBitmap += 0x400;
1179 }
1180 else
1181 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1182
1183 Assert(iBit <= 0x1fff);
1184 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1185 ASMBitSet(pbMsrBitmap, iBit);
1186 else
1187 ASMBitClear(pbMsrBitmap, iBit);
1188
1189 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1190 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1191 else
1192 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1193}
1194
1195
1196#ifdef VBOX_STRICT
1197/**
1198 * Gets the permission bits for the specified MSR in the MSR bitmap.
1199 *
1200 * @returns VBox status code.
1201 * @retval VINF_SUCCESS if the specified MSR is found.
1202 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1203 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1204 *
1205 * @param pVCpu The cross context virtual CPU structure.
1206 * @param uMsr The MSR.
1207 * @param penmRead Where to store the read permissions.
1208 * @param penmWrite Where to store the write permissions.
1209 */
1210static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1211{
1212 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1213 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1214 int32_t iBit;
1215 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1216
1217 /* See hmR0VmxSetMsrPermission() for the layout. */
1218 if (uMsr <= 0x00001FFF)
1219 iBit = uMsr;
1220 else if ( uMsr >= 0xC0000000
1221 && uMsr <= 0xC0001FFF)
1222 {
1223 iBit = (uMsr - 0xC0000000);
1224 pbMsrBitmap += 0x400;
1225 }
1226 else
1227 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1228
1229 Assert(iBit <= 0x1fff);
1230 if (ASMBitTest(pbMsrBitmap, iBit))
1231 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1232 else
1233 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1234
1235 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1236 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1237 else
1238 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1239 return VINF_SUCCESS;
1240}
1241#endif /* VBOX_STRICT */
1242
1243
1244/**
1245 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1246 * area.
1247 *
1248 * @returns VBox status code.
1249 * @param pVCpu The cross context virtual CPU structure.
1250 * @param cMsrs The number of MSRs.
1251 */
1252DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1253{
1254 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1255 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1256 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1257 {
1258 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1259 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1260 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1261 }
1262
1263 /* Update number of guest MSRs to load/store across the world-switch. */
1264 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1265 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1266
1267 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1268 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1269 AssertRCReturn(rc, rc);
1270
1271 /* Update the VCPU's copy of the MSR count. */
1272 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1273
1274 return VINF_SUCCESS;
1275}
1276
1277
1278/**
1279 * Adds a new (or updates the value of an existing) guest/host MSR
1280 * pair to be swapped during the world-switch as part of the
1281 * auto-load/store MSR area in the VMCS.
1282 *
1283 * @returns VBox status code.
1284 * @param pVCpu The cross context virtual CPU structure.
1285 * @param uMsr The MSR.
1286 * @param uGuestMsrValue Value of the guest MSR.
1287 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1288 * necessary.
1289 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1290 * its value was updated. Optional, can be NULL.
1291 */
1292static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1293 bool *pfAddedAndUpdated)
1294{
1295 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1296 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1297 uint32_t i;
1298 for (i = 0; i < cMsrs; i++)
1299 {
1300 if (pGuestMsr->u32Msr == uMsr)
1301 break;
1302 pGuestMsr++;
1303 }
1304
1305 bool fAdded = false;
1306 if (i == cMsrs)
1307 {
1308 ++cMsrs;
1309 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1310 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1311
1312 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1313 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1314 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1315
1316 fAdded = true;
1317 }
1318
1319 /* Update the MSR values in the auto-load/store MSR area. */
1320 pGuestMsr->u32Msr = uMsr;
1321 pGuestMsr->u64Value = uGuestMsrValue;
1322
1323 /* Create/update the MSR slot in the host MSR area. */
1324 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1325 pHostMsr += i;
1326 pHostMsr->u32Msr = uMsr;
1327
1328 /*
1329 * Update the host MSR only when requested by the caller AND when we're
1330 * adding it to the auto-load/store area. Otherwise, it would have been
1331 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1332 */
1333 bool fUpdatedMsrValue = false;
1334 if ( fAdded
1335 && fUpdateHostMsr)
1336 {
1337 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1338 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1339 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1340 fUpdatedMsrValue = true;
1341 }
1342
1343 if (pfAddedAndUpdated)
1344 *pfAddedAndUpdated = fUpdatedMsrValue;
1345 return VINF_SUCCESS;
1346}
1347
1348
1349/**
1350 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1351 * auto-load/store MSR area in the VMCS.
1352 *
1353 * @returns VBox status code.
1354 * @param pVCpu The cross context virtual CPU structure.
1355 * @param uMsr The MSR.
1356 */
1357static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1358{
1359 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1360 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1361 for (uint32_t i = 0; i < cMsrs; i++)
1362 {
1363 /* Find the MSR. */
1364 if (pGuestMsr->u32Msr == uMsr)
1365 {
1366 /* If it's the last MSR, simply reduce the count. */
1367 if (i == cMsrs - 1)
1368 {
1369 --cMsrs;
1370 break;
1371 }
1372
1373 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1374 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1375 pLastGuestMsr += cMsrs - 1;
1376 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1377 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1378
1379 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1380 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1381 pLastHostMsr += cMsrs - 1;
1382 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1383 pHostMsr->u64Value = pLastHostMsr->u64Value;
1384 --cMsrs;
1385 break;
1386 }
1387 pGuestMsr++;
1388 }
1389
1390 /* Update the VMCS if the count changed (meaning the MSR was found). */
1391 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1392 {
1393 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1394 AssertRCReturn(rc, rc);
1395
1396 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1397 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1398 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1399
1400 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1401 return VINF_SUCCESS;
1402 }
1403
1404 return VERR_NOT_FOUND;
1405}
1406
1407
1408/**
1409 * Checks if the specified guest MSR is part of the auto-load/store area in
1410 * the VMCS.
1411 *
1412 * @returns true if found, false otherwise.
1413 * @param pVCpu The cross context virtual CPU structure.
1414 * @param uMsr The MSR to find.
1415 */
1416static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1417{
1418 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1419 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1420
1421 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1422 {
1423 if (pGuestMsr->u32Msr == uMsr)
1424 return true;
1425 }
1426 return false;
1427}
1428
1429
1430/**
1431 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1432 *
1433 * @param pVCpu The cross context virtual CPU structure.
1434 *
1435 * @remarks No-long-jump zone!!!
1436 */
1437static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1438{
1439 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1440 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1441 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1442 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1443
1444 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1445 {
1446 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1447
1448 /*
1449 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1450 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1451 */
1452 if (pHostMsr->u32Msr == MSR_K6_EFER)
1453 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1454 else
1455 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1456 }
1457
1458 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1459}
1460
1461
1462/**
1463 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1464 * perform lazy restoration of the host MSRs while leaving VT-x.
1465 *
1466 * @param pVCpu The cross context virtual CPU structure.
1467 *
1468 * @remarks No-long-jump zone!!!
1469 */
1470static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1471{
1472 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1473
1474 /*
1475 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1476 */
1477 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1478 {
1479 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1480#if HC_ARCH_BITS == 64
1481 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1482 {
1483 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1484 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1485 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1486 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1487 }
1488#endif
1489 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1490 }
1491}
1492
1493
1494/**
1495 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1496 * lazily while leaving VT-x.
1497 *
1498 * @returns true if it does, false otherwise.
1499 * @param pVCpu The cross context virtual CPU structure.
1500 * @param uMsr The MSR to check.
1501 */
1502static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1503{
1504 NOREF(pVCpu);
1505#if HC_ARCH_BITS == 64
1506 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1507 {
1508 switch (uMsr)
1509 {
1510 case MSR_K8_LSTAR:
1511 case MSR_K6_STAR:
1512 case MSR_K8_SF_MASK:
1513 case MSR_K8_KERNEL_GS_BASE:
1514 return true;
1515 }
1516 }
1517#else
1518 RT_NOREF(pVCpu, uMsr);
1519#endif
1520 return false;
1521}
1522
1523
1524/**
1525 * Saves a set of guest MSRs back into the guest-CPU context.
1526 *
1527 * @param pVCpu The cross context virtual CPU structure.
1528 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1529 * out-of-sync. Make sure to update the required fields
1530 * before using them.
1531 *
1532 * @remarks No-long-jump zone!!!
1533 */
1534static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1535{
1536 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1537 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1538
1539 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1540 {
1541 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1542#if HC_ARCH_BITS == 64
1543 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1544 {
1545 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1546 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1547 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1548 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1549 }
1550#else
1551 NOREF(pMixedCtx);
1552#endif
1553 }
1554}
1555
1556
1557/**
1558 * Loads a set of guests MSRs to allow read/passthru to the guest.
1559 *
1560 * The name of this function is slightly confusing. This function does NOT
1561 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1562 * common prefix for functions dealing with "lazy restoration" of the shared
1563 * MSRs.
1564 *
1565 * @param pVCpu The cross context virtual CPU structure.
1566 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1567 * out-of-sync. Make sure to update the required fields
1568 * before using them.
1569 *
1570 * @remarks No-long-jump zone!!!
1571 */
1572static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1573{
1574 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1575 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1576
1577#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1578 do { \
1579 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1580 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1581 else \
1582 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1583 } while (0)
1584
1585 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1586 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1587 {
1588#if HC_ARCH_BITS == 64
1589 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1590 {
1591 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1592 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1593 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1594 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1595 }
1596#else
1597 RT_NOREF(pMixedCtx);
1598#endif
1599 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1600 }
1601
1602#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1603}
1604
1605
1606/**
1607 * Performs lazy restoration of the set of host MSRs if they were previously
1608 * loaded with guest MSR values.
1609 *
1610 * @param pVCpu The cross context virtual CPU structure.
1611 *
1612 * @remarks No-long-jump zone!!!
1613 * @remarks The guest MSRs should have been saved back into the guest-CPU
1614 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1615 */
1616static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1617{
1618 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1619 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1620
1621 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1622 {
1623 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1624#if HC_ARCH_BITS == 64
1625 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1626 {
1627 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1628 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1629 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1630 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1631 }
1632#endif
1633 }
1634 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1635}
1636
1637
1638/**
1639 * Verifies that our cached values of the VMCS controls are all
1640 * consistent with what's actually present in the VMCS.
1641 *
1642 * @returns VBox status code.
1643 * @param pVCpu The cross context virtual CPU structure.
1644 */
1645static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1646{
1647 uint32_t u32Val;
1648 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1649 AssertRCReturn(rc, rc);
1650 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1651 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1652
1653 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1654 AssertRCReturn(rc, rc);
1655 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1656 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1657
1658 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1659 AssertRCReturn(rc, rc);
1660 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1661 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1662
1663 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1664 AssertRCReturn(rc, rc);
1665 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1666 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1667
1668 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1669 {
1670 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1671 AssertRCReturn(rc, rc);
1672 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1673 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1674 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1675 }
1676
1677 return VINF_SUCCESS;
1678}
1679
1680
1681#ifdef VBOX_STRICT
1682/**
1683 * Verifies that our cached host EFER value has not changed
1684 * since we cached it.
1685 *
1686 * @param pVCpu The cross context virtual CPU structure.
1687 */
1688static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1689{
1690 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1691
1692 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1693 {
1694 uint64_t u64Val;
1695 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1696 AssertRC(rc);
1697
1698 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1699 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1700 }
1701}
1702
1703
1704/**
1705 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1706 * VMCS are correct.
1707 *
1708 * @param pVCpu The cross context virtual CPU structure.
1709 */
1710static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1711{
1712 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1713
1714 /* Verify MSR counts in the VMCS are what we think it should be. */
1715 uint32_t cMsrs;
1716 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1717 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1718
1719 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1720 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1721
1722 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1723 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1724
1725 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1726 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1727 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1728 {
1729 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1730 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1731 pGuestMsr->u32Msr, cMsrs));
1732
1733 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1734 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1735 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1736
1737 /* Verify that the permissions are as expected in the MSR bitmap. */
1738 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1739 {
1740 VMXMSREXITREAD enmRead;
1741 VMXMSREXITWRITE enmWrite;
1742 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1743 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1744 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1745 {
1746 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1747 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1748 }
1749 else
1750 {
1751 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1752 pGuestMsr->u32Msr, cMsrs));
1753 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1754 pGuestMsr->u32Msr, cMsrs));
1755 }
1756 }
1757 }
1758}
1759#endif /* VBOX_STRICT */
1760
1761
1762/**
1763 * Flushes the TLB using EPT.
1764 *
1765 * @returns VBox status code.
1766 * @param pVCpu The cross context virtual CPU structure of the calling
1767 * EMT. Can be NULL depending on @a enmFlush.
1768 * @param enmFlush Type of flush.
1769 *
1770 * @remarks Caller is responsible for making sure this function is called only
1771 * when NestedPaging is supported and providing @a enmFlush that is
1772 * supported by the CPU.
1773 * @remarks Can be called with interrupts disabled.
1774 */
1775static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1776{
1777 uint64_t au64Descriptor[2];
1778 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1779 au64Descriptor[0] = 0;
1780 else
1781 {
1782 Assert(pVCpu);
1783 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1784 }
1785 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1786
1787 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1788 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1789 rc));
1790 if ( RT_SUCCESS(rc)
1791 && pVCpu)
1792 {
1793 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1794 }
1795}
1796
1797
1798/**
1799 * Flushes the TLB using VPID.
1800 *
1801 * @returns VBox status code.
1802 * @param pVM The cross context VM structure.
1803 * @param pVCpu The cross context virtual CPU structure of the calling
1804 * EMT. Can be NULL depending on @a enmFlush.
1805 * @param enmFlush Type of flush.
1806 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1807 * on @a enmFlush).
1808 *
1809 * @remarks Can be called with interrupts disabled.
1810 */
1811static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1812{
1813 NOREF(pVM);
1814 AssertPtr(pVM);
1815 Assert(pVM->hm.s.vmx.fVpid);
1816
1817 uint64_t au64Descriptor[2];
1818 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1819 {
1820 au64Descriptor[0] = 0;
1821 au64Descriptor[1] = 0;
1822 }
1823 else
1824 {
1825 AssertPtr(pVCpu);
1826 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1827 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1828 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1829 au64Descriptor[1] = GCPtr;
1830 }
1831
1832 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1833 AssertMsg(rc == VINF_SUCCESS,
1834 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1835 if ( RT_SUCCESS(rc)
1836 && pVCpu)
1837 {
1838 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1839 }
1840}
1841
1842
1843/**
1844 * Invalidates a guest page by guest virtual address. Only relevant for
1845 * EPT/VPID, otherwise there is nothing really to invalidate.
1846 *
1847 * @returns VBox status code.
1848 * @param pVM The cross context VM structure.
1849 * @param pVCpu The cross context virtual CPU structure.
1850 * @param GCVirt Guest virtual address of the page to invalidate.
1851 */
1852VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1853{
1854 AssertPtr(pVM);
1855 AssertPtr(pVCpu);
1856 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1857
1858 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1859 if (!fFlushPending)
1860 {
1861 /*
1862 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1863 * See @bugref{6043} and @bugref{6177}.
1864 *
1865 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1866 * function maybe called in a loop with individual addresses.
1867 */
1868 if (pVM->hm.s.vmx.fVpid)
1869 {
1870 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1871 {
1872 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1873 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1874 }
1875 else
1876 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1877 }
1878 else if (pVM->hm.s.fNestedPaging)
1879 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1880 }
1881
1882 return VINF_SUCCESS;
1883}
1884
1885
1886/**
1887 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1888 * otherwise there is nothing really to invalidate.
1889 *
1890 * @returns VBox status code.
1891 * @param pVM The cross context VM structure.
1892 * @param pVCpu The cross context virtual CPU structure.
1893 * @param GCPhys Guest physical address of the page to invalidate.
1894 */
1895VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1896{
1897 NOREF(pVM); NOREF(GCPhys);
1898 LogFlowFunc(("%RGp\n", GCPhys));
1899
1900 /*
1901 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1902 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1903 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1904 */
1905 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1906 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1907 return VINF_SUCCESS;
1908}
1909
1910
1911/**
1912 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1913 * case where neither EPT nor VPID is supported by the CPU.
1914 *
1915 * @param pVM The cross context VM structure.
1916 * @param pVCpu The cross context virtual CPU structure.
1917 * @param pCpu Pointer to the global HM struct.
1918 *
1919 * @remarks Called with interrupts disabled.
1920 */
1921static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1922{
1923 AssertPtr(pVCpu);
1924 AssertPtr(pCpu);
1925 NOREF(pVM);
1926
1927 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1928
1929 Assert(pCpu->idCpu != NIL_RTCPUID);
1930 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1931 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1932 pVCpu->hm.s.fForceTLBFlush = false;
1933 return;
1934}
1935
1936
1937/**
1938 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1939 *
1940 * @param pVM The cross context VM structure.
1941 * @param pVCpu The cross context virtual CPU structure.
1942 * @param pCpu Pointer to the global HM CPU struct.
1943 * @remarks All references to "ASID" in this function pertains to "VPID" in
1944 * Intel's nomenclature. The reason is, to avoid confusion in compare
1945 * statements since the host-CPU copies are named "ASID".
1946 *
1947 * @remarks Called with interrupts disabled.
1948 */
1949static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1950{
1951#ifdef VBOX_WITH_STATISTICS
1952 bool fTlbFlushed = false;
1953# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1954# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1955 if (!fTlbFlushed) \
1956 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1957 } while (0)
1958#else
1959# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1960# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1961#endif
1962
1963 AssertPtr(pVM);
1964 AssertPtr(pCpu);
1965 AssertPtr(pVCpu);
1966 Assert(pCpu->idCpu != NIL_RTCPUID);
1967
1968 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1969 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1970 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1971
1972 /*
1973 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1974 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1975 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1976 */
1977 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1978 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1979 {
1980 ++pCpu->uCurrentAsid;
1981 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1982 {
1983 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1984 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1985 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1986 }
1987
1988 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1989 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1990 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1991
1992 /*
1993 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1994 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1995 */
1996 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1997 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1998 HMVMX_SET_TAGGED_TLB_FLUSHED();
1999 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
2000 }
2001
2002 /* Check for explicit TLB flushes. */
2003 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2004 {
2005 /*
2006 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
2007 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2008 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2009 * but not guest-physical mappings.
2010 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2011 */
2012 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2013 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2014 HMVMX_SET_TAGGED_TLB_FLUSHED();
2015 }
2016
2017 pVCpu->hm.s.fForceTLBFlush = false;
2018 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2019
2020 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2021 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2022 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2023 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2024 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2025 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2026 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2027 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2028 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2029
2030 /* Update VMCS with the VPID. */
2031 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2032 AssertRC(rc);
2033
2034#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2035}
2036
2037
2038/**
2039 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2040 *
2041 * @returns VBox status code.
2042 * @param pVM The cross context VM structure.
2043 * @param pVCpu The cross context virtual CPU structure.
2044 * @param pCpu Pointer to the global HM CPU struct.
2045 *
2046 * @remarks Called with interrupts disabled.
2047 */
2048static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2049{
2050 AssertPtr(pVM);
2051 AssertPtr(pVCpu);
2052 AssertPtr(pCpu);
2053 Assert(pCpu->idCpu != NIL_RTCPUID);
2054 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2055 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2056
2057 /*
2058 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2059 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2060 */
2061 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2062 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2063 {
2064 pVCpu->hm.s.fForceTLBFlush = true;
2065 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2066 }
2067
2068 /* Check for explicit TLB flushes. */
2069 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2070 {
2071 pVCpu->hm.s.fForceTLBFlush = true;
2072 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2073 }
2074
2075 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2076 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2077
2078 if (pVCpu->hm.s.fForceTLBFlush)
2079 {
2080 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2081 pVCpu->hm.s.fForceTLBFlush = false;
2082 }
2083}
2084
2085
2086/**
2087 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2088 *
2089 * @returns VBox status code.
2090 * @param pVM The cross context VM structure.
2091 * @param pVCpu The cross context virtual CPU structure.
2092 * @param pCpu Pointer to the global HM CPU struct.
2093 *
2094 * @remarks Called with interrupts disabled.
2095 */
2096static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2097{
2098 AssertPtr(pVM);
2099 AssertPtr(pVCpu);
2100 AssertPtr(pCpu);
2101 Assert(pCpu->idCpu != NIL_RTCPUID);
2102 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2103 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2104
2105 /*
2106 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2107 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2108 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2109 */
2110 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2111 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2112 {
2113 pVCpu->hm.s.fForceTLBFlush = true;
2114 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2115 }
2116
2117 /* Check for explicit TLB flushes. */
2118 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2119 {
2120 /*
2121 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2122 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2123 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2124 */
2125 pVCpu->hm.s.fForceTLBFlush = true;
2126 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2127 }
2128
2129 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2130 if (pVCpu->hm.s.fForceTLBFlush)
2131 {
2132 ++pCpu->uCurrentAsid;
2133 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2134 {
2135 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2136 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2137 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2138 }
2139
2140 pVCpu->hm.s.fForceTLBFlush = false;
2141 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2142 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2143 if (pCpu->fFlushAsidBeforeUse)
2144 {
2145 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2146 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2147 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2148 {
2149 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2150 pCpu->fFlushAsidBeforeUse = false;
2151 }
2152 else
2153 {
2154 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2155 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2156 }
2157 }
2158 }
2159
2160 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2161 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2162 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2163 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2164 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2165 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2166 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2167
2168 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2169 AssertRC(rc);
2170}
2171
2172
2173/**
2174 * Flushes the guest TLB entry based on CPU capabilities.
2175 *
2176 * @param pVCpu The cross context virtual CPU structure.
2177 * @param pCpu Pointer to the global HM CPU struct.
2178 */
2179DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2180{
2181#ifdef HMVMX_ALWAYS_FLUSH_TLB
2182 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2183#endif
2184 PVM pVM = pVCpu->CTX_SUFF(pVM);
2185 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2186 {
2187 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2188 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2189 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2190 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2191 default:
2192 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2193 break;
2194 }
2195
2196 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2197}
2198
2199
2200/**
2201 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2202 * TLB entries from the host TLB before VM-entry.
2203 *
2204 * @returns VBox status code.
2205 * @param pVM The cross context VM structure.
2206 */
2207static int hmR0VmxSetupTaggedTlb(PVM pVM)
2208{
2209 /*
2210 * Determine optimal flush type for Nested Paging.
2211 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2212 * guest execution (see hmR3InitFinalizeR0()).
2213 */
2214 if (pVM->hm.s.fNestedPaging)
2215 {
2216 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2217 {
2218 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2219 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2220 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2221 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2222 else
2223 {
2224 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2225 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2226 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2227 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2228 }
2229
2230 /* Make sure the write-back cacheable memory type for EPT is supported. */
2231 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2232 {
2233 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2234 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2235 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2236 }
2237
2238 /* EPT requires a page-walk length of 4. */
2239 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2240 {
2241 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2242 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2243 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2244 }
2245 }
2246 else
2247 {
2248 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2249 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2250 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2251 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2252 }
2253 }
2254
2255 /*
2256 * Determine optimal flush type for VPID.
2257 */
2258 if (pVM->hm.s.vmx.fVpid)
2259 {
2260 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2261 {
2262 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2263 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2264 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2265 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2266 else
2267 {
2268 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2269 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2270 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2271 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2272 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2273 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2274 pVM->hm.s.vmx.fVpid = false;
2275 }
2276 }
2277 else
2278 {
2279 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2280 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2281 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2282 pVM->hm.s.vmx.fVpid = false;
2283 }
2284 }
2285
2286 /*
2287 * Setup the handler for flushing tagged-TLBs.
2288 */
2289 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2290 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2291 else if (pVM->hm.s.fNestedPaging)
2292 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2293 else if (pVM->hm.s.vmx.fVpid)
2294 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2295 else
2296 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2297 return VINF_SUCCESS;
2298}
2299
2300
2301/**
2302 * Sets up pin-based VM-execution controls in the VMCS.
2303 *
2304 * @returns VBox status code.
2305 * @param pVM The cross context VM structure.
2306 * @param pVCpu The cross context virtual CPU structure.
2307 */
2308static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2309{
2310 AssertPtr(pVM);
2311 AssertPtr(pVCpu);
2312
2313 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2314 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2315
2316 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2317 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2318
2319 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2320 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2321
2322 /* Enable the VMX preemption timer. */
2323 if (pVM->hm.s.vmx.fUsePreemptTimer)
2324 {
2325 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2326 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2327 }
2328
2329#if 0
2330 /* Enable posted-interrupt processing. */
2331 if (pVM->hm.s.fPostedIntrs)
2332 {
2333 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2334 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2335 val |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2336 }
2337#endif
2338
2339 if ((val & zap) != val)
2340 {
2341 LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2342 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2343 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2344 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2345 }
2346
2347 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2348 AssertRCReturn(rc, rc);
2349
2350 pVCpu->hm.s.vmx.u32PinCtls = val;
2351 return rc;
2352}
2353
2354
2355/**
2356 * Sets up processor-based VM-execution controls in the VMCS.
2357 *
2358 * @returns VBox status code.
2359 * @param pVM The cross context VM structure.
2360 * @param pVCpu The cross context virtual CPU structure.
2361 */
2362static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2363{
2364 AssertPtr(pVM);
2365 AssertPtr(pVCpu);
2366
2367 int rc = VERR_INTERNAL_ERROR_5;
2368 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2369 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2370
2371 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2372 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2373 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2374 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2375 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2376 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2377 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2378
2379 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2380 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2381 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2382 {
2383 LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2384 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2385 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2386 }
2387
2388 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2389 if (!pVM->hm.s.fNestedPaging)
2390 {
2391 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2392 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2393 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2394 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2395 }
2396
2397 /* Use TPR shadowing if supported by the CPU. */
2398 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2399 {
2400 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2401 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2402 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2403 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2404 AssertRCReturn(rc, rc);
2405
2406 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2407 /* CR8 writes cause a VM-exit based on TPR threshold. */
2408 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2409 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2410 }
2411 else
2412 {
2413 /*
2414 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2415 * Set this control only for 64-bit guests.
2416 */
2417 if (pVM->hm.s.fAllow64BitGuests)
2418 {
2419 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2420 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2421 }
2422 }
2423
2424 /* Use MSR-bitmaps if supported by the CPU. */
2425 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2426 {
2427 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2428
2429 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2430 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2431 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2432 AssertRCReturn(rc, rc);
2433
2434 /*
2435 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2436 * automatically using dedicated fields in the VMCS.
2437 */
2438 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2439 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2440 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2441 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2442 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2443
2444#if HC_ARCH_BITS == 64
2445 /*
2446 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2447 */
2448 if (pVM->hm.s.fAllow64BitGuests)
2449 {
2450 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2451 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2452 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2453 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2454 }
2455#endif
2456 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2457 }
2458
2459 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2460 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2461 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2462
2463 if ((val & zap) != val)
2464 {
2465 LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2466 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2467 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2468 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2469 }
2470
2471 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2472 AssertRCReturn(rc, rc);
2473
2474 pVCpu->hm.s.vmx.u32ProcCtls = val;
2475
2476 /*
2477 * Secondary processor-based VM-execution controls.
2478 */
2479 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2480 {
2481 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2482 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2483
2484 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2485 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2486
2487 if (pVM->hm.s.fNestedPaging)
2488 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2489 else
2490 {
2491 /*
2492 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2493 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2494 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2495 */
2496 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2497 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2498 }
2499
2500 if (pVM->hm.s.vmx.fVpid)
2501 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2502
2503 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2504 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2505
2506#if 0
2507 if (pVM->hm.s.fVirtApicRegs)
2508 {
2509 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2510 val |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2511
2512 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2513 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2514 }
2515#endif
2516
2517 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2518 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2519 * done dynamically. */
2520 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2521 {
2522 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2523 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2524 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2525 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2526 AssertRCReturn(rc, rc);
2527 }
2528
2529 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2530 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2531
2532 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2533 && pVM->hm.s.vmx.cPleGapTicks
2534 && pVM->hm.s.vmx.cPleWindowTicks)
2535 {
2536 val |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2537
2538 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2539 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2540 AssertRCReturn(rc, rc);
2541 }
2542
2543 if ((val & zap) != val)
2544 {
2545 LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2546 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2547 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2548 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2549 }
2550
2551 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2552 AssertRCReturn(rc, rc);
2553
2554 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2555 }
2556 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2557 {
2558 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2559 "available\n"));
2560 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2561 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2562 }
2563
2564 return VINF_SUCCESS;
2565}
2566
2567
2568/**
2569 * Sets up miscellaneous (everything other than Pin & Processor-based
2570 * VM-execution) control fields in the VMCS.
2571 *
2572 * @returns VBox status code.
2573 * @param pVM The cross context VM structure.
2574 * @param pVCpu The cross context virtual CPU structure.
2575 */
2576static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2577{
2578 NOREF(pVM);
2579 AssertPtr(pVM);
2580 AssertPtr(pVCpu);
2581
2582 int rc = VERR_GENERAL_FAILURE;
2583
2584 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2585#if 0
2586 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2587 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2588 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2589
2590 /*
2591 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2592 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2593 * We thus use the exception bitmap to control it rather than use both.
2594 */
2595 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2596 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2597
2598 /** @todo Explore possibility of using IO-bitmaps. */
2599 /* All IO & IOIO instructions cause VM-exits. */
2600 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2601 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2602
2603 /* Initialize the MSR-bitmap area. */
2604 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2605 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2606 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2607 AssertRCReturn(rc, rc);
2608#endif
2609
2610 /* Setup MSR auto-load/store area. */
2611 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2612 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2613 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2614 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2615 AssertRCReturn(rc, rc);
2616
2617 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2618 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2619 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2620 AssertRCReturn(rc, rc);
2621
2622 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2623 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2624 AssertRCReturn(rc, rc);
2625
2626 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2627#if 0
2628 /* Setup debug controls */
2629 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2630 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2631 AssertRCReturn(rc, rc);
2632#endif
2633
2634 return rc;
2635}
2636
2637
2638/**
2639 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2640 *
2641 * @returns VBox status code.
2642 * @param pVM The cross context VM structure.
2643 * @param pVCpu The cross context virtual CPU structure.
2644 */
2645static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2646{
2647 AssertPtr(pVM);
2648 AssertPtr(pVCpu);
2649
2650 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2651
2652 /** @todo r=ramshankar: Shouldn't setting up \#UD intercepts be handled by
2653 * hmR0VmxLoadGuestXcptIntercepts()? Why do we check it here? */
2654 uint32_t u32XcptBitmap = pVCpu->hm.s.fGIMTrapXcptUD ? RT_BIT(X86_XCPT_UD) : 0;
2655
2656 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2657 u32XcptBitmap |= RT_BIT_32(X86_XCPT_AC);
2658
2659 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2660 and writes, and because recursive #DBs can cause the CPU hang, we must always
2661 intercept #DB. */
2662 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2663
2664 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2665 if (!pVM->hm.s.fNestedPaging)
2666 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2667
2668 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2669 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2670 AssertRCReturn(rc, rc);
2671 return rc;
2672}
2673
2674
2675/**
2676 * Sets up the initial guest-state mask. The guest-state mask is consulted
2677 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2678 * for the nested virtualization case (as it would cause a VM-exit).
2679 *
2680 * @param pVCpu The cross context virtual CPU structure.
2681 */
2682static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2683{
2684 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2685 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2686 return VINF_SUCCESS;
2687}
2688
2689
2690/**
2691 * Does per-VM VT-x initialization.
2692 *
2693 * @returns VBox status code.
2694 * @param pVM The cross context VM structure.
2695 */
2696VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2697{
2698 LogFlowFunc(("pVM=%p\n", pVM));
2699
2700 int rc = hmR0VmxStructsAlloc(pVM);
2701 if (RT_FAILURE(rc))
2702 {
2703 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2704 return rc;
2705 }
2706
2707 return VINF_SUCCESS;
2708}
2709
2710
2711/**
2712 * Does per-VM VT-x termination.
2713 *
2714 * @returns VBox status code.
2715 * @param pVM The cross context VM structure.
2716 */
2717VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2718{
2719 LogFlowFunc(("pVM=%p\n", pVM));
2720
2721#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2722 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2723 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2724#endif
2725 hmR0VmxStructsFree(pVM);
2726 return VINF_SUCCESS;
2727}
2728
2729
2730/**
2731 * Sets up the VM for execution under VT-x.
2732 * This function is only called once per-VM during initialization.
2733 *
2734 * @returns VBox status code.
2735 * @param pVM The cross context VM structure.
2736 */
2737VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2738{
2739 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2740 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2741
2742 LogFlowFunc(("pVM=%p\n", pVM));
2743
2744 /*
2745 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2746 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2747 */
2748 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2749 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2750 || !pVM->hm.s.vmx.pRealModeTSS))
2751 {
2752 LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2753 return VERR_INTERNAL_ERROR;
2754 }
2755
2756 /* Initialize these always, see hmR3InitFinalizeR0().*/
2757 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2758 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2759
2760 /* Setup the tagged-TLB flush handlers. */
2761 int rc = hmR0VmxSetupTaggedTlb(pVM);
2762 if (RT_FAILURE(rc))
2763 {
2764 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2765 return rc;
2766 }
2767
2768 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2769 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2770#if HC_ARCH_BITS == 64
2771 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2772 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2773 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2774 {
2775 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2776 }
2777#endif
2778
2779 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2780 RTCCUINTREG uHostCR4 = ASMGetCR4();
2781 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2782 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2783
2784 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2785 {
2786 PVMCPU pVCpu = &pVM->aCpus[i];
2787 AssertPtr(pVCpu);
2788 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2789
2790 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2791 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2792
2793 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2794 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2795 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2796
2797 /* Set revision dword at the beginning of the VMCS structure. */
2798 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2799
2800 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2801 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2802 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2803 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2804
2805 /* Load this VMCS as the current VMCS. */
2806 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2807 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2808 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2809
2810 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2811 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2812 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2813
2814 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2815 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2816 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2817
2818 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2819 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2820 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2821
2822 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2823 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2824 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2825
2826 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2827 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2828 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2829
2830#if HC_ARCH_BITS == 32
2831 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2832 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2833 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2834#endif
2835
2836 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2837 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2838 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2839 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2840
2841 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2842
2843 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2844 }
2845
2846 return VINF_SUCCESS;
2847}
2848
2849
2850/**
2851 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2852 * the VMCS.
2853 *
2854 * @returns VBox status code.
2855 * @param pVM The cross context VM structure.
2856 * @param pVCpu The cross context virtual CPU structure.
2857 */
2858DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2859{
2860 NOREF(pVM); NOREF(pVCpu);
2861
2862 RTCCUINTREG uReg = ASMGetCR0();
2863 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2864 AssertRCReturn(rc, rc);
2865
2866 uReg = ASMGetCR3();
2867 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2868 AssertRCReturn(rc, rc);
2869
2870 uReg = ASMGetCR4();
2871 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2872 AssertRCReturn(rc, rc);
2873 return rc;
2874}
2875
2876
2877#if HC_ARCH_BITS == 64
2878/**
2879 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2880 * requirements. See hmR0VmxSaveHostSegmentRegs().
2881 */
2882# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2883 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2884 { \
2885 bool fValidSelector = true; \
2886 if ((selValue) & X86_SEL_LDT) \
2887 { \
2888 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2889 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2890 } \
2891 if (fValidSelector) \
2892 { \
2893 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2894 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2895 } \
2896 (selValue) = 0; \
2897 }
2898#endif
2899
2900
2901/**
2902 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2903 * the host-state area in the VMCS.
2904 *
2905 * @returns VBox status code.
2906 * @param pVM The cross context VM structure.
2907 * @param pVCpu The cross context virtual CPU structure.
2908 */
2909DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2910{
2911 int rc = VERR_INTERNAL_ERROR_5;
2912
2913#if HC_ARCH_BITS == 64
2914 /*
2915 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2916 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2917 *
2918 * This apparently can happen (most likely the FPU changes), deal with it rather than asserting.
2919 * Was observed booting Solaris10u10 32-bit guest.
2920 */
2921 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2922 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2923 {
2924 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2925 pVCpu->idCpu));
2926 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2927 }
2928 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2929#else
2930 RT_NOREF(pVCpu);
2931#endif
2932
2933 /*
2934 * Host DS, ES, FS and GS segment registers.
2935 */
2936#if HC_ARCH_BITS == 64
2937 RTSEL uSelDS = ASMGetDS();
2938 RTSEL uSelES = ASMGetES();
2939 RTSEL uSelFS = ASMGetFS();
2940 RTSEL uSelGS = ASMGetGS();
2941#else
2942 RTSEL uSelDS = 0;
2943 RTSEL uSelES = 0;
2944 RTSEL uSelFS = 0;
2945 RTSEL uSelGS = 0;
2946#endif
2947
2948 /*
2949 * Host CS and SS segment registers.
2950 */
2951 RTSEL uSelCS = ASMGetCS();
2952 RTSEL uSelSS = ASMGetSS();
2953
2954 /*
2955 * Host TR segment register.
2956 */
2957 RTSEL uSelTR = ASMGetTR();
2958
2959#if HC_ARCH_BITS == 64
2960 /*
2961 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2962 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2963 */
2964 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2965 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2966 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2967 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2968# undef VMXLOCAL_ADJUST_HOST_SEG
2969#endif
2970
2971 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2972 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2973 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2974 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2975 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2976 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2977 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2978 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2979 Assert(uSelCS);
2980 Assert(uSelTR);
2981
2982 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2983#if 0
2984 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2985 Assert(uSelSS != 0);
2986#endif
2987
2988 /* Write these host selector fields into the host-state area in the VMCS. */
2989 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2990 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2991#if HC_ARCH_BITS == 64
2992 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2993 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2994 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2995 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2996#else
2997 NOREF(uSelDS);
2998 NOREF(uSelES);
2999 NOREF(uSelFS);
3000 NOREF(uSelGS);
3001#endif
3002 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3003 AssertRCReturn(rc, rc);
3004
3005 /*
3006 * Host GDTR and IDTR.
3007 */
3008 RTGDTR Gdtr;
3009 RTIDTR Idtr;
3010 RT_ZERO(Gdtr);
3011 RT_ZERO(Idtr);
3012 ASMGetGDTR(&Gdtr);
3013 ASMGetIDTR(&Idtr);
3014 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3015 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3016 AssertRCReturn(rc, rc);
3017
3018#if HC_ARCH_BITS == 64
3019 /*
3020 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3021 * maximum limit (0xffff) on every VM-exit.
3022 */
3023 if (Gdtr.cbGdt != 0xffff)
3024 {
3025 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3026 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3027 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3028 }
3029
3030 /*
3031 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3032 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3033 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3034 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3035 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3036 * hosts where we are pretty sure it won't cause trouble.
3037 */
3038# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3039 if (Idtr.cbIdt < 0x0fff)
3040# else
3041 if (Idtr.cbIdt != 0xffff)
3042# endif
3043 {
3044 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3045 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3046 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3047 }
3048#endif
3049
3050 /*
3051 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3052 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3053 */
3054 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3055 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3056 VERR_VMX_INVALID_HOST_STATE);
3057
3058 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3059#if HC_ARCH_BITS == 64
3060 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3061
3062 /*
3063 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3064 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3065 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3066 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3067 *
3068 * [1] See Intel spec. 3.5 "System Descriptor Types".
3069 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3070 */
3071 Assert(pDesc->System.u4Type == 11);
3072 if ( pDesc->System.u16LimitLow != 0x67
3073 || pDesc->System.u4LimitHigh)
3074 {
3075 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3076 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3077 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3078 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3079 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3080
3081 /* Store the GDTR here as we need it while restoring TR. */
3082 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3083 }
3084#else
3085 NOREF(pVM);
3086 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3087#endif
3088 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3089 AssertRCReturn(rc, rc);
3090
3091 /*
3092 * Host FS base and GS base.
3093 */
3094#if HC_ARCH_BITS == 64
3095 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3096 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3097 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3098 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3099 AssertRCReturn(rc, rc);
3100
3101 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3102 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3103 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3104 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3105 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3106#endif
3107 return rc;
3108}
3109
3110
3111/**
3112 * Saves certain host MSRs in the VM-exit MSR-load area and some in the
3113 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3114 * the host after every successful VM-exit.
3115 *
3116 * @returns VBox status code.
3117 * @param pVM The cross context VM structure.
3118 * @param pVCpu The cross context virtual CPU structure.
3119 *
3120 * @remarks No-long-jump zone!!!
3121 */
3122DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3123{
3124 NOREF(pVM);
3125
3126 AssertPtr(pVCpu);
3127 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3128
3129 /*
3130 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3131 * rather than swapping them on every VM-entry.
3132 */
3133 hmR0VmxLazySaveHostMsrs(pVCpu);
3134
3135 /*
3136 * Host Sysenter MSRs.
3137 */
3138 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3139#if HC_ARCH_BITS == 32
3140 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3141 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3142#else
3143 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3144 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3145#endif
3146 AssertRCReturn(rc, rc);
3147
3148 /*
3149 * Host EFER MSR.
3150 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3151 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3152 */
3153 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3154 {
3155 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3156 AssertRCReturn(rc, rc);
3157 }
3158
3159 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3160 * hmR0VmxLoadGuestExitCtls() !! */
3161
3162 return rc;
3163}
3164
3165
3166/**
3167 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3168 *
3169 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3170 * these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3171 * hmR0VMxLoadGuestEntryCtls().
3172 *
3173 * @returns true if we need to load guest EFER, false otherwise.
3174 * @param pVCpu The cross context virtual CPU structure.
3175 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3176 * out-of-sync. Make sure to update the required fields
3177 * before using them.
3178 *
3179 * @remarks Requires EFER, CR4.
3180 * @remarks No-long-jump zone!!!
3181 */
3182static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3183{
3184#ifdef HMVMX_ALWAYS_SWAP_EFER
3185 return true;
3186#endif
3187
3188#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3189 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3190 if (CPUMIsGuestInLongMode(pVCpu))
3191 return false;
3192#endif
3193
3194 PVM pVM = pVCpu->CTX_SUFF(pVM);
3195 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3196 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3197
3198 /*
3199 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3200 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3201 */
3202 if ( CPUMIsGuestInLongMode(pVCpu)
3203 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3204 {
3205 return true;
3206 }
3207
3208 /*
3209 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3210 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3211 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3212 */
3213 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3214 && (pMixedCtx->cr0 & X86_CR0_PG)
3215 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3216 {
3217 /* Assert that host is PAE capable. */
3218 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3219 return true;
3220 }
3221
3222 /** @todo Check the latest Intel spec. for any other bits,
3223 * like SMEP/SMAP? */
3224 return false;
3225}
3226
3227
3228/**
3229 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3230 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3231 * controls".
3232 *
3233 * @returns VBox status code.
3234 * @param pVCpu The cross context virtual CPU structure.
3235 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3236 * out-of-sync. Make sure to update the required fields
3237 * before using them.
3238 *
3239 * @remarks Requires EFER.
3240 * @remarks No-long-jump zone!!!
3241 */
3242DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3243{
3244 int rc = VINF_SUCCESS;
3245 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3246 {
3247 PVM pVM = pVCpu->CTX_SUFF(pVM);
3248 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3249 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3250
3251 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3252 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3253
3254 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3255 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3256 {
3257 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3258 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3259 }
3260 else
3261 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3262
3263 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3264 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3265 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3266 {
3267 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3268 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3269 }
3270
3271 /*
3272 * The following should -not- be set (since we're not in SMM mode):
3273 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3274 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3275 */
3276
3277 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3278 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3279
3280 if ((val & zap) != val)
3281 {
3282 LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3283 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3284 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3285 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3286 }
3287
3288 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3289 AssertRCReturn(rc, rc);
3290
3291 pVCpu->hm.s.vmx.u32EntryCtls = val;
3292 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3293 }
3294 return rc;
3295}
3296
3297
3298/**
3299 * Sets up the VM-exit controls in the VMCS.
3300 *
3301 * @returns VBox status code.
3302 * @param pVCpu The cross context virtual CPU structure.
3303 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3304 * out-of-sync. Make sure to update the required fields
3305 * before using them.
3306 *
3307 * @remarks Requires EFER.
3308 */
3309DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3310{
3311 NOREF(pMixedCtx);
3312
3313 int rc = VINF_SUCCESS;
3314 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3315 {
3316 PVM pVM = pVCpu->CTX_SUFF(pVM);
3317 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3318 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3319
3320 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3321 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3322
3323 /*
3324 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3325 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3326 */
3327#if HC_ARCH_BITS == 64
3328 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3329 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3330#else
3331 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3332 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3333 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3334 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3335 {
3336 /* The switcher returns to long mode, EFER is managed by the switcher. */
3337 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3338 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3339 }
3340 else
3341 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3342#endif
3343
3344 /* If the newer VMCS fields for managing EFER exists, use it. */
3345 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3346 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3347 {
3348 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3349 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3350 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3351 }
3352
3353 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3354 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3355
3356 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3357 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3358 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3359
3360 if ( pVM->hm.s.vmx.fUsePreemptTimer
3361 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3362 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3363
3364 if ((val & zap) != val)
3365 {
3366 LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3367 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3368 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3369 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3370 }
3371
3372 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3373 AssertRCReturn(rc, rc);
3374
3375 pVCpu->hm.s.vmx.u32ExitCtls = val;
3376 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3377 }
3378 return rc;
3379}
3380
3381
3382/**
3383 * Sets the TPR threshold in the VMCS.
3384 *
3385 * @returns VBox status code.
3386 * @param pVCpu The cross context virtual CPU structure.
3387 * @param u32TprThreshold The TPR threshold (task-priority class only).
3388 */
3389DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3390{
3391 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3392 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3393 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3394}
3395
3396
3397/**
3398 * Loads the guest APIC and related state.
3399 *
3400 * @returns VBox status code.
3401 * @param pVCpu The cross context virtual CPU structure.
3402 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3403 * out-of-sync. Make sure to update the required fields
3404 * before using them.
3405 */
3406DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3407{
3408 NOREF(pMixedCtx);
3409
3410 int rc = VINF_SUCCESS;
3411 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3412 {
3413 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3414 && APICIsEnabled(pVCpu))
3415 {
3416 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3417 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3418 {
3419 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3420
3421 bool fPendingIntr = false;
3422 uint8_t u8Tpr = 0;
3423 uint8_t u8PendingIntr = 0;
3424 rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3425 AssertRCReturn(rc, rc);
3426
3427 /*
3428 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3429 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3430 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3431 * the interrupt when we VM-exit for other reasons.
3432 */
3433 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3434 uint32_t u32TprThreshold = 0;
3435 if (fPendingIntr)
3436 {
3437 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3438 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3439 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3440 if (u8PendingPriority <= u8TprPriority)
3441 u32TprThreshold = u8PendingPriority;
3442 else
3443 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3444 }
3445
3446 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3447 AssertRCReturn(rc, rc);
3448 }
3449
3450#ifndef IEM_VERIFICATION_MODE_FULL
3451 /* Setup the Virtualized APIC accesses. */
3452 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
3453 {
3454 uint64_t u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
3455 if (u64MsrApicBase != pVCpu->hm.s.vmx.u64MsrApicBase)
3456 {
3457 PVM pVM = pVCpu->CTX_SUFF(pVM);
3458 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
3459 RTGCPHYS GCPhysApicBase;
3460 GCPhysApicBase = u64MsrApicBase;
3461 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3462
3463 /* Unalias any existing mapping. */
3464 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
3465 AssertRCReturn(rc, rc);
3466
3467 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
3468 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGp\n", GCPhysApicBase));
3469 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
3470 AssertRCReturn(rc, rc);
3471
3472 /* Update VMX's cache of the APIC base. */
3473 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
3474 }
3475 }
3476#endif /* !IEM_VERIFICATION_MODE_FULL */
3477 }
3478 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3479 }
3480
3481 return rc;
3482}
3483
3484
3485/**
3486 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3487 *
3488 * @returns Guest's interruptibility-state.
3489 * @param pVCpu The cross context virtual CPU structure.
3490 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3491 * out-of-sync. Make sure to update the required fields
3492 * before using them.
3493 *
3494 * @remarks No-long-jump zone!!!
3495 */
3496DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3497{
3498 /*
3499 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3500 */
3501 uint32_t uIntrState = 0;
3502 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3503 {
3504 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3505 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3506 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3507 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3508 {
3509 if (pMixedCtx->eflags.Bits.u1IF)
3510 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3511 else
3512 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3513 }
3514 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3515 {
3516 /*
3517 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3518 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3519 */
3520 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3521 }
3522 }
3523
3524 /*
3525 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3526 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3527 * setting this would block host-NMIs and IRET will not clear the blocking.
3528 *
3529 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3530 */
3531 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3532 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3533 {
3534 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3535 }
3536
3537 return uIntrState;
3538}
3539
3540
3541/**
3542 * Loads the guest's interruptibility-state into the guest-state area in the
3543 * VMCS.
3544 *
3545 * @returns VBox status code.
3546 * @param pVCpu The cross context virtual CPU structure.
3547 * @param uIntrState The interruptibility-state to set.
3548 */
3549static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3550{
3551 NOREF(pVCpu);
3552 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3553 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3554 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3555 AssertRC(rc);
3556 return rc;
3557}
3558
3559
3560/**
3561 * Loads the exception intercepts required for guest execution in the VMCS.
3562 *
3563 * @returns VBox status code.
3564 * @param pVCpu The cross context virtual CPU structure.
3565 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3566 * out-of-sync. Make sure to update the required fields
3567 * before using them.
3568 */
3569static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3570{
3571 NOREF(pMixedCtx);
3572 int rc = VINF_SUCCESS;
3573 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3574 {
3575 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3576 if (pVCpu->hm.s.fGIMTrapXcptUD)
3577 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3578#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3579 else
3580 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3581#endif
3582
3583 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3584 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3585
3586 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3587 AssertRCReturn(rc, rc);
3588
3589 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3590 Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3591 pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3592 }
3593 return rc;
3594}
3595
3596
3597/**
3598 * Loads the guest's RIP into the guest-state area in the VMCS.
3599 *
3600 * @returns VBox status code.
3601 * @param pVCpu The cross context virtual CPU structure.
3602 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3603 * out-of-sync. Make sure to update the required fields
3604 * before using them.
3605 *
3606 * @remarks No-long-jump zone!!!
3607 */
3608static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3609{
3610 int rc = VINF_SUCCESS;
3611 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3612 {
3613 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3614 AssertRCReturn(rc, rc);
3615
3616 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3617 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3618 HMCPU_CF_VALUE(pVCpu)));
3619 }
3620 return rc;
3621}
3622
3623
3624/**
3625 * Loads the guest's RSP into the guest-state area in the VMCS.
3626 *
3627 * @returns VBox status code.
3628 * @param pVCpu The cross context virtual CPU structure.
3629 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3630 * out-of-sync. Make sure to update the required fields
3631 * before using them.
3632 *
3633 * @remarks No-long-jump zone!!!
3634 */
3635static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3636{
3637 int rc = VINF_SUCCESS;
3638 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3639 {
3640 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3641 AssertRCReturn(rc, rc);
3642
3643 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3644 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3645 }
3646 return rc;
3647}
3648
3649
3650/**
3651 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3652 *
3653 * @returns VBox status code.
3654 * @param pVCpu The cross context virtual CPU structure.
3655 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3656 * out-of-sync. Make sure to update the required fields
3657 * before using them.
3658 *
3659 * @remarks No-long-jump zone!!!
3660 */
3661static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3662{
3663 int rc = VINF_SUCCESS;
3664 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3665 {
3666 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3667 Let us assert it as such and use 32-bit VMWRITE. */
3668 Assert(!(pMixedCtx->rflags.u64 >> 32));
3669 X86EFLAGS Eflags = pMixedCtx->eflags;
3670 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3671 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3672 * These will never be cleared/set, unless some other part of the VMM
3673 * code is buggy - in which case we're better of finding and fixing
3674 * those bugs than hiding them. */
3675 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3676 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3677 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3678 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3679
3680 /*
3681 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3682 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3683 */
3684 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3685 {
3686 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3687 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3688 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3689 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3690 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3691 }
3692
3693 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3694 AssertRCReturn(rc, rc);
3695
3696 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3697 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3698 }
3699 return rc;
3700}
3701
3702
3703/**
3704 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3705 *
3706 * @returns VBox status code.
3707 * @param pVCpu The cross context virtual CPU structure.
3708 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3709 * out-of-sync. Make sure to update the required fields
3710 * before using them.
3711 *
3712 * @remarks No-long-jump zone!!!
3713 */
3714DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3715{
3716 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3717 rc |= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3718 rc |= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3719 AssertRCReturn(rc, rc);
3720 return rc;
3721}
3722
3723
3724/**
3725 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3726 * CR0 is partially shared with the host and we have to consider the FPU bits.
3727 *
3728 * @returns VBox status code.
3729 * @param pVCpu The cross context virtual CPU structure.
3730 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3731 * out-of-sync. Make sure to update the required fields
3732 * before using them.
3733 *
3734 * @remarks No-long-jump zone!!!
3735 */
3736static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3737{
3738 /*
3739 * Guest CR0.
3740 * Guest FPU.
3741 */
3742 int rc = VINF_SUCCESS;
3743 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3744 {
3745 Assert(!(pMixedCtx->cr0 >> 32));
3746 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3747 PVM pVM = pVCpu->CTX_SUFF(pVM);
3748
3749 /* The guest's view (read access) of its CR0 is unblemished. */
3750 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3751 AssertRCReturn(rc, rc);
3752 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3753
3754 /* Setup VT-x's view of the guest CR0. */
3755 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3756 if (pVM->hm.s.fNestedPaging)
3757 {
3758 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3759 {
3760 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3761 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3762 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3763 }
3764 else
3765 {
3766 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3767 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3768 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3769 }
3770
3771 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3772 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3773 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3774
3775 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3776 AssertRCReturn(rc, rc);
3777 }
3778 else
3779 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3780
3781 /*
3782 * Guest FPU bits.
3783 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3784 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3785 */
3786 u32GuestCR0 |= X86_CR0_NE;
3787 bool fInterceptNM = false;
3788 if (CPUMIsGuestFPUStateActive(pVCpu))
3789 {
3790 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3791 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3792 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3793 }
3794 else
3795 {
3796 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3797 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3798 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3799 }
3800
3801 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3802 bool fInterceptMF = false;
3803 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3804 fInterceptMF = true;
3805
3806 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3807 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3808 {
3809 Assert(PDMVmmDevHeapIsEnabled(pVM));
3810 Assert(pVM->hm.s.vmx.pRealModeTSS);
3811 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3812 fInterceptNM = true;
3813 fInterceptMF = true;
3814 }
3815 else
3816 {
3817 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3818 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3819 }
3820 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3821
3822 if (fInterceptNM)
3823 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3824 else
3825 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3826
3827 if (fInterceptMF)
3828 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3829 else
3830 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3831
3832 /* Additional intercepts for debugging, define these yourself explicitly. */
3833#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3834 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3835 | RT_BIT(X86_XCPT_BP)
3836 | RT_BIT(X86_XCPT_DE)
3837 | RT_BIT(X86_XCPT_NM)
3838 | RT_BIT(X86_XCPT_TS)
3839 | RT_BIT(X86_XCPT_UD)
3840 | RT_BIT(X86_XCPT_NP)
3841 | RT_BIT(X86_XCPT_SS)
3842 | RT_BIT(X86_XCPT_GP)
3843 | RT_BIT(X86_XCPT_PF)
3844 | RT_BIT(X86_XCPT_MF)
3845 ;
3846#elif defined(HMVMX_ALWAYS_TRAP_PF)
3847 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3848#endif
3849
3850 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3851
3852 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3853 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3854 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3855 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3856 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3857 else
3858 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3859
3860 u32GuestCR0 |= uSetCR0;
3861 u32GuestCR0 &= uZapCR0;
3862 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3863
3864 /* Write VT-x's view of the guest CR0 into the VMCS. */
3865 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3866 AssertRCReturn(rc, rc);
3867 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3868 uZapCR0));
3869
3870 /*
3871 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3872 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3873 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3874 */
3875 uint32_t u32CR0Mask = 0;
3876 u32CR0Mask = X86_CR0_PE
3877 | X86_CR0_NE
3878 | X86_CR0_WP
3879 | X86_CR0_PG
3880 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3881 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3882 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3883
3884 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3885 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3886 * and @bugref{6944}. */
3887#if 0
3888 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3889 u32CR0Mask &= ~X86_CR0_PE;
3890#endif
3891 if (pVM->hm.s.fNestedPaging)
3892 u32CR0Mask &= ~X86_CR0_WP;
3893
3894 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3895 if (fInterceptNM)
3896 {
3897 u32CR0Mask |= X86_CR0_TS
3898 | X86_CR0_MP;
3899 }
3900
3901 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3902 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3903 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3904 AssertRCReturn(rc, rc);
3905 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3906
3907 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3908 }
3909 return rc;
3910}
3911
3912
3913/**
3914 * Loads the guest control registers (CR3, CR4) into the guest-state area
3915 * in the VMCS.
3916 *
3917 * @returns VBox strict status code.
3918 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3919 * without unrestricted guest access and the VMMDev is not presently
3920 * mapped (e.g. EFI32).
3921 *
3922 * @param pVCpu The cross context virtual CPU structure.
3923 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3924 * out-of-sync. Make sure to update the required fields
3925 * before using them.
3926 *
3927 * @remarks No-long-jump zone!!!
3928 */
3929static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3930{
3931 int rc = VINF_SUCCESS;
3932 PVM pVM = pVCpu->CTX_SUFF(pVM);
3933
3934 /*
3935 * Guest CR2.
3936 * It's always loaded in the assembler code. Nothing to do here.
3937 */
3938
3939 /*
3940 * Guest CR3.
3941 */
3942 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3943 {
3944 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3945 if (pVM->hm.s.fNestedPaging)
3946 {
3947 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3948
3949 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3950 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3951 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3952 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3953
3954 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3955 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3956 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3957
3958 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3959 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3960 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3961 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3962 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3963 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3964 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3965
3966 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3967 AssertRCReturn(rc, rc);
3968 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3969
3970 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3971 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3972 {
3973 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3974 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3975 {
3976 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3977 AssertRCReturn(rc, rc);
3978 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3979 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3980 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3981 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3982 AssertRCReturn(rc, rc);
3983 }
3984
3985 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3986 have Unrestricted Execution to handle the guest when it's not using paging. */
3987 GCPhysGuestCR3 = pMixedCtx->cr3;
3988 }
3989 else
3990 {
3991 /*
3992 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3993 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3994 * EPT takes care of translating it to host-physical addresses.
3995 */
3996 RTGCPHYS GCPhys;
3997 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3998
3999 /* We obtain it here every time as the guest could have relocated this PCI region. */
4000 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
4001 if (RT_SUCCESS(rc))
4002 { /* likely */ }
4003 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4004 {
4005 Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
4006 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4007 }
4008 else
4009 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4010
4011 GCPhysGuestCR3 = GCPhys;
4012 }
4013
4014 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
4015 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4016 }
4017 else
4018 {
4019 /* Non-nested paging case, just use the hypervisor's CR3. */
4020 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4021
4022 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
4023 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4024 }
4025 AssertRCReturn(rc, rc);
4026
4027 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
4028 }
4029
4030 /*
4031 * Guest CR4.
4032 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
4033 */
4034 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
4035 {
4036 Assert(!(pMixedCtx->cr4 >> 32));
4037 uint32_t u32GuestCR4 = pMixedCtx->cr4;
4038
4039 /* The guest's view of its CR4 is unblemished. */
4040 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4041 AssertRCReturn(rc, rc);
4042 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4043
4044 /* Setup VT-x's view of the guest CR4. */
4045 /*
4046 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4047 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4048 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4049 */
4050 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4051 {
4052 Assert(pVM->hm.s.vmx.pRealModeTSS);
4053 Assert(PDMVmmDevHeapIsEnabled(pVM));
4054 u32GuestCR4 &= ~X86_CR4_VME;
4055 }
4056
4057 if (pVM->hm.s.fNestedPaging)
4058 {
4059 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4060 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4061 {
4062 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4063 u32GuestCR4 |= X86_CR4_PSE;
4064 /* Our identity mapping is a 32-bit page directory. */
4065 u32GuestCR4 &= ~X86_CR4_PAE;
4066 }
4067 /* else use guest CR4.*/
4068 }
4069 else
4070 {
4071 /*
4072 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4073 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4074 */
4075 switch (pVCpu->hm.s.enmShadowMode)
4076 {
4077 case PGMMODE_REAL: /* Real-mode. */
4078 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4079 case PGMMODE_32_BIT: /* 32-bit paging. */
4080 {
4081 u32GuestCR4 &= ~X86_CR4_PAE;
4082 break;
4083 }
4084
4085 case PGMMODE_PAE: /* PAE paging. */
4086 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4087 {
4088 u32GuestCR4 |= X86_CR4_PAE;
4089 break;
4090 }
4091
4092 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4093 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4094#ifdef VBOX_ENABLE_64_BITS_GUESTS
4095 break;
4096#endif
4097 default:
4098 AssertFailed();
4099 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4100 }
4101 }
4102
4103 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4104 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4105 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4106 u32GuestCR4 |= uSetCR4;
4107 u32GuestCR4 &= uZapCR4;
4108
4109 /* Write VT-x's view of the guest CR4 into the VMCS. */
4110 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4111 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4112 AssertRCReturn(rc, rc);
4113
4114 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4115 uint32_t u32CR4Mask = X86_CR4_VME
4116 | X86_CR4_PAE
4117 | X86_CR4_PGE
4118 | X86_CR4_PSE
4119 | X86_CR4_VMXE;
4120 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4121 u32CR4Mask |= X86_CR4_OSXSAVE;
4122 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4123 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4124 AssertRCReturn(rc, rc);
4125
4126 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4127 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4128
4129 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4130 }
4131 return rc;
4132}
4133
4134
4135/**
4136 * Loads the guest debug registers into the guest-state area in the VMCS.
4137 *
4138 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4139 *
4140 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4141 *
4142 * @returns VBox status code.
4143 * @param pVCpu The cross context virtual CPU structure.
4144 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4145 * out-of-sync. Make sure to update the required fields
4146 * before using them.
4147 *
4148 * @remarks No-long-jump zone!!!
4149 */
4150static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4151{
4152 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4153 return VINF_SUCCESS;
4154
4155#ifdef VBOX_STRICT
4156 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4157 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4158 {
4159 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4160 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4161 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4162 }
4163#endif
4164
4165 int rc;
4166 PVM pVM = pVCpu->CTX_SUFF(pVM);
4167 bool fSteppingDB = false;
4168 bool fInterceptMovDRx = false;
4169 if (pVCpu->hm.s.fSingleInstruction)
4170 {
4171 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4172 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4173 {
4174 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4175 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4176 AssertRCReturn(rc, rc);
4177 Assert(fSteppingDB == false);
4178 }
4179 else
4180 {
4181 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4182 pVCpu->hm.s.fClearTrapFlag = true;
4183 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4184 fSteppingDB = true;
4185 }
4186 }
4187
4188 if ( fSteppingDB
4189 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4190 {
4191 /*
4192 * Use the combined guest and host DRx values found in the hypervisor
4193 * register set because the debugger has breakpoints active or someone
4194 * is single stepping on the host side without a monitor trap flag.
4195 *
4196 * Note! DBGF expects a clean DR6 state before executing guest code.
4197 */
4198#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4199 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4200 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4201 {
4202 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4203 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4204 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4205 }
4206 else
4207#endif
4208 if (!CPUMIsHyperDebugStateActive(pVCpu))
4209 {
4210 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4211 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4212 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4213 }
4214
4215 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4216 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4217 AssertRCReturn(rc, rc);
4218
4219 pVCpu->hm.s.fUsingHyperDR7 = true;
4220 fInterceptMovDRx = true;
4221 }
4222 else
4223 {
4224 /*
4225 * If the guest has enabled debug registers, we need to load them prior to
4226 * executing guest code so they'll trigger at the right time.
4227 */
4228 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4229 {
4230#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4231 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4232 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4233 {
4234 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4235 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4236 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4237 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4238 }
4239 else
4240#endif
4241 if (!CPUMIsGuestDebugStateActive(pVCpu))
4242 {
4243 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4244 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4245 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4246 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4247 }
4248 Assert(!fInterceptMovDRx);
4249 }
4250 /*
4251 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4252 * must intercept #DB in order to maintain a correct DR6 guest value, and
4253 * because we need to intercept it to prevent nested #DBs from hanging the
4254 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4255 */
4256#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4257 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4258 && !CPUMIsGuestDebugStateActive(pVCpu))
4259#else
4260 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4261#endif
4262 {
4263 fInterceptMovDRx = true;
4264 }
4265
4266 /* Update guest DR7. */
4267 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4268 AssertRCReturn(rc, rc);
4269
4270 pVCpu->hm.s.fUsingHyperDR7 = false;
4271 }
4272
4273 /*
4274 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4275 */
4276 if (fInterceptMovDRx)
4277 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4278 else
4279 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4280 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4281 AssertRCReturn(rc, rc);
4282
4283 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4284 return VINF_SUCCESS;
4285}
4286
4287
4288#ifdef VBOX_STRICT
4289/**
4290 * Strict function to validate segment registers.
4291 *
4292 * @remarks ASSUMES CR0 is up to date.
4293 */
4294static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4295{
4296 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4297 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4298 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4299 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4300 && ( !CPUMIsGuestInRealModeEx(pCtx)
4301 && !CPUMIsGuestInV86ModeEx(pCtx)))
4302 {
4303 /* Protected mode checks */
4304 /* CS */
4305 Assert(pCtx->cs.Attr.n.u1Present);
4306 Assert(!(pCtx->cs.Attr.u & 0xf00));
4307 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4308 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4309 || !(pCtx->cs.Attr.n.u1Granularity));
4310 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4311 || (pCtx->cs.Attr.n.u1Granularity));
4312 /* CS cannot be loaded with NULL in protected mode. */
4313 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4314 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4315 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4316 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4317 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4318 else
4319 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4320 /* SS */
4321 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4322 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4323 if ( !(pCtx->cr0 & X86_CR0_PE)
4324 || pCtx->cs.Attr.n.u4Type == 3)
4325 {
4326 Assert(!pCtx->ss.Attr.n.u2Dpl);
4327 }
4328 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4329 {
4330 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4331 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4332 Assert(pCtx->ss.Attr.n.u1Present);
4333 Assert(!(pCtx->ss.Attr.u & 0xf00));
4334 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4335 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4336 || !(pCtx->ss.Attr.n.u1Granularity));
4337 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4338 || (pCtx->ss.Attr.n.u1Granularity));
4339 }
4340 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4341 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4342 {
4343 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4344 Assert(pCtx->ds.Attr.n.u1Present);
4345 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4346 Assert(!(pCtx->ds.Attr.u & 0xf00));
4347 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4348 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4349 || !(pCtx->ds.Attr.n.u1Granularity));
4350 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4351 || (pCtx->ds.Attr.n.u1Granularity));
4352 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4353 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4354 }
4355 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4356 {
4357 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4358 Assert(pCtx->es.Attr.n.u1Present);
4359 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4360 Assert(!(pCtx->es.Attr.u & 0xf00));
4361 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4362 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4363 || !(pCtx->es.Attr.n.u1Granularity));
4364 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4365 || (pCtx->es.Attr.n.u1Granularity));
4366 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4367 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4368 }
4369 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4370 {
4371 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4372 Assert(pCtx->fs.Attr.n.u1Present);
4373 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4374 Assert(!(pCtx->fs.Attr.u & 0xf00));
4375 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4376 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4377 || !(pCtx->fs.Attr.n.u1Granularity));
4378 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4379 || (pCtx->fs.Attr.n.u1Granularity));
4380 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4381 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4382 }
4383 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4384 {
4385 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4386 Assert(pCtx->gs.Attr.n.u1Present);
4387 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4388 Assert(!(pCtx->gs.Attr.u & 0xf00));
4389 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4390 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4391 || !(pCtx->gs.Attr.n.u1Granularity));
4392 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4393 || (pCtx->gs.Attr.n.u1Granularity));
4394 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4395 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4396 }
4397 /* 64-bit capable CPUs. */
4398# if HC_ARCH_BITS == 64
4399 Assert(!(pCtx->cs.u64Base >> 32));
4400 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4401 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4402 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4403# endif
4404 }
4405 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4406 || ( CPUMIsGuestInRealModeEx(pCtx)
4407 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4408 {
4409 /* Real and v86 mode checks. */
4410 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4411 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4412 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4413 {
4414 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4415 }
4416 else
4417 {
4418 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4419 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4420 }
4421
4422 /* CS */
4423 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4424 Assert(pCtx->cs.u32Limit == 0xffff);
4425 Assert(u32CSAttr == 0xf3);
4426 /* SS */
4427 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4428 Assert(pCtx->ss.u32Limit == 0xffff);
4429 Assert(u32SSAttr == 0xf3);
4430 /* DS */
4431 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4432 Assert(pCtx->ds.u32Limit == 0xffff);
4433 Assert(u32DSAttr == 0xf3);
4434 /* ES */
4435 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4436 Assert(pCtx->es.u32Limit == 0xffff);
4437 Assert(u32ESAttr == 0xf3);
4438 /* FS */
4439 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4440 Assert(pCtx->fs.u32Limit == 0xffff);
4441 Assert(u32FSAttr == 0xf3);
4442 /* GS */
4443 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4444 Assert(pCtx->gs.u32Limit == 0xffff);
4445 Assert(u32GSAttr == 0xf3);
4446 /* 64-bit capable CPUs. */
4447# if HC_ARCH_BITS == 64
4448 Assert(!(pCtx->cs.u64Base >> 32));
4449 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4450 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4451 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4452# endif
4453 }
4454}
4455#endif /* VBOX_STRICT */
4456
4457
4458/**
4459 * Writes a guest segment register into the guest-state area in the VMCS.
4460 *
4461 * @returns VBox status code.
4462 * @param pVCpu The cross context virtual CPU structure.
4463 * @param idxSel Index of the selector in the VMCS.
4464 * @param idxLimit Index of the segment limit in the VMCS.
4465 * @param idxBase Index of the segment base in the VMCS.
4466 * @param idxAccess Index of the access rights of the segment in the VMCS.
4467 * @param pSelReg Pointer to the segment selector.
4468 *
4469 * @remarks No-long-jump zone!!!
4470 */
4471static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4472 uint32_t idxAccess, PCPUMSELREG pSelReg)
4473{
4474 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4475 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4476 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4477 AssertRCReturn(rc, rc);
4478
4479 uint32_t u32Access = pSelReg->Attr.u;
4480 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4481 {
4482 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4483 u32Access = 0xf3;
4484 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4485 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4486 }
4487 else
4488 {
4489 /*
4490 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4491 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4492 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4493 * loaded in protected-mode have their attribute as 0.
4494 */
4495 if (!u32Access)
4496 u32Access = X86DESCATTR_UNUSABLE;
4497 }
4498
4499 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4500 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4501 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4502
4503 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4504 AssertRCReturn(rc, rc);
4505 return rc;
4506}
4507
4508
4509/**
4510 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4511 * into the guest-state area in the VMCS.
4512 *
4513 * @returns VBox status code.
4514 * @param pVCpu The cross context virtual CPU structure.
4515 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4516 * out-of-sync. Make sure to update the required fields
4517 * before using them.
4518 *
4519 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4520 * @remarks No-long-jump zone!!!
4521 */
4522static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4523{
4524 int rc = VERR_INTERNAL_ERROR_5;
4525 PVM pVM = pVCpu->CTX_SUFF(pVM);
4526
4527 /*
4528 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4529 */
4530 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4531 {
4532 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4533 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4534 {
4535 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4536 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4537 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4538 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4539 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4540 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4541 }
4542
4543#ifdef VBOX_WITH_REM
4544 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4545 {
4546 Assert(pVM->hm.s.vmx.pRealModeTSS);
4547 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4548 if ( pVCpu->hm.s.vmx.fWasInRealMode
4549 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4550 {
4551 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4552 in real-mode (e.g. OpenBSD 4.0) */
4553 REMFlushTBs(pVM);
4554 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4555 pVCpu->hm.s.vmx.fWasInRealMode = false;
4556 }
4557 }
4558#endif
4559 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4560 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4561 AssertRCReturn(rc, rc);
4562 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4563 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4564 AssertRCReturn(rc, rc);
4565 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4566 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4567 AssertRCReturn(rc, rc);
4568 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4569 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4570 AssertRCReturn(rc, rc);
4571 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4572 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4573 AssertRCReturn(rc, rc);
4574 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4575 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4576 AssertRCReturn(rc, rc);
4577
4578#ifdef VBOX_STRICT
4579 /* Validate. */
4580 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4581#endif
4582
4583 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4584 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4585 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4586 }
4587
4588 /*
4589 * Guest TR.
4590 */
4591 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4592 {
4593 /*
4594 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4595 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4596 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4597 */
4598 uint16_t u16Sel = 0;
4599 uint32_t u32Limit = 0;
4600 uint64_t u64Base = 0;
4601 uint32_t u32AccessRights = 0;
4602
4603 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4604 {
4605 u16Sel = pMixedCtx->tr.Sel;
4606 u32Limit = pMixedCtx->tr.u32Limit;
4607 u64Base = pMixedCtx->tr.u64Base;
4608 u32AccessRights = pMixedCtx->tr.Attr.u;
4609 }
4610 else
4611 {
4612 Assert(pVM->hm.s.vmx.pRealModeTSS);
4613 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4614
4615 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4616 RTGCPHYS GCPhys;
4617 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4618 AssertRCReturn(rc, rc);
4619
4620 X86DESCATTR DescAttr;
4621 DescAttr.u = 0;
4622 DescAttr.n.u1Present = 1;
4623 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4624
4625 u16Sel = 0;
4626 u32Limit = HM_VTX_TSS_SIZE;
4627 u64Base = GCPhys; /* in real-mode phys = virt. */
4628 u32AccessRights = DescAttr.u;
4629 }
4630
4631 /* Validate. */
4632 Assert(!(u16Sel & RT_BIT(2)));
4633 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4634 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4635 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4636 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4637 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4638 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4639 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4640 Assert( (u32Limit & 0xfff) == 0xfff
4641 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4642 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4643 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4644
4645 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4646 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4647 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4648 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4649 AssertRCReturn(rc, rc);
4650
4651 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4652 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4653 }
4654
4655 /*
4656 * Guest GDTR.
4657 */
4658 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4659 {
4660 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4661 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4662 AssertRCReturn(rc, rc);
4663
4664 /* Validate. */
4665 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4666
4667 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4668 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4669 }
4670
4671 /*
4672 * Guest LDTR.
4673 */
4674 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4675 {
4676 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4677 uint32_t u32Access = 0;
4678 if (!pMixedCtx->ldtr.Attr.u)
4679 u32Access = X86DESCATTR_UNUSABLE;
4680 else
4681 u32Access = pMixedCtx->ldtr.Attr.u;
4682
4683 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4684 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4685 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4686 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4687 AssertRCReturn(rc, rc);
4688
4689 /* Validate. */
4690 if (!(u32Access & X86DESCATTR_UNUSABLE))
4691 {
4692 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4693 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4694 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4695 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4696 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4697 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4698 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4699 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4700 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4701 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4702 }
4703
4704 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4705 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4706 }
4707
4708 /*
4709 * Guest IDTR.
4710 */
4711 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4712 {
4713 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4714 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4715 AssertRCReturn(rc, rc);
4716
4717 /* Validate. */
4718 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4719
4720 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4721 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4722 }
4723
4724 return VINF_SUCCESS;
4725}
4726
4727
4728/**
4729 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4730 * areas.
4731 *
4732 * These MSRs will automatically be loaded to the host CPU on every successful
4733 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4734 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4735 * -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4736 *
4737 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4738 *
4739 * @returns VBox status code.
4740 * @param pVCpu The cross context virtual CPU structure.
4741 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4742 * out-of-sync. Make sure to update the required fields
4743 * before using them.
4744 *
4745 * @remarks No-long-jump zone!!!
4746 */
4747static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4748{
4749 AssertPtr(pVCpu);
4750 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4751
4752 /*
4753 * MSRs that we use the auto-load/store MSR area in the VMCS.
4754 */
4755 PVM pVM = pVCpu->CTX_SUFF(pVM);
4756 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4757 {
4758 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4759#if HC_ARCH_BITS == 32
4760 if (pVM->hm.s.fAllow64BitGuests)
4761 {
4762 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4763 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4764 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4765 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4766 AssertRCReturn(rc, rc);
4767# ifdef LOG_ENABLED
4768 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4769 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4770 {
4771 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4772 pMsr->u64Value));
4773 }
4774# endif
4775 }
4776#endif
4777 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4778 }
4779
4780 /*
4781 * Guest Sysenter MSRs.
4782 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4783 * VM-exits on WRMSRs for these MSRs.
4784 */
4785 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4786 {
4787 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4788 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4789 }
4790
4791 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4792 {
4793 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4794 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4795 }
4796
4797 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4798 {
4799 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4800 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4801 }
4802
4803 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4804 {
4805 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4806 {
4807 /*
4808 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4809 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4810 */
4811 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4812 {
4813 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4814 AssertRCReturn(rc,rc);
4815 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4816 }
4817 else
4818 {
4819 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4820 NULL /* pfAddedAndUpdated */);
4821 AssertRCReturn(rc, rc);
4822
4823 /* We need to intercept reads too, see @bugref{7386#c16}. */
4824 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4825 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4826 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4827 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4828 }
4829 }
4830 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4831 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4832 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4833 }
4834
4835 return VINF_SUCCESS;
4836}
4837
4838
4839/**
4840 * Loads the guest activity state into the guest-state area in the VMCS.
4841 *
4842 * @returns VBox status code.
4843 * @param pVCpu The cross context virtual CPU structure.
4844 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4845 * out-of-sync. Make sure to update the required fields
4846 * before using them.
4847 *
4848 * @remarks No-long-jump zone!!!
4849 */
4850static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4851{
4852 NOREF(pMixedCtx);
4853 /** @todo See if we can make use of other states, e.g.
4854 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4855 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4856 {
4857 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4858 AssertRCReturn(rc, rc);
4859
4860 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4861 }
4862 return VINF_SUCCESS;
4863}
4864
4865
4866#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4867/**
4868 * Check if guest state allows safe use of 32-bit switcher again.
4869 *
4870 * Segment bases and protected mode structures must be 32-bit addressable
4871 * because the 32-bit switcher will ignore high dword when writing these VMCS
4872 * fields. See @bugref{8432} for details.
4873 *
4874 * @returns true if safe, false if must continue to use the 64-bit switcher.
4875 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4876 * out-of-sync. Make sure to update the required fields
4877 * before using them.
4878 *
4879 * @remarks No-long-jump zone!!!
4880 */
4881static bool hmR0VmxIs32BitSwitcherSafe(PCPUMCTX pMixedCtx)
4882{
4883 if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000))
4884 return false;
4885 if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000))
4886 return false;
4887 if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000))
4888 return false;
4889 if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000))
4890 return false;
4891 if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000))
4892 return false;
4893 if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000))
4894 return false;
4895 if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000))
4896 return false;
4897 if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000))
4898 return false;
4899 if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000))
4900 return false;
4901 if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000))
4902 return false;
4903 /* All good, bases are 32-bit. */
4904 return true;
4905}
4906#endif
4907
4908
4909/**
4910 * Sets up the appropriate function to run guest code.
4911 *
4912 * @returns VBox status code.
4913 * @param pVCpu The cross context virtual CPU structure.
4914 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4915 * out-of-sync. Make sure to update the required fields
4916 * before using them.
4917 *
4918 * @remarks No-long-jump zone!!!
4919 */
4920static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4921{
4922 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4923 {
4924#ifndef VBOX_ENABLE_64_BITS_GUESTS
4925 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4926#endif
4927 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4928#if HC_ARCH_BITS == 32
4929 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4930 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4931 {
4932 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4933 {
4934 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4935 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4936 | HM_CHANGED_VMX_ENTRY_CTLS
4937 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4938 }
4939 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4940
4941 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4942 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4943 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4944 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 64-bit switcher\n", pVCpu->idCpu));
4945 }
4946#else
4947 /* 64-bit host. */
4948 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4949#endif
4950 }
4951 else
4952 {
4953 /* Guest is not in long mode, use the 32-bit handler. */
4954#if HC_ARCH_BITS == 32
4955 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4956 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4957 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4958 {
4959 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4960 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4961 | HM_CHANGED_VMX_ENTRY_CTLS
4962 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4963 }
4964# ifdef VBOX_ENABLE_64_BITS_GUESTS
4965 /*
4966 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel design, see @bugref{8432#c7}.
4967 * If real-on-v86 mode is active, clear the 64-bit switcher flag because now we know the guest is in a sane
4968 * state where it's safe to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4969 * the much faster 32-bit switcher again.
4970 */
4971 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4972 {
4973 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4974 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher\n", pVCpu->idCpu));
4975 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4976 }
4977 else
4978 {
4979 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4980 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4981 || hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
4982 {
4983 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4984 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4985 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR
4986 | HM_CHANGED_VMX_ENTRY_CTLS
4987 | HM_CHANGED_VMX_EXIT_CTLS
4988 | HM_CHANGED_HOST_CONTEXT);
4989 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher (safe)\n", pVCpu->idCpu));
4990 }
4991 }
4992# else
4993 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4994# endif
4995#else
4996 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4997#endif
4998 }
4999 Assert(pVCpu->hm.s.vmx.pfnStartVM);
5000 return VINF_SUCCESS;
5001}
5002
5003
5004/**
5005 * Wrapper for running the guest code in VT-x.
5006 *
5007 * @returns VBox status code, no informational status codes.
5008 * @param pVM The cross context VM structure.
5009 * @param pVCpu The cross context virtual CPU structure.
5010 * @param pCtx Pointer to the guest-CPU context.
5011 *
5012 * @remarks No-long-jump zone!!!
5013 */
5014DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5015{
5016 /*
5017 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
5018 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
5019 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
5020 */
5021 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
5022 /** @todo Add stats for resume vs launch. */
5023#ifdef VBOX_WITH_KERNEL_USING_XMM
5024 int rc = HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
5025#else
5026 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
5027#endif
5028 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
5029 return rc;
5030}
5031
5032
5033/**
5034 * Reports world-switch error and dumps some useful debug info.
5035 *
5036 * @param pVM The cross context VM structure.
5037 * @param pVCpu The cross context virtual CPU structure.
5038 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5039 * @param pCtx Pointer to the guest-CPU context.
5040 * @param pVmxTransient Pointer to the VMX transient structure (only
5041 * exitReason updated).
5042 */
5043static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
5044{
5045 Assert(pVM);
5046 Assert(pVCpu);
5047 Assert(pCtx);
5048 Assert(pVmxTransient);
5049 HMVMX_ASSERT_PREEMPT_SAFE();
5050
5051 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
5052 switch (rcVMRun)
5053 {
5054 case VERR_VMX_INVALID_VMXON_PTR:
5055 AssertFailed();
5056 break;
5057 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5058 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5059 {
5060 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5061 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5062 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5063 AssertRC(rc);
5064
5065 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5066 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5067 Cannot do it here as we may have been long preempted. */
5068
5069#ifdef VBOX_STRICT
5070 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5071 pVmxTransient->uExitReason));
5072 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5073 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5074 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5075 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5076 else
5077 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5078 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5079 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5080
5081 /* VMX control bits. */
5082 uint32_t u32Val;
5083 uint64_t u64Val;
5084 RTHCUINTREG uHCReg;
5085 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5086 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5087 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5088 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5089 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5090 {
5091 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5092 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5093 }
5094 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5095 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5096 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5097 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5098 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5099 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5100 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5101 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5102 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5103 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5104 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5105 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5106 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5107 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5108 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5109 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5110 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5111 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5112 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5113 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5114 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5115 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5116 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5117 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5118 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5119 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5120 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5121 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5122 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5123 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5124 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5125 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5126 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5127 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5128 if (pVM->hm.s.fNestedPaging)
5129 {
5130 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5131 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5132 }
5133
5134 /* Guest bits. */
5135 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5136 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5137 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5138 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5139 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5140 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5141 if (pVM->hm.s.vmx.fVpid)
5142 {
5143 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5144 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5145 }
5146
5147 /* Host bits. */
5148 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5149 Log4(("Host CR0 %#RHr\n", uHCReg));
5150 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5151 Log4(("Host CR3 %#RHr\n", uHCReg));
5152 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5153 Log4(("Host CR4 %#RHr\n", uHCReg));
5154
5155 RTGDTR HostGdtr;
5156 PCX86DESCHC pDesc;
5157 ASMGetGDTR(&HostGdtr);
5158 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5159 Log4(("Host CS %#08x\n", u32Val));
5160 if (u32Val < HostGdtr.cbGdt)
5161 {
5162 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5163 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5164 }
5165
5166 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5167 Log4(("Host DS %#08x\n", u32Val));
5168 if (u32Val < HostGdtr.cbGdt)
5169 {
5170 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5171 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5172 }
5173
5174 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5175 Log4(("Host ES %#08x\n", u32Val));
5176 if (u32Val < HostGdtr.cbGdt)
5177 {
5178 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5179 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5180 }
5181
5182 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5183 Log4(("Host FS %#08x\n", u32Val));
5184 if (u32Val < HostGdtr.cbGdt)
5185 {
5186 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5187 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5188 }
5189
5190 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5191 Log4(("Host GS %#08x\n", u32Val));
5192 if (u32Val < HostGdtr.cbGdt)
5193 {
5194 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5195 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5196 }
5197
5198 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5199 Log4(("Host SS %#08x\n", u32Val));
5200 if (u32Val < HostGdtr.cbGdt)
5201 {
5202 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5203 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5204 }
5205
5206 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5207 Log4(("Host TR %#08x\n", u32Val));
5208 if (u32Val < HostGdtr.cbGdt)
5209 {
5210 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5211 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5212 }
5213
5214 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5215 Log4(("Host TR Base %#RHv\n", uHCReg));
5216 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5217 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5218 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5219 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5220 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5221 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5222 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5223 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5224 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5225 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5226 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5227 Log4(("Host RSP %#RHv\n", uHCReg));
5228 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5229 Log4(("Host RIP %#RHv\n", uHCReg));
5230# if HC_ARCH_BITS == 64
5231 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5232 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5233 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5234 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5235 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5236 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5237# endif
5238#endif /* VBOX_STRICT */
5239 break;
5240 }
5241
5242 default:
5243 /* Impossible */
5244 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5245 break;
5246 }
5247 NOREF(pVM); NOREF(pCtx);
5248}
5249
5250
5251#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5252#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5253# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5254#endif
5255#ifdef VBOX_STRICT
5256static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5257{
5258 switch (idxField)
5259 {
5260 case VMX_VMCS_GUEST_RIP:
5261 case VMX_VMCS_GUEST_RSP:
5262 case VMX_VMCS_GUEST_SYSENTER_EIP:
5263 case VMX_VMCS_GUEST_SYSENTER_ESP:
5264 case VMX_VMCS_GUEST_GDTR_BASE:
5265 case VMX_VMCS_GUEST_IDTR_BASE:
5266 case VMX_VMCS_GUEST_CS_BASE:
5267 case VMX_VMCS_GUEST_DS_BASE:
5268 case VMX_VMCS_GUEST_ES_BASE:
5269 case VMX_VMCS_GUEST_FS_BASE:
5270 case VMX_VMCS_GUEST_GS_BASE:
5271 case VMX_VMCS_GUEST_SS_BASE:
5272 case VMX_VMCS_GUEST_LDTR_BASE:
5273 case VMX_VMCS_GUEST_TR_BASE:
5274 case VMX_VMCS_GUEST_CR3:
5275 return true;
5276 }
5277 return false;
5278}
5279
5280static bool hmR0VmxIsValidReadField(uint32_t idxField)
5281{
5282 switch (idxField)
5283 {
5284 /* Read-only fields. */
5285 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5286 return true;
5287 }
5288 /* Remaining readable fields should also be writable. */
5289 return hmR0VmxIsValidWriteField(idxField);
5290}
5291#endif /* VBOX_STRICT */
5292
5293
5294/**
5295 * Executes the specified handler in 64-bit mode.
5296 *
5297 * @returns VBox status code (no informational status codes).
5298 * @param pVM The cross context VM structure.
5299 * @param pVCpu The cross context virtual CPU structure.
5300 * @param pCtx Pointer to the guest CPU context.
5301 * @param enmOp The operation to perform.
5302 * @param cParams Number of parameters.
5303 * @param paParam Array of 32-bit parameters.
5304 */
5305VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5306 uint32_t cParams, uint32_t *paParam)
5307{
5308 NOREF(pCtx);
5309
5310 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5311 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5312 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5313 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5314
5315#ifdef VBOX_STRICT
5316 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5317 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5318
5319 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5320 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5321#endif
5322
5323 /* Disable interrupts. */
5324 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5325
5326#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5327 RTCPUID idHostCpu = RTMpCpuId();
5328 CPUMR0SetLApic(pVCpu, idHostCpu);
5329#endif
5330
5331 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5332 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5333
5334 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5335 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5336 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5337
5338 /* Leave VMX Root Mode. */
5339 VMXDisable();
5340
5341 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5342
5343 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5344 CPUMSetHyperEIP(pVCpu, enmOp);
5345 for (int i = (int)cParams - 1; i >= 0; i--)
5346 CPUMPushHyper(pVCpu, paParam[i]);
5347
5348 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5349
5350 /* Call the switcher. */
5351 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5352 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5353
5354 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5355 /* Make sure the VMX instructions don't cause #UD faults. */
5356 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5357
5358 /* Re-enter VMX Root Mode */
5359 int rc2 = VMXEnable(HCPhysCpuPage);
5360 if (RT_FAILURE(rc2))
5361 {
5362 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5363 ASMSetFlags(fOldEFlags);
5364 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5365 return rc2;
5366 }
5367
5368 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5369 AssertRC(rc2);
5370 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5371 Assert(!(ASMGetFlags() & X86_EFL_IF));
5372 ASMSetFlags(fOldEFlags);
5373 return rc;
5374}
5375
5376
5377/**
5378 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5379 * supporting 64-bit guests.
5380 *
5381 * @returns VBox status code.
5382 * @param fResume Whether to VMLAUNCH or VMRESUME.
5383 * @param pCtx Pointer to the guest-CPU context.
5384 * @param pCache Pointer to the VMCS cache.
5385 * @param pVM The cross context VM structure.
5386 * @param pVCpu The cross context virtual CPU structure.
5387 */
5388DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5389{
5390 NOREF(fResume);
5391
5392 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5393 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5394
5395#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5396 pCache->uPos = 1;
5397 pCache->interPD = PGMGetInterPaeCR3(pVM);
5398 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5399#endif
5400
5401#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5402 pCache->TestIn.HCPhysCpuPage = 0;
5403 pCache->TestIn.HCPhysVmcs = 0;
5404 pCache->TestIn.pCache = 0;
5405 pCache->TestOut.HCPhysVmcs = 0;
5406 pCache->TestOut.pCache = 0;
5407 pCache->TestOut.pCtx = 0;
5408 pCache->TestOut.eflags = 0;
5409#else
5410 NOREF(pCache);
5411#endif
5412
5413 uint32_t aParam[10];
5414 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5415 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5416 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5417 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5418 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5419 aParam[5] = 0;
5420 aParam[6] = VM_RC_ADDR(pVM, pVM);
5421 aParam[7] = 0;
5422 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5423 aParam[9] = 0;
5424
5425#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5426 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5427 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5428#endif
5429 int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5430
5431#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5432 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5433 Assert(pCtx->dr[4] == 10);
5434 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5435#endif
5436
5437#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5438 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5439 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5440 pVCpu->hm.s.vmx.HCPhysVmcs));
5441 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5442 pCache->TestOut.HCPhysVmcs));
5443 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5444 pCache->TestOut.pCache));
5445 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5446 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5447 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5448 pCache->TestOut.pCtx));
5449 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5450#endif
5451 return rc;
5452}
5453
5454
5455/**
5456 * Initialize the VMCS-Read cache.
5457 *
5458 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5459 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5460 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5461 * (those that have a 32-bit FULL & HIGH part).
5462 *
5463 * @returns VBox status code.
5464 * @param pVM The cross context VM structure.
5465 * @param pVCpu The cross context virtual CPU structure.
5466 */
5467static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5468{
5469#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5470{ \
5471 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5472 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5473 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5474 ++cReadFields; \
5475}
5476
5477 AssertPtr(pVM);
5478 AssertPtr(pVCpu);
5479 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5480 uint32_t cReadFields = 0;
5481
5482 /*
5483 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5484 * and serve to indicate exceptions to the rules.
5485 */
5486
5487 /* Guest-natural selector base fields. */
5488#if 0
5489 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5490 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5491 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5492#endif
5493 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5494 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5495 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5496 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5497 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5498 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5499 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5500 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5501 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5502 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5503 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5504 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5505#if 0
5506 /* Unused natural width guest-state fields. */
5507 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5508 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5509#endif
5510 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5511 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5512
5513 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5514#if 0
5515 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5516 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5517 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5518 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5519 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5520 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5521 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5522 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5523 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5524#endif
5525
5526 /* Natural width guest-state fields. */
5527 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5528#if 0
5529 /* Currently unused field. */
5530 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5531#endif
5532
5533 if (pVM->hm.s.fNestedPaging)
5534 {
5535 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5536 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5537 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5538 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5539 }
5540 else
5541 {
5542 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5543 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5544 }
5545
5546#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5547 return VINF_SUCCESS;
5548}
5549
5550
5551/**
5552 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5553 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5554 * darwin, running 64-bit guests).
5555 *
5556 * @returns VBox status code.
5557 * @param pVCpu The cross context virtual CPU structure.
5558 * @param idxField The VMCS field encoding.
5559 * @param u64Val 16, 32 or 64-bit value.
5560 */
5561VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5562{
5563 int rc;
5564 switch (idxField)
5565 {
5566 /*
5567 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5568 */
5569 /* 64-bit Control fields. */
5570 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5571 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5572 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5573 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5574 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5575 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5576 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5577 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5578 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5579 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5580 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5581 case VMX_VMCS64_CTRL_EPTP_FULL:
5582 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5583 /* 64-bit Guest-state fields. */
5584 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5585 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5586 case VMX_VMCS64_GUEST_PAT_FULL:
5587 case VMX_VMCS64_GUEST_EFER_FULL:
5588 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5589 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5590 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5591 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5592 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5593 /* 64-bit Host-state fields. */
5594 case VMX_VMCS64_HOST_PAT_FULL:
5595 case VMX_VMCS64_HOST_EFER_FULL:
5596 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5597 {
5598 rc = VMXWriteVmcs32(idxField, u64Val);
5599 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5600 break;
5601 }
5602
5603 /*
5604 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5605 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5606 */
5607 /* Natural-width Guest-state fields. */
5608 case VMX_VMCS_GUEST_CR3:
5609 case VMX_VMCS_GUEST_ES_BASE:
5610 case VMX_VMCS_GUEST_CS_BASE:
5611 case VMX_VMCS_GUEST_SS_BASE:
5612 case VMX_VMCS_GUEST_DS_BASE:
5613 case VMX_VMCS_GUEST_FS_BASE:
5614 case VMX_VMCS_GUEST_GS_BASE:
5615 case VMX_VMCS_GUEST_LDTR_BASE:
5616 case VMX_VMCS_GUEST_TR_BASE:
5617 case VMX_VMCS_GUEST_GDTR_BASE:
5618 case VMX_VMCS_GUEST_IDTR_BASE:
5619 case VMX_VMCS_GUEST_RSP:
5620 case VMX_VMCS_GUEST_RIP:
5621 case VMX_VMCS_GUEST_SYSENTER_ESP:
5622 case VMX_VMCS_GUEST_SYSENTER_EIP:
5623 {
5624 if (!(u64Val >> 32))
5625 {
5626 /* If this field is 64-bit, VT-x will zero out the top bits. */
5627 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5628 }
5629 else
5630 {
5631 /* Assert that only the 32->64 switcher case should ever come here. */
5632 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5633 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5634 }
5635 break;
5636 }
5637
5638 default:
5639 {
5640 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5641 rc = VERR_INVALID_PARAMETER;
5642 break;
5643 }
5644 }
5645 AssertRCReturn(rc, rc);
5646 return rc;
5647}
5648
5649
5650/**
5651 * Queue up a VMWRITE by using the VMCS write cache.
5652 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5653 *
5654 * @param pVCpu The cross context virtual CPU structure.
5655 * @param idxField The VMCS field encoding.
5656 * @param u64Val 16, 32 or 64-bit value.
5657 */
5658VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5659{
5660 AssertPtr(pVCpu);
5661 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5662
5663 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5664 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5665
5666 /* Make sure there are no duplicates. */
5667 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5668 {
5669 if (pCache->Write.aField[i] == idxField)
5670 {
5671 pCache->Write.aFieldVal[i] = u64Val;
5672 return VINF_SUCCESS;
5673 }
5674 }
5675
5676 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5677 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5678 pCache->Write.cValidEntries++;
5679 return VINF_SUCCESS;
5680}
5681#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5682
5683
5684/**
5685 * Sets up the usage of TSC-offsetting and updates the VMCS.
5686 *
5687 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5688 * VMX preemption timer.
5689 *
5690 * @returns VBox status code.
5691 * @param pVM The cross context VM structure.
5692 * @param pVCpu The cross context virtual CPU structure.
5693 *
5694 * @remarks No-long-jump zone!!!
5695 */
5696static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5697{
5698 int rc;
5699 bool fOffsettedTsc;
5700 bool fParavirtTsc;
5701 if (pVM->hm.s.vmx.fUsePreemptTimer)
5702 {
5703 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5704 &fOffsettedTsc, &fParavirtTsc);
5705
5706 /* Make sure the returned values have sane upper and lower boundaries. */
5707 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5708 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5709 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5710 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5711
5712 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5713 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5714 }
5715 else
5716 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5717
5718 /** @todo later optimize this to be done elsewhere and not before every
5719 * VM-entry. */
5720 if (fParavirtTsc)
5721 {
5722 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5723 information before every VM-entry, hence disable it for performance sake. */
5724#if 0
5725 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5726 AssertRC(rc);
5727#endif
5728 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5729 }
5730
5731 if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5732 {
5733 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5734 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5735
5736 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5737 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5738 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5739 }
5740 else
5741 {
5742 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5743 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5744 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5745 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5746 }
5747}
5748
5749
5750/**
5751 * Determines if an exception is a contributory exception.
5752 *
5753 * Contributory exceptions are ones which can cause double-faults unless the
5754 * original exception was a benign exception. Page-fault is intentionally not
5755 * included here as it's a conditional contributory exception.
5756 *
5757 * @returns true if the exception is contributory, false otherwise.
5758 * @param uVector The exception vector.
5759 */
5760DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5761{
5762 switch (uVector)
5763 {
5764 case X86_XCPT_GP:
5765 case X86_XCPT_SS:
5766 case X86_XCPT_NP:
5767 case X86_XCPT_TS:
5768 case X86_XCPT_DE:
5769 return true;
5770 default:
5771 break;
5772 }
5773 return false;
5774}
5775
5776
5777/**
5778 * Sets an event as a pending event to be injected into the guest.
5779 *
5780 * @param pVCpu The cross context virtual CPU structure.
5781 * @param u32IntInfo The VM-entry interruption-information field.
5782 * @param cbInstr The VM-entry instruction length in bytes (for software
5783 * interrupts, exceptions and privileged software
5784 * exceptions).
5785 * @param u32ErrCode The VM-entry exception error code.
5786 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5787 * page-fault.
5788 *
5789 * @remarks Statistics counter assumes this is a guest event being injected or
5790 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5791 * always incremented.
5792 */
5793DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5794 RTGCUINTPTR GCPtrFaultAddress)
5795{
5796 Assert(!pVCpu->hm.s.Event.fPending);
5797 pVCpu->hm.s.Event.fPending = true;
5798 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5799 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5800 pVCpu->hm.s.Event.cbInstr = cbInstr;
5801 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5802}
5803
5804
5805/**
5806 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5807 *
5808 * @param pVCpu The cross context virtual CPU structure.
5809 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5810 * out-of-sync. Make sure to update the required fields
5811 * before using them.
5812 */
5813DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5814{
5815 NOREF(pMixedCtx);
5816 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5817 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5818 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5819 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5820}
5821
5822
5823/**
5824 * Handle a condition that occurred while delivering an event through the guest
5825 * IDT.
5826 *
5827 * @returns Strict VBox status code (i.e. informational status codes too).
5828 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5829 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5830 * to continue execution of the guest which will delivery the \#DF.
5831 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5832 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5833 *
5834 * @param pVCpu The cross context virtual CPU structure.
5835 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5836 * out-of-sync. Make sure to update the required fields
5837 * before using them.
5838 * @param pVmxTransient Pointer to the VMX transient structure.
5839 *
5840 * @remarks No-long-jump zone!!!
5841 */
5842static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5843{
5844 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5845
5846 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5847 rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5848
5849 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5850 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5851 {
5852 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5853 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5854
5855 typedef enum
5856 {
5857 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5858 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5859 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5860 VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
5861 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5862 } VMXREFLECTXCPT;
5863
5864 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5865 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5866 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5867 {
5868 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5869 {
5870 enmReflect = VMXREFLECTXCPT_XCPT;
5871#ifdef VBOX_STRICT
5872 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5873 && uExitVector == X86_XCPT_PF)
5874 {
5875 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5876 }
5877#endif
5878 if ( uExitVector == X86_XCPT_PF
5879 && uIdtVector == X86_XCPT_PF)
5880 {
5881 pVmxTransient->fVectoringDoublePF = true;
5882 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5883 }
5884 else if ( uExitVector == X86_XCPT_AC
5885 && uIdtVector == X86_XCPT_AC)
5886 {
5887 enmReflect = VMXREFLECTXCPT_HANG;
5888 Log4(("IDT: Nested #AC - Bad guest\n"));
5889 }
5890 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5891 && hmR0VmxIsContributoryXcpt(uExitVector)
5892 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5893 || uIdtVector == X86_XCPT_PF))
5894 {
5895 enmReflect = VMXREFLECTXCPT_DF;
5896 }
5897 else if (uIdtVector == X86_XCPT_DF)
5898 enmReflect = VMXREFLECTXCPT_TF;
5899 }
5900 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5901 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5902 {
5903 /*
5904 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5905 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5906 */
5907 enmReflect = VMXREFLECTXCPT_XCPT;
5908
5909 if (uExitVector == X86_XCPT_PF)
5910 {
5911 pVmxTransient->fVectoringPF = true;
5912 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5913 }
5914 }
5915 }
5916 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5917 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5918 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5919 {
5920 /*
5921 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5922 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5923 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5924 */
5925 enmReflect = VMXREFLECTXCPT_XCPT;
5926 }
5927
5928 /*
5929 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5930 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5931 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5932 *
5933 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5934 */
5935 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5936 && enmReflect == VMXREFLECTXCPT_XCPT
5937 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5938 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5939 {
5940 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5941 }
5942
5943 switch (enmReflect)
5944 {
5945 case VMXREFLECTXCPT_XCPT:
5946 {
5947 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5948 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5949 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5950
5951 uint32_t u32ErrCode = 0;
5952 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5953 {
5954 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5955 AssertRCReturn(rc2, rc2);
5956 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5957 }
5958
5959 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5960 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5961 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5962 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5963 rcStrict = VINF_SUCCESS;
5964 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5965 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5966
5967 break;
5968 }
5969
5970 case VMXREFLECTXCPT_DF:
5971 {
5972 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5973 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5974 rcStrict = VINF_HM_DOUBLE_FAULT;
5975 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5976 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5977
5978 break;
5979 }
5980
5981 case VMXREFLECTXCPT_TF:
5982 {
5983 rcStrict = VINF_EM_RESET;
5984 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5985 uExitVector));
5986 break;
5987 }
5988
5989 case VMXREFLECTXCPT_HANG:
5990 {
5991 rcStrict = VERR_EM_GUEST_CPU_HANG;
5992 break;
5993 }
5994
5995 default:
5996 Assert(rcStrict == VINF_SUCCESS);
5997 break;
5998 }
5999 }
6000 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6001 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6002 && uExitVector != X86_XCPT_DF
6003 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6004 {
6005 /*
6006 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6007 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6008 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6009 */
6010 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6011 {
6012 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
6013 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6014 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6015 }
6016 }
6017
6018 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
6019 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
6020 return rcStrict;
6021}
6022
6023
6024/**
6025 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
6026 *
6027 * @returns VBox status code.
6028 * @param pVCpu The cross context virtual CPU structure.
6029 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6030 * out-of-sync. Make sure to update the required fields
6031 * before using them.
6032 *
6033 * @remarks No-long-jump zone!!!
6034 */
6035static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6036{
6037 NOREF(pMixedCtx);
6038
6039 /*
6040 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
6041 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
6042 */
6043 VMMRZCallRing3Disable(pVCpu);
6044 HM_DISABLE_PREEMPT();
6045
6046 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
6047 {
6048 uint32_t uVal = 0;
6049 uint32_t uShadow = 0;
6050 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
6051 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
6052 AssertRCReturn(rc, rc);
6053
6054 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
6055 CPUMSetGuestCR0(pVCpu, uVal);
6056 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
6057 }
6058
6059 HM_RESTORE_PREEMPT();
6060 VMMRZCallRing3Enable(pVCpu);
6061 return VINF_SUCCESS;
6062}
6063
6064
6065/**
6066 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
6067 *
6068 * @returns VBox status code.
6069 * @param pVCpu The cross context virtual CPU structure.
6070 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6071 * out-of-sync. Make sure to update the required fields
6072 * before using them.
6073 *
6074 * @remarks No-long-jump zone!!!
6075 */
6076static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6077{
6078 NOREF(pMixedCtx);
6079
6080 int rc = VINF_SUCCESS;
6081 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
6082 {
6083 uint32_t uVal = 0;
6084 uint32_t uShadow = 0;
6085 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
6086 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
6087 AssertRCReturn(rc, rc);
6088
6089 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
6090 CPUMSetGuestCR4(pVCpu, uVal);
6091 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6092 }
6093 return rc;
6094}
6095
6096
6097/**
6098 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
6099 *
6100 * @returns VBox status code.
6101 * @param pVCpu The cross context virtual CPU structure.
6102 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6103 * out-of-sync. Make sure to update the required fields
6104 * before using them.
6105 *
6106 * @remarks No-long-jump zone!!!
6107 */
6108static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6109{
6110 int rc = VINF_SUCCESS;
6111 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6112 {
6113 uint64_t u64Val = 0;
6114 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6115 AssertRCReturn(rc, rc);
6116
6117 pMixedCtx->rip = u64Val;
6118 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6119 }
6120 return rc;
6121}
6122
6123
6124/**
6125 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6126 *
6127 * @returns VBox status code.
6128 * @param pVCpu The cross context virtual CPU structure.
6129 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6130 * out-of-sync. Make sure to update the required fields
6131 * before using them.
6132 *
6133 * @remarks No-long-jump zone!!!
6134 */
6135static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6136{
6137 int rc = VINF_SUCCESS;
6138 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6139 {
6140 uint64_t u64Val = 0;
6141 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6142 AssertRCReturn(rc, rc);
6143
6144 pMixedCtx->rsp = u64Val;
6145 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6146 }
6147 return rc;
6148}
6149
6150
6151/**
6152 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6153 *
6154 * @returns VBox status code.
6155 * @param pVCpu The cross context virtual CPU structure.
6156 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6157 * out-of-sync. Make sure to update the required fields
6158 * before using them.
6159 *
6160 * @remarks No-long-jump zone!!!
6161 */
6162static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6163{
6164 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6165 {
6166 uint32_t uVal = 0;
6167 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6168 AssertRCReturn(rc, rc);
6169
6170 pMixedCtx->eflags.u32 = uVal;
6171 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6172 {
6173 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6174 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6175
6176 pMixedCtx->eflags.Bits.u1VM = 0;
6177 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6178 }
6179
6180 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6181 }
6182 return VINF_SUCCESS;
6183}
6184
6185
6186/**
6187 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6188 * guest-CPU context.
6189 */
6190DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6191{
6192 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6193 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6194 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6195 return rc;
6196}
6197
6198
6199/**
6200 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6201 * from the guest-state area in the VMCS.
6202 *
6203 * @param pVCpu The cross context virtual CPU structure.
6204 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6205 * out-of-sync. Make sure to update the required fields
6206 * before using them.
6207 *
6208 * @remarks No-long-jump zone!!!
6209 */
6210static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6211{
6212 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6213 {
6214 uint32_t uIntrState = 0;
6215 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6216 AssertRC(rc);
6217
6218 if (!uIntrState)
6219 {
6220 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6221 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6222
6223 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6224 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6225 }
6226 else
6227 {
6228 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6229 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6230 {
6231 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6232 AssertRC(rc);
6233 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6234 AssertRC(rc);
6235
6236 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6237 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6238 }
6239 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6240 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6241
6242 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6243 {
6244 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6245 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6246 }
6247 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6248 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6249 }
6250
6251 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6252 }
6253}
6254
6255
6256/**
6257 * Saves the guest's activity state.
6258 *
6259 * @returns VBox status code.
6260 * @param pVCpu The cross context virtual CPU structure.
6261 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6262 * out-of-sync. Make sure to update the required fields
6263 * before using them.
6264 *
6265 * @remarks No-long-jump zone!!!
6266 */
6267static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6268{
6269 NOREF(pMixedCtx);
6270 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6271 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6272 return VINF_SUCCESS;
6273}
6274
6275
6276/**
6277 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6278 * the current VMCS into the guest-CPU context.
6279 *
6280 * @returns VBox status code.
6281 * @param pVCpu The cross context virtual CPU structure.
6282 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6283 * out-of-sync. Make sure to update the required fields
6284 * before using them.
6285 *
6286 * @remarks No-long-jump zone!!!
6287 */
6288static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6289{
6290 int rc = VINF_SUCCESS;
6291 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6292 {
6293 uint32_t u32Val = 0;
6294 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6295 pMixedCtx->SysEnter.cs = u32Val;
6296 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6297 }
6298
6299 uint64_t u64Val = 0;
6300 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6301 {
6302 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6303 pMixedCtx->SysEnter.eip = u64Val;
6304 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6305 }
6306 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6307 {
6308 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6309 pMixedCtx->SysEnter.esp = u64Val;
6310 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6311 }
6312 return rc;
6313}
6314
6315
6316/**
6317 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6318 * the CPU back into the guest-CPU context.
6319 *
6320 * @returns VBox status code.
6321 * @param pVCpu The cross context virtual CPU structure.
6322 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6323 * out-of-sync. Make sure to update the required fields
6324 * before using them.
6325 *
6326 * @remarks No-long-jump zone!!!
6327 */
6328static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6329{
6330 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6331 VMMRZCallRing3Disable(pVCpu);
6332 HM_DISABLE_PREEMPT();
6333
6334 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6335 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6336 {
6337 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6338 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6339 }
6340
6341 HM_RESTORE_PREEMPT();
6342 VMMRZCallRing3Enable(pVCpu);
6343
6344 return VINF_SUCCESS;
6345}
6346
6347
6348/**
6349 * Saves the auto load/store'd guest MSRs from the current VMCS into
6350 * the guest-CPU context.
6351 *
6352 * @returns VBox status code.
6353 * @param pVCpu The cross context virtual CPU structure.
6354 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6355 * out-of-sync. Make sure to update the required fields
6356 * before using them.
6357 *
6358 * @remarks No-long-jump zone!!!
6359 */
6360static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6361{
6362 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6363 return VINF_SUCCESS;
6364
6365 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6366 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6367 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6368 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6369 {
6370 switch (pMsr->u32Msr)
6371 {
6372 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6373 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6374 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6375 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6376 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6377 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6378 break;
6379
6380 default:
6381 {
6382 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6383 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6384 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6385 }
6386 }
6387 }
6388
6389 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6390 return VINF_SUCCESS;
6391}
6392
6393
6394/**
6395 * Saves the guest control registers from the current VMCS into the guest-CPU
6396 * context.
6397 *
6398 * @returns VBox status code.
6399 * @param pVCpu The cross context virtual CPU structure.
6400 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6401 * out-of-sync. Make sure to update the required fields
6402 * before using them.
6403 *
6404 * @remarks No-long-jump zone!!!
6405 */
6406static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6407{
6408 /* Guest CR0. Guest FPU. */
6409 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6410 AssertRCReturn(rc, rc);
6411
6412 /* Guest CR4. */
6413 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6414 AssertRCReturn(rc, rc);
6415
6416 /* Guest CR2 - updated always during the world-switch or in #PF. */
6417 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6418 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6419 {
6420 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6421 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6422
6423 PVM pVM = pVCpu->CTX_SUFF(pVM);
6424 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6425 || ( pVM->hm.s.fNestedPaging
6426 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6427 {
6428 uint64_t u64Val = 0;
6429 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6430 if (pMixedCtx->cr3 != u64Val)
6431 {
6432 CPUMSetGuestCR3(pVCpu, u64Val);
6433 if (VMMRZCallRing3IsEnabled(pVCpu))
6434 {
6435 PGMUpdateCR3(pVCpu, u64Val);
6436 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6437 }
6438 else
6439 {
6440 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6441 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6442 }
6443 }
6444
6445 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6446 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6447 {
6448 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6449 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6450 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6451 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6452 AssertRCReturn(rc, rc);
6453
6454 if (VMMRZCallRing3IsEnabled(pVCpu))
6455 {
6456 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6457 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6458 }
6459 else
6460 {
6461 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6462 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6463 }
6464 }
6465 }
6466
6467 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6468 }
6469
6470 /*
6471 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6472 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6473 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6474 *
6475 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6476 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6477 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6478 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6479 *
6480 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6481 */
6482 if (VMMRZCallRing3IsEnabled(pVCpu))
6483 {
6484 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6485 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6486
6487 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6488 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6489
6490 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6491 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6492 }
6493
6494 return rc;
6495}
6496
6497
6498/**
6499 * Reads a guest segment register from the current VMCS into the guest-CPU
6500 * context.
6501 *
6502 * @returns VBox status code.
6503 * @param pVCpu The cross context virtual CPU structure.
6504 * @param idxSel Index of the selector in the VMCS.
6505 * @param idxLimit Index of the segment limit in the VMCS.
6506 * @param idxBase Index of the segment base in the VMCS.
6507 * @param idxAccess Index of the access rights of the segment in the VMCS.
6508 * @param pSelReg Pointer to the segment selector.
6509 *
6510 * @remarks No-long-jump zone!!!
6511 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6512 * macro as that takes care of whether to read from the VMCS cache or
6513 * not.
6514 */
6515DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6516 PCPUMSELREG pSelReg)
6517{
6518 NOREF(pVCpu);
6519
6520 uint32_t u32Val = 0;
6521 int rc = VMXReadVmcs32(idxSel, &u32Val);
6522 AssertRCReturn(rc, rc);
6523 pSelReg->Sel = (uint16_t)u32Val;
6524 pSelReg->ValidSel = (uint16_t)u32Val;
6525 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6526
6527 rc = VMXReadVmcs32(idxLimit, &u32Val);
6528 AssertRCReturn(rc, rc);
6529 pSelReg->u32Limit = u32Val;
6530
6531 uint64_t u64Val = 0;
6532 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6533 AssertRCReturn(rc, rc);
6534 pSelReg->u64Base = u64Val;
6535
6536 rc = VMXReadVmcs32(idxAccess, &u32Val);
6537 AssertRCReturn(rc, rc);
6538 pSelReg->Attr.u = u32Val;
6539
6540 /*
6541 * If VT-x marks the segment as unusable, most other bits remain undefined:
6542 * - For CS the L, D and G bits have meaning.
6543 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6544 * - For the remaining data segments no bits are defined.
6545 *
6546 * The present bit and the unusable bit has been observed to be set at the
6547 * same time (the selector was supposed to be invalid as we started executing
6548 * a V8086 interrupt in ring-0).
6549 *
6550 * What should be important for the rest of the VBox code, is that the P bit is
6551 * cleared. Some of the other VBox code recognizes the unusable bit, but
6552 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6553 * safe side here, we'll strip off P and other bits we don't care about. If
6554 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6555 *
6556 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6557 */
6558 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6559 {
6560 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6561
6562 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6563 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6564 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6565
6566 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6567#ifdef DEBUG_bird
6568 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6569 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6570 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6571#endif
6572 }
6573 return VINF_SUCCESS;
6574}
6575
6576
6577#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6578# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6579 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6580 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6581#else
6582# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6583 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6584 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6585#endif
6586
6587
6588/**
6589 * Saves the guest segment registers from the current VMCS into the guest-CPU
6590 * context.
6591 *
6592 * @returns VBox status code.
6593 * @param pVCpu The cross context virtual CPU structure.
6594 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6595 * out-of-sync. Make sure to update the required fields
6596 * before using them.
6597 *
6598 * @remarks No-long-jump zone!!!
6599 */
6600static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6601{
6602 /* Guest segment registers. */
6603 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6604 {
6605 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6606 AssertRCReturn(rc, rc);
6607
6608 rc = VMXLOCAL_READ_SEG(CS, cs);
6609 rc |= VMXLOCAL_READ_SEG(SS, ss);
6610 rc |= VMXLOCAL_READ_SEG(DS, ds);
6611 rc |= VMXLOCAL_READ_SEG(ES, es);
6612 rc |= VMXLOCAL_READ_SEG(FS, fs);
6613 rc |= VMXLOCAL_READ_SEG(GS, gs);
6614 AssertRCReturn(rc, rc);
6615
6616 /* Restore segment attributes for real-on-v86 mode hack. */
6617 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6618 {
6619 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6620 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6621 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6622 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6623 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6624 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6625 }
6626 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6627 }
6628
6629 return VINF_SUCCESS;
6630}
6631
6632
6633/**
6634 * Saves the guest descriptor table registers and task register from the current
6635 * VMCS into the guest-CPU context.
6636 *
6637 * @returns VBox status code.
6638 * @param pVCpu The cross context virtual CPU structure.
6639 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6640 * out-of-sync. Make sure to update the required fields
6641 * before using them.
6642 *
6643 * @remarks No-long-jump zone!!!
6644 */
6645static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6646{
6647 int rc = VINF_SUCCESS;
6648
6649 /* Guest LDTR. */
6650 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6651 {
6652 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6653 AssertRCReturn(rc, rc);
6654 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6655 }
6656
6657 /* Guest GDTR. */
6658 uint64_t u64Val = 0;
6659 uint32_t u32Val = 0;
6660 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6661 {
6662 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6663 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6664 pMixedCtx->gdtr.pGdt = u64Val;
6665 pMixedCtx->gdtr.cbGdt = u32Val;
6666 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6667 }
6668
6669 /* Guest IDTR. */
6670 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6671 {
6672 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6673 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6674 pMixedCtx->idtr.pIdt = u64Val;
6675 pMixedCtx->idtr.cbIdt = u32Val;
6676 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6677 }
6678
6679 /* Guest TR. */
6680 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6681 {
6682 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6683 AssertRCReturn(rc, rc);
6684
6685 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6686 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6687 {
6688 rc = VMXLOCAL_READ_SEG(TR, tr);
6689 AssertRCReturn(rc, rc);
6690 }
6691 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6692 }
6693 return rc;
6694}
6695
6696#undef VMXLOCAL_READ_SEG
6697
6698
6699/**
6700 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6701 * context.
6702 *
6703 * @returns VBox status code.
6704 * @param pVCpu The cross context virtual CPU structure.
6705 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6706 * out-of-sync. Make sure to update the required fields
6707 * before using them.
6708 *
6709 * @remarks No-long-jump zone!!!
6710 */
6711static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6712{
6713 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6714 {
6715 if (!pVCpu->hm.s.fUsingHyperDR7)
6716 {
6717 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6718 uint32_t u32Val;
6719 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6720 pMixedCtx->dr[7] = u32Val;
6721 }
6722
6723 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6724 }
6725 return VINF_SUCCESS;
6726}
6727
6728
6729/**
6730 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6731 *
6732 * @returns VBox status code.
6733 * @param pVCpu The cross context virtual CPU structure.
6734 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6735 * out-of-sync. Make sure to update the required fields
6736 * before using them.
6737 *
6738 * @remarks No-long-jump zone!!!
6739 */
6740static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6741{
6742 NOREF(pMixedCtx);
6743
6744 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6745 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6746 return VINF_SUCCESS;
6747}
6748
6749
6750/**
6751 * Saves the entire guest state from the currently active VMCS into the
6752 * guest-CPU context.
6753 *
6754 * This essentially VMREADs all guest-data.
6755 *
6756 * @returns VBox status code.
6757 * @param pVCpu The cross context virtual CPU structure.
6758 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6759 * out-of-sync. Make sure to update the required fields
6760 * before using them.
6761 */
6762static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6763{
6764 Assert(pVCpu);
6765 Assert(pMixedCtx);
6766
6767 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6768 return VINF_SUCCESS;
6769
6770 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6771 again on the ring-3 callback path, there is no real need to. */
6772 if (VMMRZCallRing3IsEnabled(pVCpu))
6773 VMMR0LogFlushDisable(pVCpu);
6774 else
6775 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6776 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6777
6778 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6779 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6780
6781 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6782 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6783
6784 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6785 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6786
6787 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6788 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6789
6790 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6791 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6792
6793 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6794 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6795
6796 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6797 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6798
6799 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6800 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6801
6802 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6803 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6804
6805 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6806 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6807
6808 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6809 ("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
6810 HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
6811
6812 if (VMMRZCallRing3IsEnabled(pVCpu))
6813 VMMR0LogFlushEnable(pVCpu);
6814
6815 return VINF_SUCCESS;
6816}
6817
6818
6819/**
6820 * Saves basic guest registers needed for IEM instruction execution.
6821 *
6822 * @returns VBox status code (OR-able).
6823 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6824 * @param pMixedCtx Pointer to the CPU context of the guest.
6825 * @param fMemory Whether the instruction being executed operates on
6826 * memory or not. Only CR0 is synced up if clear.
6827 * @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
6828 */
6829static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
6830{
6831 /*
6832 * We assume all general purpose registers other than RSP are available.
6833 *
6834 * RIP is a must, as it will be incremented or otherwise changed.
6835 *
6836 * RFLAGS are always required to figure the CPL.
6837 *
6838 * RSP isn't always required, however it's a GPR, so frequently required.
6839 *
6840 * SS and CS are the only segment register needed if IEM doesn't do memory
6841 * access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
6842 *
6843 * CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
6844 * be required for memory accesses.
6845 *
6846 * Note! Before IEM dispatches an exception, it will call us to sync in everything.
6847 */
6848 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6849 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6850 if (fNeedRsp)
6851 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6852 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6853 if (!fMemory)
6854 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6855 else
6856 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6857 AssertRCReturn(rc, rc);
6858 return rc;
6859}
6860
6861
6862/**
6863 * Ensures that we've got a complete basic guest-context.
6864 *
6865 * This excludes the FPU, SSE, AVX, and similar extended state. The interface
6866 * is for the interpreter.
6867 *
6868 * @returns VBox status code.
6869 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6870 * @param pMixedCtx Pointer to the guest-CPU context which may have data
6871 * needing to be synced in.
6872 * @thread EMT(pVCpu)
6873 */
6874VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6875{
6876 /* Note! Since this is only applicable to VT-x, the implementation is placed
6877 in the VT-x part of the sources instead of the generic stuff. */
6878 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
6879 {
6880 /* For now, imply that the caller might change everything too. */
6881 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6882 return hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6883 }
6884 return VINF_SUCCESS;
6885}
6886
6887
6888/**
6889 * Check per-VM and per-VCPU force flag actions that require us to go back to
6890 * ring-3 for one reason or another.
6891 *
6892 * @returns Strict VBox status code (i.e. informational status codes too)
6893 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6894 * ring-3.
6895 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6896 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6897 * interrupts)
6898 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6899 * all EMTs to be in ring-3.
6900 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6901 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6902 * to the EM loop.
6903 *
6904 * @param pVM The cross context VM structure.
6905 * @param pVCpu The cross context virtual CPU structure.
6906 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6907 * out-of-sync. Make sure to update the required fields
6908 * before using them.
6909 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6910 */
6911static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6912{
6913 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6914
6915 /*
6916 * Anything pending? Should be more likely than not if we're doing a good job.
6917 */
6918 if ( !fStepping
6919 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6920 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6921 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6922 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6923 return VINF_SUCCESS;
6924
6925 /* We need the control registers now, make sure the guest-CPU context is updated. */
6926 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6927 AssertRCReturn(rc3, rc3);
6928
6929 /* Pending HM CR3 sync. */
6930 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6931 {
6932 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6933 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6934 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6935 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6936 }
6937
6938 /* Pending HM PAE PDPEs. */
6939 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6940 {
6941 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6942 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6943 }
6944
6945 /* Pending PGM C3 sync. */
6946 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6947 {
6948 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6949 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6950 if (rcStrict2 != VINF_SUCCESS)
6951 {
6952 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6953 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6954 return rcStrict2;
6955 }
6956 }
6957
6958 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6959 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6960 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6961 {
6962 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6963 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6964 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6965 return rc2;
6966 }
6967
6968 /* Pending VM request packets, such as hardware interrupts. */
6969 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6970 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6971 {
6972 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6973 return VINF_EM_PENDING_REQUEST;
6974 }
6975
6976 /* Pending PGM pool flushes. */
6977 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6978 {
6979 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6980 return VINF_PGM_POOL_FLUSH_PENDING;
6981 }
6982
6983 /* Pending DMA requests. */
6984 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6985 {
6986 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6987 return VINF_EM_RAW_TO_R3;
6988 }
6989
6990 return VINF_SUCCESS;
6991}
6992
6993
6994/**
6995 * Converts any TRPM trap into a pending HM event. This is typically used when
6996 * entering from ring-3 (not longjmp returns).
6997 *
6998 * @param pVCpu The cross context virtual CPU structure.
6999 */
7000static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7001{
7002 Assert(TRPMHasTrap(pVCpu));
7003 Assert(!pVCpu->hm.s.Event.fPending);
7004
7005 uint8_t uVector;
7006 TRPMEVENT enmTrpmEvent;
7007 RTGCUINT uErrCode;
7008 RTGCUINTPTR GCPtrFaultAddress;
7009 uint8_t cbInstr;
7010
7011 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7012 AssertRC(rc);
7013
7014 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7015 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7016 if (enmTrpmEvent == TRPM_TRAP)
7017 {
7018 switch (uVector)
7019 {
7020 case X86_XCPT_NMI:
7021 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7022 break;
7023
7024 case X86_XCPT_BP:
7025 case X86_XCPT_OF:
7026 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7027 break;
7028
7029 case X86_XCPT_PF:
7030 case X86_XCPT_DF:
7031 case X86_XCPT_TS:
7032 case X86_XCPT_NP:
7033 case X86_XCPT_SS:
7034 case X86_XCPT_GP:
7035 case X86_XCPT_AC:
7036 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7037 /* no break! */
7038 default:
7039 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7040 break;
7041 }
7042 }
7043 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7044 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7045 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7046 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7047 else
7048 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7049
7050 rc = TRPMResetTrap(pVCpu);
7051 AssertRC(rc);
7052 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7053 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7054
7055 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7056}
7057
7058
7059/**
7060 * Converts the pending HM event into a TRPM trap.
7061 *
7062 * @param pVCpu The cross context virtual CPU structure.
7063 */
7064static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7065{
7066 Assert(pVCpu->hm.s.Event.fPending);
7067
7068 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7069 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7070 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
7071 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7072
7073 /* If a trap was already pending, we did something wrong! */
7074 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7075
7076 TRPMEVENT enmTrapType;
7077 switch (uVectorType)
7078 {
7079 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7080 enmTrapType = TRPM_HARDWARE_INT;
7081 break;
7082
7083 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7084 enmTrapType = TRPM_SOFTWARE_INT;
7085 break;
7086
7087 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7088 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7089 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7090 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7091 enmTrapType = TRPM_TRAP;
7092 break;
7093
7094 default:
7095 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7096 enmTrapType = TRPM_32BIT_HACK;
7097 break;
7098 }
7099
7100 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7101
7102 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7103 AssertRC(rc);
7104
7105 if (fErrorCodeValid)
7106 TRPMSetErrorCode(pVCpu, uErrorCode);
7107
7108 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7109 && uVector == X86_XCPT_PF)
7110 {
7111 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7112 }
7113 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7114 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7115 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7116 {
7117 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7118 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
7119 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7120 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7121 }
7122
7123 /* Clear any pending events from the VMCS. */
7124 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
7125 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
7126
7127 /* We're now done converting the pending event. */
7128 pVCpu->hm.s.Event.fPending = false;
7129}
7130
7131
7132/**
7133 * Does the necessary state syncing before returning to ring-3 for any reason
7134 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7135 *
7136 * @returns VBox status code.
7137 * @param pVCpu The cross context virtual CPU structure.
7138 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7139 * be out-of-sync. Make sure to update the required
7140 * fields before using them.
7141 * @param fSaveGuestState Whether to save the guest state or not.
7142 *
7143 * @remarks No-long-jmp zone!!!
7144 */
7145static int hmR0VmxLeave(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7146{
7147 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7148 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7149
7150 RTCPUID idCpu = RTMpCpuId();
7151 Log4Func(("HostCpuId=%u\n", idCpu));
7152
7153 /*
7154 * !!! IMPORTANT !!!
7155 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7156 */
7157
7158 /* Save the guest state if necessary. */
7159 if ( fSaveGuestState
7160 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7161 {
7162 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7163 AssertRCReturn(rc, rc);
7164 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7165 }
7166
7167 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
7168 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7169 {
7170 if (fSaveGuestState)
7171 {
7172 /* We shouldn't reload CR0 without saving it first. */
7173 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7174 AssertRCReturn(rc, rc);
7175 }
7176 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7177 }
7178
7179 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7180#ifdef VBOX_STRICT
7181 if (CPUMIsHyperDebugStateActive(pVCpu))
7182 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7183#endif
7184 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7185 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7186 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7187 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7188
7189#if HC_ARCH_BITS == 64
7190 /* Restore host-state bits that VT-x only restores partially. */
7191 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7192 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7193 {
7194 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7195 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7196 }
7197 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7198#endif
7199
7200 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7201 if (pVCpu->hm.s.vmx.fLazyMsrs)
7202 {
7203 /* We shouldn't reload the guest MSRs without saving it first. */
7204 if (!fSaveGuestState)
7205 {
7206 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7207 AssertRCReturn(rc, rc);
7208 }
7209 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7210 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7211 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7212 }
7213
7214 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7215 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7216
7217 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7218 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7219 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7220 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7221 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7222 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7223 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7224 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7225
7226 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7227
7228 /** @todo This partially defeats the purpose of having preemption hooks.
7229 * The problem is, deregistering the hooks should be moved to a place that
7230 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7231 * context.
7232 */
7233 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7234 {
7235 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7236 AssertRCReturn(rc, rc);
7237
7238 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7239 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7240 }
7241 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7242 NOREF(idCpu);
7243
7244 return VINF_SUCCESS;
7245}
7246
7247
7248/**
7249 * Leaves the VT-x session.
7250 *
7251 * @returns VBox status code.
7252 * @param pVCpu The cross context virtual CPU structure.
7253 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7254 * out-of-sync. Make sure to update the required fields
7255 * before using them.
7256 *
7257 * @remarks No-long-jmp zone!!!
7258 */
7259DECLINLINE(int) hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7260{
7261 HM_DISABLE_PREEMPT();
7262 HMVMX_ASSERT_CPU_SAFE();
7263 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7264 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7265
7266 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7267 and done this from the VMXR0ThreadCtxCallback(). */
7268 if (!pVCpu->hm.s.fLeaveDone)
7269 {
7270 int rc2 = hmR0VmxLeave(pVCpu, pMixedCtx, true /* fSaveGuestState */);
7271 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7272 pVCpu->hm.s.fLeaveDone = true;
7273 }
7274 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7275
7276 /*
7277 * !!! IMPORTANT !!!
7278 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7279 */
7280
7281 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7282 /** @todo Deregistering here means we need to VMCLEAR always
7283 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7284 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7285 VMMR0ThreadCtxHookDisable(pVCpu);
7286
7287 /* Leave HM context. This takes care of local init (term). */
7288 int rc = HMR0LeaveCpu(pVCpu);
7289
7290 HM_RESTORE_PREEMPT();
7291 return rc;
7292}
7293
7294
7295/**
7296 * Does the necessary state syncing before doing a longjmp to ring-3.
7297 *
7298 * @returns VBox status code.
7299 * @param pVCpu The cross context virtual CPU structure.
7300 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7301 * out-of-sync. Make sure to update the required fields
7302 * before using them.
7303 *
7304 * @remarks No-long-jmp zone!!!
7305 */
7306DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7307{
7308 return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7309}
7310
7311
7312/**
7313 * Take necessary actions before going back to ring-3.
7314 *
7315 * An action requires us to go back to ring-3. This function does the necessary
7316 * steps before we can safely return to ring-3. This is not the same as longjmps
7317 * to ring-3, this is voluntary and prepares the guest so it may continue
7318 * executing outside HM (recompiler/IEM).
7319 *
7320 * @returns VBox status code.
7321 * @param pVM The cross context VM structure.
7322 * @param pVCpu The cross context virtual CPU structure.
7323 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7324 * out-of-sync. Make sure to update the required fields
7325 * before using them.
7326 * @param rcExit The reason for exiting to ring-3. Can be
7327 * VINF_VMM_UNKNOWN_RING3_CALL.
7328 */
7329static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7330{
7331 Assert(pVM);
7332 Assert(pVCpu);
7333 Assert(pMixedCtx);
7334 HMVMX_ASSERT_PREEMPT_SAFE();
7335
7336 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7337 {
7338 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7339 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7340 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7341 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7342 }
7343
7344 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7345 VMMRZCallRing3Disable(pVCpu);
7346 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7347
7348 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7349 if (pVCpu->hm.s.Event.fPending)
7350 {
7351 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7352 Assert(!pVCpu->hm.s.Event.fPending);
7353 }
7354
7355 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7356 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7357
7358 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7359 and if we're injecting an event we should have a TRPM trap pending. */
7360 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7361#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7362 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7363#endif
7364
7365 /* Save guest state and restore host state bits. */
7366 int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7367 AssertRCReturn(rc, rc);
7368 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7369 /* Thread-context hooks are unregistered at this point!!! */
7370
7371 /* Sync recompiler state. */
7372 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7373 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7374 | CPUM_CHANGED_LDTR
7375 | CPUM_CHANGED_GDTR
7376 | CPUM_CHANGED_IDTR
7377 | CPUM_CHANGED_TR
7378 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7379 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7380 if ( pVM->hm.s.fNestedPaging
7381 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7382 {
7383 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7384 }
7385
7386 Assert(!pVCpu->hm.s.fClearTrapFlag);
7387
7388 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7389 if (rcExit != VINF_EM_RAW_INTERRUPT)
7390 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7391
7392 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7393
7394 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7395 VMMRZCallRing3RemoveNotification(pVCpu);
7396 VMMRZCallRing3Enable(pVCpu);
7397
7398 return rc;
7399}
7400
7401
7402/**
7403 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7404 * longjump to ring-3 and possibly get preempted.
7405 *
7406 * @returns VBox status code.
7407 * @param pVCpu The cross context virtual CPU structure.
7408 * @param enmOperation The operation causing the ring-3 longjump.
7409 * @param pvUser Opaque pointer to the guest-CPU context. The data
7410 * may be out-of-sync. Make sure to update the required
7411 * fields before using them.
7412 */
7413static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7414{
7415 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7416 {
7417 /*
7418 * !!! IMPORTANT !!!
7419 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7420 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7421 */
7422 VMMRZCallRing3RemoveNotification(pVCpu);
7423 VMMRZCallRing3Disable(pVCpu);
7424 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7425 RTThreadPreemptDisable(&PreemptState);
7426
7427 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7428 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7429
7430#if HC_ARCH_BITS == 64
7431 /* Restore host-state bits that VT-x only restores partially. */
7432 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7433 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7434 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7435 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7436#endif
7437 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7438 if (pVCpu->hm.s.vmx.fLazyMsrs)
7439 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7440
7441 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7442 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7443 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7444 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7445 {
7446 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7447 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7448 }
7449
7450 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7451 VMMR0ThreadCtxHookDisable(pVCpu);
7452 HMR0LeaveCpu(pVCpu);
7453 RTThreadPreemptRestore(&PreemptState);
7454 return VINF_SUCCESS;
7455 }
7456
7457 Assert(pVCpu);
7458 Assert(pvUser);
7459 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7460 HMVMX_ASSERT_PREEMPT_SAFE();
7461
7462 VMMRZCallRing3Disable(pVCpu);
7463 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7464
7465 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7466 enmOperation));
7467
7468 int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7469 AssertRCReturn(rc, rc);
7470
7471 VMMRZCallRing3Enable(pVCpu);
7472 return VINF_SUCCESS;
7473}
7474
7475
7476/**
7477 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7478 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7479 *
7480 * @param pVCpu The cross context virtual CPU structure.
7481 */
7482DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7483{
7484 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7485 {
7486 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7487 {
7488 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7489 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7490 AssertRC(rc);
7491 Log4(("Setup interrupt-window exiting\n"));
7492 }
7493 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7494}
7495
7496
7497/**
7498 * Clears the interrupt-window exiting control in the VMCS.
7499 *
7500 * @param pVCpu The cross context virtual CPU structure.
7501 */
7502DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7503{
7504 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7505 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7506 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7507 AssertRC(rc);
7508 Log4(("Cleared interrupt-window exiting\n"));
7509}
7510
7511
7512/**
7513 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7514 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7515 *
7516 * @param pVCpu The cross context virtual CPU structure.
7517 */
7518DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7519{
7520 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7521 {
7522 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7523 {
7524 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7525 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7526 AssertRC(rc);
7527 Log4(("Setup NMI-window exiting\n"));
7528 }
7529 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7530}
7531
7532
7533/**
7534 * Clears the NMI-window exiting control in the VMCS.
7535 *
7536 * @param pVCpu The cross context virtual CPU structure.
7537 */
7538DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7539{
7540 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7541 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7542 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7543 AssertRC(rc);
7544 Log4(("Cleared NMI-window exiting\n"));
7545}
7546
7547
7548/**
7549 * Evaluates the event to be delivered to the guest and sets it as the pending
7550 * event.
7551 *
7552 * @returns The VT-x guest-interruptibility state.
7553 * @param pVCpu The cross context virtual CPU structure.
7554 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7555 * out-of-sync. Make sure to update the required fields
7556 * before using them.
7557 */
7558static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7559{
7560 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7561 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7562 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7563 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7564 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7565
7566 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7567 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7568 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7569 Assert(!TRPMHasTrap(pVCpu));
7570
7571 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7572 APICUpdatePendingInterrupts(pVCpu);
7573
7574 /*
7575 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7576 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7577 */
7578 /** @todo SMI. SMIs take priority over NMIs. */
7579 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7580 {
7581 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7582 if ( !pVCpu->hm.s.Event.fPending
7583 && !fBlockNmi
7584 && !fBlockSti
7585 && !fBlockMovSS)
7586 {
7587 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7588 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7589 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7590
7591 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7592 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7593 }
7594 else
7595 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7596 }
7597 /*
7598 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7599 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7600 */
7601 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7602 && !pVCpu->hm.s.fSingleInstruction)
7603 {
7604 Assert(!DBGFIsStepping(pVCpu));
7605 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7606 AssertRC(rc);
7607 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7608 if ( !pVCpu->hm.s.Event.fPending
7609 && !fBlockInt
7610 && !fBlockSti
7611 && !fBlockMovSS)
7612 {
7613 uint8_t u8Interrupt;
7614 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7615 if (RT_SUCCESS(rc))
7616 {
7617 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7618 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7619 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7620
7621 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7622 }
7623 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7624 {
7625 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7626 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7627 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7628
7629 /*
7630 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7631 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7632 * need to re-set this force-flag here.
7633 */
7634 }
7635 else
7636 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7637 }
7638 else
7639 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7640 }
7641
7642 return uIntrState;
7643}
7644
7645
7646/**
7647 * Sets a pending-debug exception to be delivered to the guest if the guest is
7648 * single-stepping in the VMCS.
7649 *
7650 * @param pVCpu The cross context virtual CPU structure.
7651 */
7652DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7653{
7654 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7655 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7656 AssertRC(rc);
7657}
7658
7659
7660/**
7661 * Injects any pending events into the guest if the guest is in a state to
7662 * receive them.
7663 *
7664 * @returns Strict VBox status code (i.e. informational status codes too).
7665 * @param pVCpu The cross context virtual CPU structure.
7666 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7667 * out-of-sync. Make sure to update the required fields
7668 * before using them.
7669 * @param uIntrState The VT-x guest-interruptibility state.
7670 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7671 * return VINF_EM_DBG_STEPPED if the event was
7672 * dispatched directly.
7673 */
7674static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7675{
7676 HMVMX_ASSERT_PREEMPT_SAFE();
7677 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7678
7679 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7680 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7681
7682 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7683 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7684 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7685 Assert(!TRPMHasTrap(pVCpu));
7686
7687 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7688 if (pVCpu->hm.s.Event.fPending)
7689 {
7690 /*
7691 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7692 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7693 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7694 *
7695 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7696 */
7697 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7698#ifdef VBOX_STRICT
7699 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7700 {
7701 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7702 Assert(!fBlockInt);
7703 Assert(!fBlockSti);
7704 Assert(!fBlockMovSS);
7705 }
7706 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7707 {
7708 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7709 Assert(!fBlockSti);
7710 Assert(!fBlockMovSS);
7711 Assert(!fBlockNmi);
7712 }
7713#endif
7714 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7715 (uint8_t)uIntType));
7716 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7717 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7718 fStepping, &uIntrState);
7719 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7720
7721 /* Update the interruptibility-state as it could have been changed by
7722 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7723 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7724 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7725
7726 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7727 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7728 else
7729 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7730 }
7731
7732 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7733 if ( fBlockSti
7734 || fBlockMovSS)
7735 {
7736 if (!pVCpu->hm.s.fSingleInstruction)
7737 {
7738 /*
7739 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7740 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7741 * See Intel spec. 27.3.4 "Saving Non-Register State".
7742 */
7743 Assert(!DBGFIsStepping(pVCpu));
7744 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7745 AssertRCReturn(rc2, rc2);
7746 if (pMixedCtx->eflags.Bits.u1TF)
7747 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7748 }
7749 else if (pMixedCtx->eflags.Bits.u1TF)
7750 {
7751 /*
7752 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7753 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7754 */
7755 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7756 uIntrState = 0;
7757 }
7758 }
7759
7760 /*
7761 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7762 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7763 */
7764 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7765 AssertRC(rc2);
7766
7767 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7768 NOREF(fBlockMovSS); NOREF(fBlockSti);
7769 return rcStrict;
7770}
7771
7772
7773/**
7774 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7775 *
7776 * @param pVCpu The cross context virtual CPU structure.
7777 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7778 * out-of-sync. Make sure to update the required fields
7779 * before using them.
7780 */
7781DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7782{
7783 NOREF(pMixedCtx);
7784 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7785 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7786}
7787
7788
7789/**
7790 * Injects a double-fault (\#DF) exception into the VM.
7791 *
7792 * @returns Strict VBox status code (i.e. informational status codes too).
7793 * @param pVCpu The cross context virtual CPU structure.
7794 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7795 * out-of-sync. Make sure to update the required fields
7796 * before using them.
7797 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7798 * and should return VINF_EM_DBG_STEPPED if the event
7799 * is injected directly (register modified by us, not
7800 * by hardware on VM-entry).
7801 * @param puIntrState Pointer to the current guest interruptibility-state.
7802 * This interruptibility-state will be updated if
7803 * necessary. This cannot not be NULL.
7804 */
7805DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7806{
7807 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7808 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7809 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7810 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7811 fStepping, puIntrState);
7812}
7813
7814
7815/**
7816 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7817 *
7818 * @param pVCpu The cross context virtual CPU structure.
7819 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7820 * out-of-sync. Make sure to update the required fields
7821 * before using them.
7822 */
7823DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7824{
7825 NOREF(pMixedCtx);
7826 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7827 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7828 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7829}
7830
7831
7832/**
7833 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7834 *
7835 * @param pVCpu The cross context virtual CPU structure.
7836 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7837 * out-of-sync. Make sure to update the required fields
7838 * before using them.
7839 * @param cbInstr The value of RIP that is to be pushed on the guest
7840 * stack.
7841 */
7842DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7843{
7844 NOREF(pMixedCtx);
7845 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7846 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7847 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7848}
7849
7850
7851/**
7852 * Injects a general-protection (\#GP) fault into the VM.
7853 *
7854 * @returns Strict VBox status code (i.e. informational status codes too).
7855 * @param pVCpu The cross context virtual CPU structure.
7856 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7857 * out-of-sync. Make sure to update the required fields
7858 * before using them.
7859 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7860 * mode, i.e. in real-mode it's not valid).
7861 * @param u32ErrorCode The error code associated with the \#GP.
7862 * @param fStepping Whether we're running in
7863 * hmR0VmxRunGuestCodeStep() and should return
7864 * VINF_EM_DBG_STEPPED if the event is injected
7865 * directly (register modified by us, not by
7866 * hardware on VM-entry).
7867 * @param puIntrState Pointer to the current guest interruptibility-state.
7868 * This interruptibility-state will be updated if
7869 * necessary. This cannot not be NULL.
7870 */
7871DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7872 bool fStepping, uint32_t *puIntrState)
7873{
7874 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7875 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7876 if (fErrorCodeValid)
7877 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7878 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7879 fStepping, puIntrState);
7880}
7881
7882
7883#if 0 /* unused */
7884/**
7885 * Sets a general-protection (\#GP) exception as pending-for-injection into the
7886 * VM.
7887 *
7888 * @param pVCpu The cross context virtual CPU structure.
7889 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7890 * out-of-sync. Make sure to update the required fields
7891 * before using them.
7892 * @param u32ErrorCode The error code associated with the \#GP.
7893 */
7894DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7895{
7896 NOREF(pMixedCtx);
7897 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7898 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7899 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7900 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7901}
7902#endif /* unused */
7903
7904
7905/**
7906 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7907 *
7908 * @param pVCpu The cross context virtual CPU structure.
7909 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7910 * out-of-sync. Make sure to update the required fields
7911 * before using them.
7912 * @param uVector The software interrupt vector number.
7913 * @param cbInstr The value of RIP that is to be pushed on the guest
7914 * stack.
7915 */
7916DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7917{
7918 NOREF(pMixedCtx);
7919 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7920 if ( uVector == X86_XCPT_BP
7921 || uVector == X86_XCPT_OF)
7922 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7923 else
7924 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7925 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7926}
7927
7928
7929/**
7930 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7931 * stack.
7932 *
7933 * @returns Strict VBox status code (i.e. informational status codes too).
7934 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7935 * @param pVM The cross context VM structure.
7936 * @param pMixedCtx Pointer to the guest-CPU context.
7937 * @param uValue The value to push to the guest stack.
7938 */
7939DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7940{
7941 /*
7942 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7943 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7944 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7945 */
7946 if (pMixedCtx->sp == 1)
7947 return VINF_EM_RESET;
7948 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7949 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7950 AssertRC(rc);
7951 return rc;
7952}
7953
7954
7955/**
7956 * Injects an event into the guest upon VM-entry by updating the relevant fields
7957 * in the VM-entry area in the VMCS.
7958 *
7959 * @returns Strict VBox status code (i.e. informational status codes too).
7960 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7961 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7962 *
7963 * @param pVCpu The cross context virtual CPU structure.
7964 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7965 * be out-of-sync. Make sure to update the required
7966 * fields before using them.
7967 * @param u64IntInfo The VM-entry interruption-information field.
7968 * @param cbInstr The VM-entry instruction length in bytes (for
7969 * software interrupts, exceptions and privileged
7970 * software exceptions).
7971 * @param u32ErrCode The VM-entry exception error code.
7972 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7973 * @param puIntrState Pointer to the current guest interruptibility-state.
7974 * This interruptibility-state will be updated if
7975 * necessary. This cannot not be NULL.
7976 * @param fStepping Whether we're running in
7977 * hmR0VmxRunGuestCodeStep() and should return
7978 * VINF_EM_DBG_STEPPED if the event is injected
7979 * directly (register modified by us, not by
7980 * hardware on VM-entry).
7981 *
7982 * @remarks Requires CR0!
7983 * @remarks No-long-jump zone!!!
7984 */
7985static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7986 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
7987 uint32_t *puIntrState)
7988{
7989 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7990 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7991 Assert(puIntrState);
7992 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7993
7994 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7995 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7996
7997#ifdef VBOX_STRICT
7998 /* Validate the error-code-valid bit for hardware exceptions. */
7999 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
8000 {
8001 switch (uVector)
8002 {
8003 case X86_XCPT_PF:
8004 case X86_XCPT_DF:
8005 case X86_XCPT_TS:
8006 case X86_XCPT_NP:
8007 case X86_XCPT_SS:
8008 case X86_XCPT_GP:
8009 case X86_XCPT_AC:
8010 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
8011 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8012 /* fallthru */
8013 default:
8014 break;
8015 }
8016 }
8017#endif
8018
8019 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
8020 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8021 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
8022
8023 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8024
8025 /* We require CR0 to check if the guest is in real-mode. */
8026 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8027 AssertRCReturn(rc, rc);
8028
8029 /*
8030 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
8031 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
8032 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
8033 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8034 */
8035 if (CPUMIsGuestInRealModeEx(pMixedCtx))
8036 {
8037 PVM pVM = pVCpu->CTX_SUFF(pVM);
8038 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
8039 {
8040 Assert(PDMVmmDevHeapIsEnabled(pVM));
8041 Assert(pVM->hm.s.vmx.pRealModeTSS);
8042
8043 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
8044 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8045 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
8046 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8047 AssertRCReturn(rc, rc);
8048 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
8049
8050 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8051 size_t const cbIdtEntry = sizeof(X86IDTR16);
8052 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
8053 {
8054 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8055 if (uVector == X86_XCPT_DF)
8056 return VINF_EM_RESET;
8057
8058 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
8059 if (uVector == X86_XCPT_GP)
8060 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
8061
8062 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
8063 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
8064 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
8065 fStepping, puIntrState);
8066 }
8067
8068 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8069 uint16_t uGuestIp = pMixedCtx->ip;
8070 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
8071 {
8072 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
8073 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
8074 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8075 }
8076 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
8077 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8078
8079 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8080 X86IDTR16 IdtEntry;
8081 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
8082 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8083 AssertRCReturn(rc, rc);
8084
8085 /* Construct the stack frame for the interrupt/exception handler. */
8086 VBOXSTRICTRC rcStrict;
8087 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
8088 if (rcStrict == VINF_SUCCESS)
8089 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
8090 if (rcStrict == VINF_SUCCESS)
8091 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
8092
8093 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
8094 if (rcStrict == VINF_SUCCESS)
8095 {
8096 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
8097 pMixedCtx->rip = IdtEntry.offSel;
8098 pMixedCtx->cs.Sel = IdtEntry.uSel;
8099 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
8100 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8101 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8102 && uVector == X86_XCPT_PF)
8103 pMixedCtx->cr2 = GCPtrFaultAddress;
8104
8105 /* If any other guest-state bits are changed here, make sure to update
8106 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
8107 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
8108 | HM_CHANGED_GUEST_RIP
8109 | HM_CHANGED_GUEST_RFLAGS
8110 | HM_CHANGED_GUEST_RSP);
8111
8112 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8113 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8114 {
8115 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8116 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8117 Log4(("Clearing inhibition due to STI.\n"));
8118 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
8119 }
8120 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8121 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
8122
8123 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8124 it, if we are returning to ring-3 before executing guest code. */
8125 pVCpu->hm.s.Event.fPending = false;
8126
8127 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8128 if (fStepping)
8129 rcStrict = VINF_EM_DBG_STEPPED;
8130 }
8131 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8132 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8133 return rcStrict;
8134 }
8135
8136 /*
8137 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8138 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8139 */
8140 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8141 }
8142
8143 /* Validate. */
8144 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8145 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8146 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8147
8148 /* Inject. */
8149 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8150 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8151 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8152 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8153
8154 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8155 && uVector == X86_XCPT_PF)
8156 pMixedCtx->cr2 = GCPtrFaultAddress;
8157
8158 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8159 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8160
8161 AssertRCReturn(rc, rc);
8162 return VINF_SUCCESS;
8163}
8164
8165
8166/**
8167 * Clears the interrupt-window exiting control in the VMCS and if necessary
8168 * clears the current event in the VMCS as well.
8169 *
8170 * @returns VBox status code.
8171 * @param pVCpu The cross context virtual CPU structure.
8172 *
8173 * @remarks Use this function only to clear events that have not yet been
8174 * delivered to the guest but are injected in the VMCS!
8175 * @remarks No-long-jump zone!!!
8176 */
8177static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8178{
8179 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8180
8181 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8182 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8183
8184 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8185 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8186}
8187
8188
8189/**
8190 * Enters the VT-x session.
8191 *
8192 * @returns VBox status code.
8193 * @param pVM The cross context VM structure.
8194 * @param pVCpu The cross context virtual CPU structure.
8195 * @param pCpu Pointer to the CPU info struct.
8196 */
8197VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8198{
8199 AssertPtr(pVM);
8200 AssertPtr(pVCpu);
8201 Assert(pVM->hm.s.vmx.fSupported);
8202 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8203 NOREF(pCpu); NOREF(pVM);
8204
8205 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8206 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8207
8208#ifdef VBOX_STRICT
8209 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8210 RTCCUINTREG uHostCR4 = ASMGetCR4();
8211 if (!(uHostCR4 & X86_CR4_VMXE))
8212 {
8213 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8214 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8215 }
8216#endif
8217
8218 /*
8219 * Load the VCPU's VMCS as the current (and active) one.
8220 */
8221 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8222 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8223 if (RT_FAILURE(rc))
8224 return rc;
8225
8226 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8227 pVCpu->hm.s.fLeaveDone = false;
8228 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8229
8230 return VINF_SUCCESS;
8231}
8232
8233
8234/**
8235 * The thread-context callback (only on platforms which support it).
8236 *
8237 * @param enmEvent The thread-context event.
8238 * @param pVCpu The cross context virtual CPU structure.
8239 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8240 * @thread EMT(pVCpu)
8241 */
8242VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8243{
8244 NOREF(fGlobalInit);
8245
8246 switch (enmEvent)
8247 {
8248 case RTTHREADCTXEVENT_OUT:
8249 {
8250 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8251 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8252 VMCPU_ASSERT_EMT(pVCpu);
8253
8254 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8255
8256 /* No longjmps (logger flushes, locks) in this fragile context. */
8257 VMMRZCallRing3Disable(pVCpu);
8258 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8259
8260 /*
8261 * Restore host-state (FPU, debug etc.)
8262 */
8263 if (!pVCpu->hm.s.fLeaveDone)
8264 {
8265 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8266 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8267 hmR0VmxLeave(pVCpu, pMixedCtx, false /* fSaveGuestState */);
8268 pVCpu->hm.s.fLeaveDone = true;
8269 }
8270
8271 /* Leave HM context, takes care of local init (term). */
8272 int rc = HMR0LeaveCpu(pVCpu);
8273 AssertRC(rc); NOREF(rc);
8274
8275 /* Restore longjmp state. */
8276 VMMRZCallRing3Enable(pVCpu);
8277 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8278 break;
8279 }
8280
8281 case RTTHREADCTXEVENT_IN:
8282 {
8283 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8284 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8285 VMCPU_ASSERT_EMT(pVCpu);
8286
8287 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8288 VMMRZCallRing3Disable(pVCpu);
8289 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8290
8291 /* Initialize the bare minimum state required for HM. This takes care of
8292 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8293 int rc = HMR0EnterCpu(pVCpu);
8294 AssertRC(rc);
8295 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8296
8297 /* Load the active VMCS as the current one. */
8298 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8299 {
8300 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8301 AssertRC(rc); NOREF(rc);
8302 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8303 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8304 }
8305 pVCpu->hm.s.fLeaveDone = false;
8306
8307 /* Restore longjmp state. */
8308 VMMRZCallRing3Enable(pVCpu);
8309 break;
8310 }
8311
8312 default:
8313 break;
8314 }
8315}
8316
8317
8318/**
8319 * Saves the host state in the VMCS host-state.
8320 * Sets up the VM-exit MSR-load area.
8321 *
8322 * The CPU state will be loaded from these fields on every successful VM-exit.
8323 *
8324 * @returns VBox status code.
8325 * @param pVM The cross context VM structure.
8326 * @param pVCpu The cross context virtual CPU structure.
8327 *
8328 * @remarks No-long-jump zone!!!
8329 */
8330static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8331{
8332 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8333
8334 int rc = VINF_SUCCESS;
8335 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8336 {
8337 rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8338 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8339
8340 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8341 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8342
8343 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8344 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8345
8346 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8347 }
8348 return rc;
8349}
8350
8351
8352/**
8353 * Saves the host state in the VMCS host-state.
8354 *
8355 * @returns VBox status code.
8356 * @param pVM The cross context VM structure.
8357 * @param pVCpu The cross context virtual CPU structure.
8358 *
8359 * @remarks No-long-jump zone!!!
8360 */
8361VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8362{
8363 AssertPtr(pVM);
8364 AssertPtr(pVCpu);
8365
8366 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8367
8368 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8369 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8370 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8371 return hmR0VmxSaveHostState(pVM, pVCpu);
8372}
8373
8374
8375/**
8376 * Loads the guest state into the VMCS guest-state area.
8377 *
8378 * The will typically be done before VM-entry when the guest-CPU state and the
8379 * VMCS state may potentially be out of sync.
8380 *
8381 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8382 * VM-entry controls.
8383 * Sets up the appropriate VMX non-root function to execute guest code based on
8384 * the guest CPU mode.
8385 *
8386 * @returns VBox strict status code.
8387 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8388 * without unrestricted guest access and the VMMDev is not presently
8389 * mapped (e.g. EFI32).
8390 *
8391 * @param pVM The cross context VM structure.
8392 * @param pVCpu The cross context virtual CPU structure.
8393 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8394 * out-of-sync. Make sure to update the required fields
8395 * before using them.
8396 *
8397 * @remarks No-long-jump zone!!! (Disables and enables long jmps for itself,
8398 * caller disables then again on successfull return. Confusing.)
8399 */
8400static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8401{
8402 AssertPtr(pVM);
8403 AssertPtr(pVCpu);
8404 AssertPtr(pMixedCtx);
8405 HMVMX_ASSERT_PREEMPT_SAFE();
8406
8407 VMMRZCallRing3Disable(pVCpu);
8408 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8409
8410 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8411
8412 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8413
8414 /* Determine real-on-v86 mode. */
8415 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8416 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8417 && CPUMIsGuestInRealModeEx(pMixedCtx))
8418 {
8419 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8420 }
8421
8422 /*
8423 * Load the guest-state into the VMCS.
8424 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8425 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8426 */
8427 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8428 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8429
8430 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8431 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8432 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8433
8434 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8435 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8436 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8437
8438 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8439 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8440
8441 VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8442 if (rcStrict == VINF_SUCCESS)
8443 { /* likely */ }
8444 else
8445 {
8446 VMMRZCallRing3Enable(pVCpu);
8447 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8448 return rcStrict;
8449 }
8450
8451 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8452 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8453 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8454
8455 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8456 determine we don't have to swap EFER after all. */
8457 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8458 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8459
8460 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8461 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8462
8463 rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8464 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8465
8466 /*
8467 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8468 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8469 */
8470 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8471 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8472
8473 /* Clear any unused and reserved bits. */
8474 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8475
8476 VMMRZCallRing3Enable(pVCpu);
8477
8478 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8479 return rc;
8480}
8481
8482
8483/**
8484 * Loads the state shared between the host and guest into the VMCS.
8485 *
8486 * @param pVM The cross context VM structure.
8487 * @param pVCpu The cross context virtual CPU structure.
8488 * @param pCtx Pointer to the guest-CPU context.
8489 *
8490 * @remarks No-long-jump zone!!!
8491 */
8492static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8493{
8494 NOREF(pVM);
8495
8496 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8497 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8498
8499 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8500 {
8501 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8502 AssertRC(rc);
8503 }
8504
8505 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8506 {
8507 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8508 AssertRC(rc);
8509
8510 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8511 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8512 {
8513 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8514 AssertRC(rc);
8515 }
8516 }
8517
8518 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8519 {
8520 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8521 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8522 }
8523
8524 /* Loading CR0, debug state might have changed intercepts, update VMCS. */
8525 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8526 {
8527 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8528 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8529 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8530 AssertRC(rc);
8531 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8532 }
8533
8534 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8535 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8536}
8537
8538
8539/**
8540 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8541 *
8542 * @returns Strict VBox status code (i.e. informational status codes too).
8543 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8544 * without unrestricted guest access and the VMMDev is not presently
8545 * mapped (e.g. EFI32).
8546 *
8547 * @param pVM The cross context VM structure.
8548 * @param pVCpu The cross context virtual CPU structure.
8549 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8550 * out-of-sync. Make sure to update the required fields
8551 * before using them.
8552 */
8553static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8554{
8555 HMVMX_ASSERT_PREEMPT_SAFE();
8556
8557 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8558#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8559 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8560#endif
8561
8562 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8563 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8564 {
8565 rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8566 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8567 { /* likely */}
8568 else
8569 {
8570 AssertMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8571 VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8572 }
8573 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8574 }
8575 else if (HMCPU_CF_VALUE(pVCpu))
8576 {
8577 rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8578 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8579 { /* likely */}
8580 else
8581 {
8582 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8583 ("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8584 return rcStrict;
8585 }
8586 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8587 }
8588
8589 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8590 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8591 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8592 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8593 return rcStrict;
8594}
8595
8596
8597/**
8598 * Does the preparations before executing guest code in VT-x.
8599 *
8600 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8601 * recompiler/IEM. We must be cautious what we do here regarding committing
8602 * guest-state information into the VMCS assuming we assuredly execute the
8603 * guest in VT-x mode.
8604 *
8605 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8606 * the common-state (TRPM/forceflags), we must undo those changes so that the
8607 * recompiler/IEM can (and should) use them when it resumes guest execution.
8608 * Otherwise such operations must be done when we can no longer exit to ring-3.
8609 *
8610 * @returns Strict VBox status code (i.e. informational status codes too).
8611 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8612 * have been disabled.
8613 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8614 * double-fault into the guest.
8615 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8616 * dispatched directly.
8617 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8618 *
8619 * @param pVM The cross context VM structure.
8620 * @param pVCpu The cross context virtual CPU structure.
8621 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8622 * out-of-sync. Make sure to update the required fields
8623 * before using them.
8624 * @param pVmxTransient Pointer to the VMX transient structure.
8625 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8626 * us ignore some of the reasons for returning to
8627 * ring-3, and return VINF_EM_DBG_STEPPED if event
8628 * dispatching took place.
8629 */
8630static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8631{
8632 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8633
8634#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8635 PGMRZDynMapFlushAutoSet(pVCpu);
8636#endif
8637
8638 /* Check force flag actions that might require us to go back to ring-3. */
8639 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8640 if (rcStrict == VINF_SUCCESS)
8641 { /* FFs doesn't get set all the time. */ }
8642 else
8643 return rcStrict;
8644
8645 if (TRPMHasTrap(pVCpu))
8646 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8647 uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8648
8649 /*
8650 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8651 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8652 */
8653 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
8654 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8655 { /* likely */ }
8656 else
8657 {
8658 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8659 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8660 return rcStrict;
8661 }
8662
8663 /*
8664 * Load the guest state bits, we can handle longjmps/getting preempted here.
8665 *
8666 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8667 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8668 * Hence, this needs to be done -after- injection of events.
8669 */
8670 rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8671 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8672 { /* likely */ }
8673 else
8674 return rcStrict;
8675
8676 /*
8677 * No longjmps to ring-3 from this point on!!!
8678 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8679 * This also disables flushing of the R0-logger instance (if any).
8680 */
8681 VMMRZCallRing3Disable(pVCpu);
8682
8683 /*
8684 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8685 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8686 *
8687 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8688 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8689 *
8690 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8691 * executing guest code.
8692 */
8693 pVmxTransient->fEFlags = ASMIntDisableFlags();
8694
8695 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8696 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8697 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8698 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8699 {
8700 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8701 {
8702 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8703 pVCpu->hm.s.Event.fPending = false;
8704
8705 return VINF_SUCCESS;
8706 }
8707
8708 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8709 rcStrict = VINF_EM_RAW_INTERRUPT;
8710 }
8711 else
8712 {
8713 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8714 rcStrict = VINF_EM_RAW_TO_R3;
8715 }
8716
8717 ASMSetFlags(pVmxTransient->fEFlags);
8718 VMMRZCallRing3Enable(pVCpu);
8719
8720 return rcStrict;
8721}
8722
8723
8724/**
8725 * Prepares to run guest code in VT-x and we've committed to doing so. This
8726 * means there is no backing out to ring-3 or anywhere else at this
8727 * point.
8728 *
8729 * @param pVM The cross context VM structure.
8730 * @param pVCpu The cross context virtual CPU structure.
8731 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8732 * out-of-sync. Make sure to update the required fields
8733 * before using them.
8734 * @param pVmxTransient Pointer to the VMX transient structure.
8735 *
8736 * @remarks Called with preemption disabled.
8737 * @remarks No-long-jump zone!!!
8738 */
8739static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8740{
8741 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8742 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8743 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8744
8745 /*
8746 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8747 */
8748 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8749 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8750
8751#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8752 if (!CPUMIsGuestFPUStateActive(pVCpu))
8753 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8754 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8755 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8756#endif
8757
8758 if ( pVCpu->hm.s.fPreloadGuestFpu
8759 && !CPUMIsGuestFPUStateActive(pVCpu))
8760 {
8761 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8762 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8763 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8764 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8765 }
8766
8767 /*
8768 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8769 */
8770 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8771 && pVCpu->hm.s.vmx.cMsrs > 0)
8772 {
8773 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8774 }
8775
8776 /*
8777 * Load the host state bits as we may've been preempted (only happens when
8778 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8779 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8780 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8781 * See @bugref{8432}.
8782 */
8783 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8784 {
8785 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8786 AssertRC(rc);
8787 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
8788 }
8789 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8790
8791 /*
8792 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8793 */
8794 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8795 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8796 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8797
8798 /* Store status of the shared guest-host state at the time of VM-entry. */
8799#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8800 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8801 {
8802 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8803 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8804 }
8805 else
8806#endif
8807 {
8808 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8809 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8810 }
8811 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8812
8813 /*
8814 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8815 */
8816 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8817 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8818
8819 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8820 RTCPUID idCurrentCpu = pCpu->idCpu;
8821 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8822 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8823 {
8824 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8825 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8826 }
8827
8828 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8829 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8830 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8831 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8832
8833 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8834
8835 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8836 to start executing. */
8837
8838 /*
8839 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8840 */
8841 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8842 {
8843 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8844 {
8845 bool fMsrUpdated;
8846 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8847 AssertRC(rc2);
8848 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8849
8850 rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8851 &fMsrUpdated);
8852 AssertRC(rc2);
8853 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8854
8855 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8856 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8857 }
8858 else
8859 {
8860 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8861 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8862 }
8863 }
8864
8865#ifdef VBOX_STRICT
8866 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8867 hmR0VmxCheckHostEferMsr(pVCpu);
8868 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8869#endif
8870#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8871 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8872 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8873 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8874#endif
8875}
8876
8877
8878/**
8879 * Performs some essential restoration of state after running guest code in
8880 * VT-x.
8881 *
8882 * @param pVM The cross context VM structure.
8883 * @param pVCpu The cross context virtual CPU structure.
8884 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8885 * out-of-sync. Make sure to update the required fields
8886 * before using them.
8887 * @param pVmxTransient Pointer to the VMX transient structure.
8888 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8889 *
8890 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8891 *
8892 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8893 * unconditionally when it is safe to do so.
8894 */
8895static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8896{
8897 NOREF(pVM);
8898
8899 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8900
8901 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8902 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8903 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8904 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8905 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8906 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8907
8908 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8909 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
8910
8911 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8912 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8913 Assert(!ASMIntAreEnabled());
8914 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8915
8916#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8917 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVM, pVCpu))
8918 {
8919 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8920 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8921 }
8922#endif
8923
8924#if HC_ARCH_BITS == 64
8925 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8926#endif
8927#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8928 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8929 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8930 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8931#else
8932 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8933#endif
8934#ifdef VBOX_STRICT
8935 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8936#endif
8937 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8938 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8939
8940 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8941 uint32_t uExitReason;
8942 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8943 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8944 AssertRC(rc);
8945 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8946 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8947
8948 /* Update the VM-exit history array. */
8949 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8950
8951 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8952 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8953 {
8954 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8955 pVmxTransient->fVMEntryFailed));
8956 return;
8957 }
8958
8959 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8960 {
8961 /** @todo We can optimize this by only syncing with our force-flags when
8962 * really needed and keeping the VMCS state as it is for most
8963 * VM-exits. */
8964 /* Update the guest interruptibility-state from the VMCS. */
8965 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8966
8967#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8968 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8969 AssertRC(rc);
8970#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8971 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8972 AssertRC(rc);
8973#endif
8974
8975 /*
8976 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8977 * we eventually get a VM-exit for any reason.
8978 *
8979 * This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is why it's done here as it's easier and
8980 * no less efficient to deal with it here than making hmR0VmxSaveGuestState() cope with longjmps safely
8981 * (see VMCPU_FF_HM_UPDATE_CR3 handling).
8982 */
8983 /** @todo r=ramshankar: The 2nd para in the above comment is
8984 * outdated, we no longer longjmp to ring-3 on setting
8985 * the TPR, but regardless we can probably rework this
8986 * portion of the code a bit. */
8987 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8988 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8989 {
8990 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8991 AssertRC(rc);
8992 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8993 }
8994 }
8995}
8996
8997
8998/**
8999 * Runs the guest code using VT-x the normal way.
9000 *
9001 * @returns VBox status code.
9002 * @param pVM The cross context VM structure.
9003 * @param pVCpu The cross context virtual CPU structure.
9004 * @param pCtx Pointer to the guest-CPU context.
9005 *
9006 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
9007 */
9008static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9009{
9010 VMXTRANSIENT VmxTransient;
9011 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9012 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9013 uint32_t cLoops = 0;
9014
9015 for (;; cLoops++)
9016 {
9017 Assert(!HMR0SuspendPending());
9018 HMVMX_ASSERT_CPU_SAFE();
9019
9020 /* Preparatory work for running guest code, this may force us to return
9021 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
9022 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9023 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
9024 if (rcStrict != VINF_SUCCESS)
9025 break;
9026
9027 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9028 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9029 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9030
9031 /* Restore any residual host-state and save any bits shared between host
9032 and guest into the guest-CPU state. Re-enables interrupts! */
9033 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
9034
9035 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9036 if (RT_SUCCESS(rcRun))
9037 { /* very likely */ }
9038 else
9039 {
9040 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9041 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9042 return rcRun;
9043 }
9044
9045 /* Profile the VM-exit. */
9046 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9047 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9048 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9049 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9050 HMVMX_START_EXIT_DISPATCH_PROF();
9051
9052 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9053
9054 /* Handle the VM-exit. */
9055#ifdef HMVMX_USE_FUNCTION_TABLE
9056 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
9057#else
9058 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
9059#endif
9060 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9061 if (rcStrict == VINF_SUCCESS)
9062 {
9063 if (cLoops <= pVM->hm.s.cMaxResumeLoops)
9064 continue; /* likely */
9065 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9066 rcStrict = VINF_EM_RAW_INTERRUPT;
9067 }
9068 break;
9069 }
9070
9071 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9072 return rcStrict;
9073}
9074
9075
9076
9077/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
9078 * probes.
9079 *
9080 * The following few functions and associated structure contains the bloat
9081 * necessary for providing detailed debug events and dtrace probes as well as
9082 * reliable host side single stepping. This works on the principle of
9083 * "subclassing" the normal execution loop and workers. We replace the loop
9084 * method completely and override selected helpers to add necessary adjustments
9085 * to their core operation.
9086 *
9087 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
9088 * any performance for debug and analysis features.
9089 *
9090 * @{
9091 */
9092
9093/**
9094 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
9095 * the debug run loop.
9096 */
9097typedef struct VMXRUNDBGSTATE
9098{
9099 /** The RIP we started executing at. This is for detecting that we stepped. */
9100 uint64_t uRipStart;
9101 /** The CS we started executing with. */
9102 uint16_t uCsStart;
9103
9104 /** Whether we've actually modified the 1st execution control field. */
9105 bool fModifiedProcCtls : 1;
9106 /** Whether we've actually modified the 2nd execution control field. */
9107 bool fModifiedProcCtls2 : 1;
9108 /** Whether we've actually modified the exception bitmap. */
9109 bool fModifiedXcptBitmap : 1;
9110
9111 /** We desire the modified the CR0 mask to be cleared. */
9112 bool fClearCr0Mask : 1;
9113 /** We desire the modified the CR4 mask to be cleared. */
9114 bool fClearCr4Mask : 1;
9115 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9116 uint32_t fCpe1Extra;
9117 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9118 uint32_t fCpe1Unwanted;
9119 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9120 uint32_t fCpe2Extra;
9121 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9122 uint32_t bmXcptExtra;
9123 /** The sequence number of the Dtrace provider settings the state was
9124 * configured against. */
9125 uint32_t uDtraceSettingsSeqNo;
9126 /** VM-exits to check (one bit per VM-exit). */
9127 uint32_t bmExitsToCheck[3];
9128
9129 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9130 uint32_t fProcCtlsInitial;
9131 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9132 uint32_t fProcCtls2Initial;
9133 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9134 uint32_t bmXcptInitial;
9135} VMXRUNDBGSTATE;
9136AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9137typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9138
9139
9140/**
9141 * Initializes the VMXRUNDBGSTATE structure.
9142 *
9143 * @param pVCpu The cross context virtual CPU structure of the
9144 * calling EMT.
9145 * @param pCtx The CPU register context to go with @a pVCpu.
9146 * @param pDbgState The structure to initialize.
9147 */
9148DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9149{
9150 pDbgState->uRipStart = pCtx->rip;
9151 pDbgState->uCsStart = pCtx->cs.Sel;
9152
9153 pDbgState->fModifiedProcCtls = false;
9154 pDbgState->fModifiedProcCtls2 = false;
9155 pDbgState->fModifiedXcptBitmap = false;
9156 pDbgState->fClearCr0Mask = false;
9157 pDbgState->fClearCr4Mask = false;
9158 pDbgState->fCpe1Extra = 0;
9159 pDbgState->fCpe1Unwanted = 0;
9160 pDbgState->fCpe2Extra = 0;
9161 pDbgState->bmXcptExtra = 0;
9162 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9163 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9164 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9165}
9166
9167
9168/**
9169 * Updates the VMSC fields with changes requested by @a pDbgState.
9170 *
9171 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9172 * immediately before executing guest code, i.e. when interrupts are disabled.
9173 * We don't check status codes here as we cannot easily assert or return in the
9174 * latter case.
9175 *
9176 * @param pVCpu The cross context virtual CPU structure.
9177 * @param pDbgState The debug state.
9178 */
9179DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9180{
9181 /*
9182 * Ensure desired flags in VMCS control fields are set.
9183 * (Ignoring write failure here, as we're committed and it's just debug extras.)
9184 *
9185 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9186 * there should be no stale data in pCtx at this point.
9187 */
9188 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9189 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9190 {
9191 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
9192 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9193 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9194 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9195 pDbgState->fModifiedProcCtls = true;
9196 }
9197
9198 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9199 {
9200 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
9201 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9202 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9203 pDbgState->fModifiedProcCtls2 = true;
9204 }
9205
9206 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9207 {
9208 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
9209 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9210 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9211 pDbgState->fModifiedXcptBitmap = true;
9212 }
9213
9214 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9215 {
9216 pVCpu->hm.s.vmx.u32CR0Mask = 0;
9217 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9218 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9219 }
9220
9221 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9222 {
9223 pVCpu->hm.s.vmx.u32CR4Mask = 0;
9224 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9225 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9226 }
9227}
9228
9229
9230DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9231{
9232 /*
9233 * Restore VM-exit control settings as we may not reenter this function the
9234 * next time around.
9235 */
9236 /* We reload the initial value, trigger what we can of recalculations the
9237 next time around. From the looks of things, that's all that's required atm. */
9238 if (pDbgState->fModifiedProcCtls)
9239 {
9240 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9241 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9242 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9243 AssertRCReturn(rc2, rc2);
9244 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9245 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0 | HM_CHANGED_GUEST_DEBUG);
9246 }
9247
9248 /* We're currently the only ones messing with this one, so just restore the
9249 cached value and reload the field. */
9250 if ( pDbgState->fModifiedProcCtls2
9251 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9252 {
9253 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9254 AssertRCReturn(rc2, rc2);
9255 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9256 }
9257
9258 /* If we've modified the exception bitmap, we restore it and trigger
9259 reloading and partial recalculation the next time around. */
9260 if (pDbgState->fModifiedXcptBitmap)
9261 {
9262 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9263 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS | HM_CHANGED_GUEST_CR0);
9264 }
9265
9266 /* We assume hmR0VmxLoadSharedCR0 will recalculate and load the CR0 mask. */
9267 if (pDbgState->fClearCr0Mask)
9268 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9269
9270 /* We assume hmR0VmxLoadGuestCR3AndCR4 will recalculate and load the CR4 mask. */
9271 if (pDbgState->fClearCr4Mask)
9272 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9273
9274 return rcStrict;
9275}
9276
9277
9278/**
9279 * Configures VM-exit controls for current DBGF and DTrace settings.
9280 *
9281 * This updates @a pDbgState and the VMCS execution control fields to reflect
9282 * the necessary VM-exits demanded by DBGF and DTrace.
9283 *
9284 * @param pVM The cross context VM structure.
9285 * @param pVCpu The cross context virtual CPU structure.
9286 * @param pCtx Pointer to the guest-CPU context.
9287 * @param pDbgState The debug state.
9288 * @param pVmxTransient Pointer to the VMX transient structure. May update
9289 * fUpdateTscOffsettingAndPreemptTimer.
9290 */
9291static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9292 PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9293{
9294 /*
9295 * Take down the dtrace serial number so we can spot changes.
9296 */
9297 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9298 ASMCompilerBarrier();
9299
9300 /*
9301 * We'll rebuild most of the middle block of data members (holding the
9302 * current settings) as we go along here, so start by clearing it all.
9303 */
9304 pDbgState->bmXcptExtra = 0;
9305 pDbgState->fCpe1Extra = 0;
9306 pDbgState->fCpe1Unwanted = 0;
9307 pDbgState->fCpe2Extra = 0;
9308 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9309 pDbgState->bmExitsToCheck[i] = 0;
9310
9311 /*
9312 * Software interrupts (INT XXh) - no idea how to trigger these...
9313 */
9314 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9315 || VBOXVMM_INT_SOFTWARE_ENABLED())
9316 {
9317 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9318 }
9319
9320 /*
9321 * INT3 breakpoints - triggered by #BP exceptions.
9322 */
9323 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9324 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9325
9326 /*
9327 * Exception bitmap and XCPT events+probes.
9328 */
9329 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9330 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9331 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9332
9333 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9334 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9335 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9336 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9337 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9338 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9339 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9340 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9341 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9342 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9343 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9344 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9345 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9346 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9347 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9348 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9349 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9350 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9351
9352 if (pDbgState->bmXcptExtra)
9353 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9354
9355 /*
9356 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9357 *
9358 * Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9359 * So, when adding/changing/removing please don't forget to update it.
9360 *
9361 * Some of the macros are picking up local variables to save horizontal space,
9362 * (being able to see it in a table is the lesser evil here).
9363 */
9364#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9365 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9366 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9367#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9368 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9369 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9370 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9371 } else do { } while (0)
9372#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9373 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9374 { \
9375 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9376 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9377 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9378 } else do { } while (0)
9379#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9380 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9381 { \
9382 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9383 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9384 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9385 } else do { } while (0)
9386#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9387 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9388 { \
9389 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9390 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9391 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9392 } else do { } while (0)
9393
9394 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9395 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9396 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9397 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9398 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9399
9400 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9401 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9402 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9403 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9404 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9405 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9406 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9407 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9408 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9409 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9410 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9411 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9412 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9413 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9414 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9415 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9416 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9417 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9418 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9419 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9420 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9421 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9422 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9423 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9424 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9425 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9426 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9427 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9428 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9429 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9430 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9431 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9432 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9433 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9434 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9435 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9436
9437 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9438 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9439 {
9440 int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9441 rc2 |= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9442 rc2 |= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9443 AssertRC(rc2);
9444
9445#if 0 /** @todo fix me */
9446 pDbgState->fClearCr0Mask = true;
9447 pDbgState->fClearCr4Mask = true;
9448#endif
9449 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9450 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9451 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9452 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9453 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9454 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9455 require clearing here and in the loop if we start using it. */
9456 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9457 }
9458 else
9459 {
9460 if (pDbgState->fClearCr0Mask)
9461 {
9462 pDbgState->fClearCr0Mask = false;
9463 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9464 }
9465 if (pDbgState->fClearCr4Mask)
9466 {
9467 pDbgState->fClearCr4Mask = false;
9468 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9469 }
9470 }
9471 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9472 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9473
9474 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9475 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9476 {
9477 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9478 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9479 }
9480 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9481 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9482
9483 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9484 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9485 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9486 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9487 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9488 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9489 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9490 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9491#if 0 /** @todo too slow, fix handler. */
9492 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9493#endif
9494 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9495
9496 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9497 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9498 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9499 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9500 {
9501 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9502 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9503 }
9504 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9505 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9506 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9507 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9508
9509 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9510 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9511 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9512 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9513 {
9514 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9515 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9516 }
9517 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9518 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9519 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9520 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9521
9522 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9523 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9524 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9525 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9526 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9527 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9528 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9529 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9530 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9531 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9532 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9533 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9534 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9535 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9536 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9537 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9538 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9539 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9540 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9541 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9542 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9543 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9544
9545#undef IS_EITHER_ENABLED
9546#undef SET_ONLY_XBM_IF_EITHER_EN
9547#undef SET_CPE1_XBM_IF_EITHER_EN
9548#undef SET_CPEU_XBM_IF_EITHER_EN
9549#undef SET_CPE2_XBM_IF_EITHER_EN
9550
9551 /*
9552 * Sanitize the control stuff.
9553 */
9554 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9555 if (pDbgState->fCpe2Extra)
9556 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9557 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9558 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9559 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9560 {
9561 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9562 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9563 }
9564
9565 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9566 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9567 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9568 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9569}
9570
9571
9572/**
9573 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9574 * appropriate.
9575 *
9576 * The caller has checked the VM-exit against the
9577 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9578 * already, so we don't have to do that either.
9579 *
9580 * @returns Strict VBox status code (i.e. informational status codes too).
9581 * @param pVM The cross context VM structure.
9582 * @param pVCpu The cross context virtual CPU structure.
9583 * @param pMixedCtx Pointer to the guest-CPU context.
9584 * @param pVmxTransient Pointer to the VMX-transient structure.
9585 * @param uExitReason The VM-exit reason.
9586 *
9587 * @remarks The name of this function is displayed by dtrace, so keep it short
9588 * and to the point. No longer than 33 chars long, please.
9589 */
9590static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9591 PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9592{
9593 /*
9594 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9595 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9596 *
9597 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9598 * does. Must add/change/remove both places. Same ordering, please.
9599 *
9600 * Added/removed events must also be reflected in the next section
9601 * where we dispatch dtrace events.
9602 */
9603 bool fDtrace1 = false;
9604 bool fDtrace2 = false;
9605 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9606 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9607 uint32_t uEventArg = 0;
9608#define SET_EXIT(a_EventSubName) \
9609 do { \
9610 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9611 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9612 } while (0)
9613#define SET_BOTH(a_EventSubName) \
9614 do { \
9615 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9616 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9617 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9618 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9619 } while (0)
9620 switch (uExitReason)
9621 {
9622 case VMX_EXIT_MTF:
9623 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9624
9625 case VMX_EXIT_XCPT_OR_NMI:
9626 {
9627 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9628 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9629 {
9630 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9631 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9632 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9633 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9634 {
9635 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9636 {
9637 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9638 uEventArg = pVmxTransient->uExitIntErrorCode;
9639 }
9640 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9641 switch (enmEvent1)
9642 {
9643 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9644 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9645 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9646 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9647 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9648 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9649 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9650 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9651 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9652 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9653 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9654 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9655 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9656 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9657 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9658 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9659 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9660 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9661 default: break;
9662 }
9663 }
9664 else
9665 AssertFailed();
9666 break;
9667
9668 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9669 uEventArg = idxVector;
9670 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9671 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9672 break;
9673 }
9674 break;
9675 }
9676
9677 case VMX_EXIT_TRIPLE_FAULT:
9678 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9679 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9680 break;
9681 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9682 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9683 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9684 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9685 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9686
9687 /* Instruction specific VM-exits: */
9688 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9689 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9690 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9691 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9692 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9693 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9694 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9695 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9696 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9697 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9698 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9699 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9700 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9701 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9702 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9703 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9704 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9705 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9706 case VMX_EXIT_MOV_CRX:
9707 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9708/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
9709* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
9710 if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
9711 == VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
9712 SET_BOTH(CRX_READ);
9713 else
9714 SET_BOTH(CRX_WRITE);
9715 uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
9716 break;
9717 case VMX_EXIT_MOV_DRX:
9718 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9719 if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
9720 == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
9721 SET_BOTH(DRX_READ);
9722 else
9723 SET_BOTH(DRX_WRITE);
9724 uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
9725 break;
9726 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9727 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9728 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9729 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9730 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9731 case VMX_EXIT_XDTR_ACCESS:
9732 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9733 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9734 {
9735 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9736 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9737 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9738 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9739 }
9740 break;
9741
9742 case VMX_EXIT_TR_ACCESS:
9743 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9744 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9745 {
9746 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9747 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9748 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9749 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9750 }
9751 break;
9752
9753 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9754 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9755 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9756 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9757 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9758 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9759 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9760 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9761 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9762 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9763 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9764
9765 /* Events that aren't relevant at this point. */
9766 case VMX_EXIT_EXT_INT:
9767 case VMX_EXIT_INT_WINDOW:
9768 case VMX_EXIT_NMI_WINDOW:
9769 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9770 case VMX_EXIT_PREEMPT_TIMER:
9771 case VMX_EXIT_IO_INSTR:
9772 break;
9773
9774 /* Errors and unexpected events. */
9775 case VMX_EXIT_INIT_SIGNAL:
9776 case VMX_EXIT_SIPI:
9777 case VMX_EXIT_IO_SMI:
9778 case VMX_EXIT_SMI:
9779 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9780 case VMX_EXIT_ERR_MSR_LOAD:
9781 case VMX_EXIT_ERR_MACHINE_CHECK:
9782 break;
9783
9784 default:
9785 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9786 break;
9787 }
9788#undef SET_BOTH
9789#undef SET_EXIT
9790
9791 /*
9792 * Dtrace tracepoints go first. We do them here at once so we don't
9793 * have to copy the guest state saving and stuff a few dozen times.
9794 * Down side is that we've got to repeat the switch, though this time
9795 * we use enmEvent since the probes are a subset of what DBGF does.
9796 */
9797 if (fDtrace1 || fDtrace2)
9798 {
9799 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9800 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9801 switch (enmEvent1)
9802 {
9803 /** @todo consider which extra parameters would be helpful for each probe. */
9804 case DBGFEVENT_END: break;
9805 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9806 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9807 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9808 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9809 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9810 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9811 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9812 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9813 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9814 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9815 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9816 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9817 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9818 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9819 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9820 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9821 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9822 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9823 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9824 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9825 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9826 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9827 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9828 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9829 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9830 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9831 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9832 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9833 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9834 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9835 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9836 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9837 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9838 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9839 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9840 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9841 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9842 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9843 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9844 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9845 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9846 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9847 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9848 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9849 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9850 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9851 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9852 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9853 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9854 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9855 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9856 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9857 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9858 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9859 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9860 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9861 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9862 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9863 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9864 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9865 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9866 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9867 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9868 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9869 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9870 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9871 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9872 }
9873 switch (enmEvent2)
9874 {
9875 /** @todo consider which extra parameters would be helpful for each probe. */
9876 case DBGFEVENT_END: break;
9877 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9878 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9879 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9880 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9881 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9882 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9883 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9884 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9885 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9886 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9887 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9888 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9889 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9890 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9891 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9892 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9893 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9894 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9895 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9896 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9897 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9898 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9899 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9900 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9901 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9902 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9903 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9904 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9905 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9906 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9907 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9908 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9909 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9910 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9911 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9912 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9913 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9914 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9915 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9916 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9917 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9918 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9919 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9920 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9921 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9922 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9923 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9924 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9925 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9926 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9927 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9928 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9929 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9930 }
9931 }
9932
9933 /*
9934 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9935 * the DBGF call will do a full check).
9936 *
9937 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9938 * Note! If we have to events, we prioritize the first, i.e. the instruction
9939 * one, in order to avoid event nesting.
9940 */
9941 if ( enmEvent1 != DBGFEVENT_END
9942 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9943 {
9944 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9945 if (rcStrict != VINF_SUCCESS)
9946 return rcStrict;
9947 }
9948 else if ( enmEvent2 != DBGFEVENT_END
9949 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9950 {
9951 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9952 if (rcStrict != VINF_SUCCESS)
9953 return rcStrict;
9954 }
9955
9956 return VINF_SUCCESS;
9957}
9958
9959
9960/**
9961 * Single-stepping VM-exit filtering.
9962 *
9963 * This is preprocessing the VM-exits and deciding whether we've gotten far
9964 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9965 * handling is performed.
9966 *
9967 * @returns Strict VBox status code (i.e. informational status codes too).
9968 * @param pVM The cross context VM structure.
9969 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9970 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9971 * out-of-sync. Make sure to update the required
9972 * fields before using them.
9973 * @param pVmxTransient Pointer to the VMX-transient structure.
9974 * @param uExitReason The VM-exit reason.
9975 * @param pDbgState The debug state.
9976 */
9977DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9978 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9979{
9980 /*
9981 * Expensive (saves context) generic dtrace VM-exit probe.
9982 */
9983 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9984 { /* more likely */ }
9985 else
9986 {
9987 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9988 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9989 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9990 }
9991
9992 /*
9993 * Check for host NMI, just to get that out of the way.
9994 */
9995 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9996 { /* normally likely */ }
9997 else
9998 {
9999 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10000 AssertRCReturn(rc2, rc2);
10001 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10002 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10003 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
10004 }
10005
10006 /*
10007 * Check for single stepping event if we're stepping.
10008 */
10009 if (pVCpu->hm.s.fSingleInstruction)
10010 {
10011 switch (uExitReason)
10012 {
10013 case VMX_EXIT_MTF:
10014 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
10015
10016 /* Various events: */
10017 case VMX_EXIT_XCPT_OR_NMI:
10018 case VMX_EXIT_EXT_INT:
10019 case VMX_EXIT_TRIPLE_FAULT:
10020 case VMX_EXIT_INT_WINDOW:
10021 case VMX_EXIT_NMI_WINDOW:
10022 case VMX_EXIT_TASK_SWITCH:
10023 case VMX_EXIT_TPR_BELOW_THRESHOLD:
10024 case VMX_EXIT_APIC_ACCESS:
10025 case VMX_EXIT_EPT_VIOLATION:
10026 case VMX_EXIT_EPT_MISCONFIG:
10027 case VMX_EXIT_PREEMPT_TIMER:
10028
10029 /* Instruction specific VM-exits: */
10030 case VMX_EXIT_CPUID:
10031 case VMX_EXIT_GETSEC:
10032 case VMX_EXIT_HLT:
10033 case VMX_EXIT_INVD:
10034 case VMX_EXIT_INVLPG:
10035 case VMX_EXIT_RDPMC:
10036 case VMX_EXIT_RDTSC:
10037 case VMX_EXIT_RSM:
10038 case VMX_EXIT_VMCALL:
10039 case VMX_EXIT_VMCLEAR:
10040 case VMX_EXIT_VMLAUNCH:
10041 case VMX_EXIT_VMPTRLD:
10042 case VMX_EXIT_VMPTRST:
10043 case VMX_EXIT_VMREAD:
10044 case VMX_EXIT_VMRESUME:
10045 case VMX_EXIT_VMWRITE:
10046 case VMX_EXIT_VMXOFF:
10047 case VMX_EXIT_VMXON:
10048 case VMX_EXIT_MOV_CRX:
10049 case VMX_EXIT_MOV_DRX:
10050 case VMX_EXIT_IO_INSTR:
10051 case VMX_EXIT_RDMSR:
10052 case VMX_EXIT_WRMSR:
10053 case VMX_EXIT_MWAIT:
10054 case VMX_EXIT_MONITOR:
10055 case VMX_EXIT_PAUSE:
10056 case VMX_EXIT_XDTR_ACCESS:
10057 case VMX_EXIT_TR_ACCESS:
10058 case VMX_EXIT_INVEPT:
10059 case VMX_EXIT_RDTSCP:
10060 case VMX_EXIT_INVVPID:
10061 case VMX_EXIT_WBINVD:
10062 case VMX_EXIT_XSETBV:
10063 case VMX_EXIT_RDRAND:
10064 case VMX_EXIT_INVPCID:
10065 case VMX_EXIT_VMFUNC:
10066 case VMX_EXIT_RDSEED:
10067 case VMX_EXIT_XSAVES:
10068 case VMX_EXIT_XRSTORS:
10069 {
10070 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10071 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10072 AssertRCReturn(rc2, rc2);
10073 if ( pMixedCtx->rip != pDbgState->uRipStart
10074 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
10075 return VINF_EM_DBG_STEPPED;
10076 break;
10077 }
10078
10079 /* Errors and unexpected events: */
10080 case VMX_EXIT_INIT_SIGNAL:
10081 case VMX_EXIT_SIPI:
10082 case VMX_EXIT_IO_SMI:
10083 case VMX_EXIT_SMI:
10084 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10085 case VMX_EXIT_ERR_MSR_LOAD:
10086 case VMX_EXIT_ERR_MACHINE_CHECK:
10087 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
10088 break;
10089
10090 default:
10091 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10092 break;
10093 }
10094 }
10095
10096 /*
10097 * Check for debugger event breakpoints and dtrace probes.
10098 */
10099 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
10100 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
10101 {
10102 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10103 if (rcStrict != VINF_SUCCESS)
10104 return rcStrict;
10105 }
10106
10107 /*
10108 * Normal processing.
10109 */
10110#ifdef HMVMX_USE_FUNCTION_TABLE
10111 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
10112#else
10113 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10114#endif
10115}
10116
10117
10118/**
10119 * Single steps guest code using VT-x.
10120 *
10121 * @returns Strict VBox status code (i.e. informational status codes too).
10122 * @param pVM The cross context VM structure.
10123 * @param pVCpu The cross context virtual CPU structure.
10124 * @param pCtx Pointer to the guest-CPU context.
10125 *
10126 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10127 */
10128static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10129{
10130 VMXTRANSIENT VmxTransient;
10131 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10132
10133 /* Set HMCPU indicators. */
10134 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10135 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
10136 pVCpu->hm.s.fDebugWantRdTscExit = false;
10137 pVCpu->hm.s.fUsingDebugLoop = true;
10138
10139 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10140 VMXRUNDBGSTATE DbgState;
10141 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10142 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10143
10144 /*
10145 * The loop.
10146 */
10147 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10148 for (uint32_t cLoops = 0; ; cLoops++)
10149 {
10150 Assert(!HMR0SuspendPending());
10151 HMVMX_ASSERT_CPU_SAFE();
10152 bool fStepping = pVCpu->hm.s.fSingleInstruction;
10153
10154 /*
10155 * Preparatory work for running guest code, this may force us to return
10156 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10157 */
10158 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10159 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10160 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10161 if (rcStrict != VINF_SUCCESS)
10162 break;
10163
10164 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10165 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10166
10167 /*
10168 * Now we can run the guest code.
10169 */
10170 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10171
10172 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10173
10174 /*
10175 * Restore any residual host-state and save any bits shared between host
10176 * and guest into the guest-CPU state. Re-enables interrupts!
10177 */
10178 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
10179
10180 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10181 if (RT_SUCCESS(rcRun))
10182 { /* very likely */ }
10183 else
10184 {
10185 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10186 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10187 return rcRun;
10188 }
10189
10190 /* Profile the VM-exit. */
10191 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10192 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10193 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10194 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10195 HMVMX_START_EXIT_DISPATCH_PROF();
10196
10197 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10198
10199 /*
10200 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10201 */
10202 rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10203 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10204 if (rcStrict != VINF_SUCCESS)
10205 break;
10206 if (cLoops > pVM->hm.s.cMaxResumeLoops)
10207 {
10208 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10209 rcStrict = VINF_EM_RAW_INTERRUPT;
10210 break;
10211 }
10212
10213 /*
10214 * Stepping: Did the RIP change, if so, consider it a single step.
10215 * Otherwise, make sure one of the TFs gets set.
10216 */
10217 if (fStepping)
10218 {
10219 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10220 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10221 AssertRCReturn(rc2, rc2);
10222 if ( pCtx->rip != DbgState.uRipStart
10223 || pCtx->cs.Sel != DbgState.uCsStart)
10224 {
10225 rcStrict = VINF_EM_DBG_STEPPED;
10226 break;
10227 }
10228 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10229 }
10230
10231 /*
10232 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
10233 */
10234 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10235 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10236 }
10237
10238 /*
10239 * Clear the X86_EFL_TF if necessary.
10240 */
10241 if (pVCpu->hm.s.fClearTrapFlag)
10242 {
10243 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10244 AssertRCReturn(rc2, rc2);
10245 pVCpu->hm.s.fClearTrapFlag = false;
10246 pCtx->eflags.Bits.u1TF = 0;
10247 }
10248 /** @todo there seems to be issues with the resume flag when the monitor trap
10249 * flag is pending without being used. Seen early in bios init when
10250 * accessing APIC page in protected mode. */
10251
10252 /*
10253 * Restore VM-exit control settings as we may not reenter this function the
10254 * next time around.
10255 */
10256 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10257
10258 /* Restore HMCPU indicators. */
10259 pVCpu->hm.s.fUsingDebugLoop = false;
10260 pVCpu->hm.s.fDebugWantRdTscExit = false;
10261 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10262
10263 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10264 return rcStrict;
10265}
10266
10267
10268/** @} */
10269
10270
10271/**
10272 * Checks if any expensive dtrace probes are enabled and we should go to the
10273 * debug loop.
10274 *
10275 * @returns true if we should use debug loop, false if not.
10276 */
10277static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10278{
10279 /* It's probably faster to OR the raw 32-bit counter variables together.
10280 Since the variables are in an array and the probes are next to one
10281 another (more or less), we have good locality. So, better read
10282 eight-nine cache lines ever time and only have one conditional, than
10283 128+ conditionals, right? */
10284 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10285 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10286 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10287 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10288 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10289 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10290 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10291 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10292 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10293 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10294 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10295 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10296 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10297 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10298 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10299 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10300 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10301 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10302 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10303 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10304 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10305 ) != 0
10306 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10307 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10308 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10309 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10310 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10311 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10312 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10313 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10314 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10315 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10316 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10317 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10318 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10319 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10320 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10321 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10322 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10323 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10324 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10325 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10326 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10327 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10328 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10329 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10330 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10331 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10332 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10333 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10334 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10335 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10336 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10337 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10338 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10339 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10340 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10341 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10342 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10343 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10344 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10345 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10346 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10347 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10348 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10349 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10350 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10351 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10352 ) != 0
10353 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10354 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10355 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10356 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10357 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10358 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10359 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10360 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10361 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10362 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10363 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10364 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10365 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10366 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10367 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10368 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10369 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10370 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10371 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10372 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10373 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10374 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10375 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10376 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10377 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10378 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10379 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10380 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10381 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10382 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10383 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10384 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10385 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10386 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10387 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10388 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10389 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10390 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10391 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10392 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10393 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10394 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10395 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10396 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10397 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10398 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10399 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10400 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10401 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10402 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10403 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10404 ) != 0;
10405}
10406
10407
10408/**
10409 * Runs the guest code using VT-x.
10410 *
10411 * @returns Strict VBox status code (i.e. informational status codes too).
10412 * @param pVM The cross context VM structure.
10413 * @param pVCpu The cross context virtual CPU structure.
10414 * @param pCtx Pointer to the guest-CPU context.
10415 */
10416VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10417{
10418 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10419 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10420 HMVMX_ASSERT_PREEMPT_SAFE();
10421
10422 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10423
10424 VBOXSTRICTRC rcStrict;
10425 if ( !pVCpu->hm.s.fUseDebugLoop
10426 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10427 && !DBGFIsStepping(pVCpu)
10428 && !pVM->dbgf.ro.cEnabledInt3Breakpoints)
10429 rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10430 else
10431 rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10432
10433 if (rcStrict == VERR_EM_INTERPRETER)
10434 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10435 else if (rcStrict == VINF_EM_RESET)
10436 rcStrict = VINF_EM_TRIPLE_FAULT;
10437
10438 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10439 if (RT_FAILURE(rc2))
10440 {
10441 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10442 rcStrict = rc2;
10443 }
10444 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10445 return rcStrict;
10446}
10447
10448
10449#ifndef HMVMX_USE_FUNCTION_TABLE
10450DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10451{
10452# ifdef DEBUG_ramshankar
10453# define RETURN_EXIT_CALL(a_CallExpr) \
10454 do { \
10455 int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10456 VBOXSTRICTRC rcStrict = a_CallExpr; \
10457 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10458 return rcStrict; \
10459 } while (0)
10460# else
10461# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10462# endif
10463 switch (rcReason)
10464 {
10465 case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10466 case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10467 case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10468 case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10469 case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10470 case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10471 case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10472 case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10473 case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10474 case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10475 case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10476 case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10477 case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10478 case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10479 case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10480 case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10481 case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10482 case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10483 case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10484 case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10485 case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10486 case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10487 case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10488 case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10489 case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10490 case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10491 case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10492 case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10493 case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10494 case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10495 case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10496 case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10497 case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10498 case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10499
10500 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10501 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10502 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10503 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10504 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10505 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10506 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10507 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10508 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10509
10510 case VMX_EXIT_VMCLEAR:
10511 case VMX_EXIT_VMLAUNCH:
10512 case VMX_EXIT_VMPTRLD:
10513 case VMX_EXIT_VMPTRST:
10514 case VMX_EXIT_VMREAD:
10515 case VMX_EXIT_VMRESUME:
10516 case VMX_EXIT_VMWRITE:
10517 case VMX_EXIT_VMXOFF:
10518 case VMX_EXIT_VMXON:
10519 case VMX_EXIT_INVEPT:
10520 case VMX_EXIT_INVVPID:
10521 case VMX_EXIT_VMFUNC:
10522 case VMX_EXIT_XSAVES:
10523 case VMX_EXIT_XRSTORS:
10524 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10525 case VMX_EXIT_ENCLS:
10526 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10527 case VMX_EXIT_PML_FULL:
10528 default:
10529 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10530 }
10531#undef RETURN_EXIT_CALL
10532}
10533#endif /* !HMVMX_USE_FUNCTION_TABLE */
10534
10535
10536#ifdef VBOX_STRICT
10537/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10538# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10539 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10540
10541# define HMVMX_ASSERT_PREEMPT_CPUID() \
10542 do { \
10543 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10544 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10545 } while (0)
10546
10547# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10548 do { \
10549 AssertPtr(pVCpu); \
10550 AssertPtr(pMixedCtx); \
10551 AssertPtr(pVmxTransient); \
10552 Assert(pVmxTransient->fVMEntryFailed == false); \
10553 Assert(ASMIntAreEnabled()); \
10554 HMVMX_ASSERT_PREEMPT_SAFE(); \
10555 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10556 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10557 HMVMX_ASSERT_PREEMPT_SAFE(); \
10558 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10559 HMVMX_ASSERT_PREEMPT_CPUID(); \
10560 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10561 } while (0)
10562
10563# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10564 do { \
10565 Log4Func(("\n")); \
10566 } while (0)
10567#else /* nonstrict builds: */
10568# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10569 do { \
10570 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10571 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10572 } while (0)
10573# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10574#endif
10575
10576
10577/**
10578 * Advances the guest RIP by the specified number of bytes.
10579 *
10580 * @param pVCpu The cross context virtual CPU structure.
10581 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10582 * out-of-sync. Make sure to update the required fields
10583 * before using them.
10584 * @param cbInstr Number of bytes to advance the RIP by.
10585 *
10586 * @remarks No-long-jump zone!!!
10587 */
10588DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10589{
10590 /* Advance the RIP. */
10591 pMixedCtx->rip += cbInstr;
10592 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10593
10594 /* Update interrupt inhibition. */
10595 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10596 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10597 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10598}
10599
10600
10601/**
10602 * Advances the guest RIP after reading it from the VMCS.
10603 *
10604 * @returns VBox status code, no informational status codes.
10605 * @param pVCpu The cross context virtual CPU structure.
10606 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10607 * out-of-sync. Make sure to update the required fields
10608 * before using them.
10609 * @param pVmxTransient Pointer to the VMX transient structure.
10610 *
10611 * @remarks No-long-jump zone!!!
10612 */
10613static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10614{
10615 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10616 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10617 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10618 AssertRCReturn(rc, rc);
10619
10620 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10621
10622 /*
10623 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10624 * pending debug exception field as it takes care of priority of events.
10625 *
10626 * See Intel spec. 32.2.1 "Debug Exceptions".
10627 */
10628 if ( !pVCpu->hm.s.fSingleInstruction
10629 && pMixedCtx->eflags.Bits.u1TF)
10630 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10631
10632 return VINF_SUCCESS;
10633}
10634
10635
10636/**
10637 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10638 * and update error record fields accordingly.
10639 *
10640 * @return VMX_IGS_* return codes.
10641 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10642 * wrong with the guest state.
10643 *
10644 * @param pVM The cross context VM structure.
10645 * @param pVCpu The cross context virtual CPU structure.
10646 * @param pCtx Pointer to the guest-CPU state.
10647 *
10648 * @remarks This function assumes our cache of the VMCS controls
10649 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10650 */
10651static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10652{
10653#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10654#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10655 uError = (err); \
10656 break; \
10657 } else do { } while (0)
10658
10659 int rc;
10660 uint32_t uError = VMX_IGS_ERROR;
10661 uint32_t u32Val;
10662 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10663
10664 do
10665 {
10666 /*
10667 * CR0.
10668 */
10669 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10670 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10671 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10672 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10673 if (fUnrestrictedGuest)
10674 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
10675
10676 uint32_t u32GuestCR0;
10677 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
10678 AssertRCBreak(rc);
10679 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
10680 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
10681 if ( !fUnrestrictedGuest
10682 && (u32GuestCR0 & X86_CR0_PG)
10683 && !(u32GuestCR0 & X86_CR0_PE))
10684 {
10685 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10686 }
10687
10688 /*
10689 * CR4.
10690 */
10691 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10692 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10693
10694 uint32_t u32GuestCR4;
10695 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
10696 AssertRCBreak(rc);
10697 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
10698 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
10699
10700 /*
10701 * IA32_DEBUGCTL MSR.
10702 */
10703 uint64_t u64Val;
10704 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10705 AssertRCBreak(rc);
10706 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10707 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10708 {
10709 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10710 }
10711 uint64_t u64DebugCtlMsr = u64Val;
10712
10713#ifdef VBOX_STRICT
10714 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10715 AssertRCBreak(rc);
10716 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10717#endif
10718 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10719
10720 /*
10721 * RIP and RFLAGS.
10722 */
10723 uint32_t u32Eflags;
10724#if HC_ARCH_BITS == 64
10725 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10726 AssertRCBreak(rc);
10727 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10728 if ( !fLongModeGuest
10729 || !pCtx->cs.Attr.n.u1Long)
10730 {
10731 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10732 }
10733 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10734 * must be identical if the "IA-32e mode guest" VM-entry
10735 * control is 1 and CS.L is 1. No check applies if the
10736 * CPU supports 64 linear-address bits. */
10737
10738 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10739 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10740 AssertRCBreak(rc);
10741 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10742 VMX_IGS_RFLAGS_RESERVED);
10743 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10744 u32Eflags = u64Val;
10745#else
10746 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10747 AssertRCBreak(rc);
10748 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10749 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10750#endif
10751
10752 if ( fLongModeGuest
10753 || ( fUnrestrictedGuest
10754 && !(u32GuestCR0 & X86_CR0_PE)))
10755 {
10756 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10757 }
10758
10759 uint32_t u32EntryInfo;
10760 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10761 AssertRCBreak(rc);
10762 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10763 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10764 {
10765 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10766 }
10767
10768 /*
10769 * 64-bit checks.
10770 */
10771#if HC_ARCH_BITS == 64
10772 if (fLongModeGuest)
10773 {
10774 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10775 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10776 }
10777
10778 if ( !fLongModeGuest
10779 && (u32GuestCR4 & X86_CR4_PCIDE))
10780 {
10781 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10782 }
10783
10784 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10785 * 51:32 beyond the processor's physical-address width are 0. */
10786
10787 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10788 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10789 {
10790 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10791 }
10792
10793 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10794 AssertRCBreak(rc);
10795 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10796
10797 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10798 AssertRCBreak(rc);
10799 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10800#endif
10801
10802 /*
10803 * PERF_GLOBAL MSR.
10804 */
10805 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10806 {
10807 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10808 AssertRCBreak(rc);
10809 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10810 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10811 }
10812
10813 /*
10814 * PAT MSR.
10815 */
10816 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10817 {
10818 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10819 AssertRCBreak(rc);
10820 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10821 for (unsigned i = 0; i < 8; i++)
10822 {
10823 uint8_t u8Val = (u64Val & 0xff);
10824 if ( u8Val != 0 /* UC */
10825 && u8Val != 1 /* WC */
10826 && u8Val != 4 /* WT */
10827 && u8Val != 5 /* WP */
10828 && u8Val != 6 /* WB */
10829 && u8Val != 7 /* UC- */)
10830 {
10831 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10832 }
10833 u64Val >>= 8;
10834 }
10835 }
10836
10837 /*
10838 * EFER MSR.
10839 */
10840 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10841 {
10842 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10843 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10844 AssertRCBreak(rc);
10845 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10846 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10847 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10848 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10849 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10850 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10851 || !(u32GuestCR0 & X86_CR0_PG)
10852 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10853 VMX_IGS_EFER_LMA_LME_MISMATCH);
10854 }
10855
10856 /*
10857 * Segment registers.
10858 */
10859 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10860 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10861 if (!(u32Eflags & X86_EFL_VM))
10862 {
10863 /* CS */
10864 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10865 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10866 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10867 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10868 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10869 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10870 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10871 /* CS cannot be loaded with NULL in protected mode. */
10872 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10873 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10874 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10875 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10876 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10877 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10878 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10879 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10880 else
10881 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10882
10883 /* SS */
10884 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10885 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10886 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10887 if ( !(pCtx->cr0 & X86_CR0_PE)
10888 || pCtx->cs.Attr.n.u4Type == 3)
10889 {
10890 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10891 }
10892 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10893 {
10894 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10895 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10896 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10897 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10898 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10899 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10900 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10901 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10902 }
10903
10904 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
10905 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10906 {
10907 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10908 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10909 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10910 || pCtx->ds.Attr.n.u4Type > 11
10911 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10912 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10913 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10914 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10915 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10916 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10917 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10918 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10919 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10920 }
10921 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10922 {
10923 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10924 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10925 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10926 || pCtx->es.Attr.n.u4Type > 11
10927 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10928 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10929 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10930 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10931 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10932 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10933 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10934 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10935 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10936 }
10937 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10938 {
10939 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10940 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10941 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10942 || pCtx->fs.Attr.n.u4Type > 11
10943 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10944 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10945 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10946 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10947 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10948 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10949 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10950 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10951 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10952 }
10953 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10954 {
10955 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10956 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10957 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10958 || pCtx->gs.Attr.n.u4Type > 11
10959 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10960 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10961 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10962 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10963 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10964 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10965 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10966 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10967 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10968 }
10969 /* 64-bit capable CPUs. */
10970#if HC_ARCH_BITS == 64
10971 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10972 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10973 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10974 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10975 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10976 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10977 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10978 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10979 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10980 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10981 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10982#endif
10983 }
10984 else
10985 {
10986 /* V86 mode checks. */
10987 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10988 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10989 {
10990 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10991 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10992 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10993 }
10994 else
10995 {
10996 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10997 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10998 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10999 }
11000
11001 /* CS */
11002 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
11003 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
11004 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
11005 /* SS */
11006 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
11007 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
11008 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
11009 /* DS */
11010 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
11011 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
11012 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
11013 /* ES */
11014 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
11015 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
11016 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
11017 /* FS */
11018 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
11019 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
11020 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
11021 /* GS */
11022 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
11023 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
11024 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
11025 /* 64-bit capable CPUs. */
11026#if HC_ARCH_BITS == 64
11027 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11028 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11029 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11030 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11031 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11032 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
11033 VMX_IGS_LONGMODE_SS_BASE_INVALID);
11034 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
11035 VMX_IGS_LONGMODE_DS_BASE_INVALID);
11036 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
11037 VMX_IGS_LONGMODE_ES_BASE_INVALID);
11038#endif
11039 }
11040
11041 /*
11042 * TR.
11043 */
11044 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
11045 /* 64-bit capable CPUs. */
11046#if HC_ARCH_BITS == 64
11047 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
11048#endif
11049 if (fLongModeGuest)
11050 {
11051 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
11052 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
11053 }
11054 else
11055 {
11056 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
11057 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
11058 VMX_IGS_TR_ATTR_TYPE_INVALID);
11059 }
11060 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
11061 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
11062 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
11063 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
11064 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11065 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
11066 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11067 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
11068
11069 /*
11070 * GDTR and IDTR.
11071 */
11072#if HC_ARCH_BITS == 64
11073 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
11074 AssertRCBreak(rc);
11075 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
11076
11077 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
11078 AssertRCBreak(rc);
11079 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
11080#endif
11081
11082 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
11083 AssertRCBreak(rc);
11084 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11085
11086 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
11087 AssertRCBreak(rc);
11088 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11089
11090 /*
11091 * Guest Non-Register State.
11092 */
11093 /* Activity State. */
11094 uint32_t u32ActivityState;
11095 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
11096 AssertRCBreak(rc);
11097 HMVMX_CHECK_BREAK( !u32ActivityState
11098 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
11099 VMX_IGS_ACTIVITY_STATE_INVALID);
11100 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
11101 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
11102 uint32_t u32IntrState;
11103 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
11104 AssertRCBreak(rc);
11105 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
11106 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11107 {
11108 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11109 }
11110
11111 /** @todo Activity state and injecting interrupts. Left as a todo since we
11112 * currently don't use activity states but ACTIVE. */
11113
11114 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11115 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11116
11117 /* Guest interruptibility-state. */
11118 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11119 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11120 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
11121 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11122 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11123 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11124 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11125 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11126 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11127 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
11128 {
11129 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11130 {
11131 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11132 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11133 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11134 }
11135 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11136 {
11137 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11138 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11139 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11140 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11141 }
11142 }
11143 /** @todo Assumes the processor is not in SMM. */
11144 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11145 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11146 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11147 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11148 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11149 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11150 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11151 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11152 {
11153 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11154 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11155 }
11156
11157 /* Pending debug exceptions. */
11158#if HC_ARCH_BITS == 64
11159 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11160 AssertRCBreak(rc);
11161 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11162 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11163 u32Val = u64Val; /* For pending debug exceptions checks below. */
11164#else
11165 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11166 AssertRCBreak(rc);
11167 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11168 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11169#endif
11170
11171 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11172 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11173 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11174 {
11175 if ( (u32Eflags & X86_EFL_TF)
11176 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11177 {
11178 /* Bit 14 is PendingDebug.BS. */
11179 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11180 }
11181 if ( !(u32Eflags & X86_EFL_TF)
11182 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11183 {
11184 /* Bit 14 is PendingDebug.BS. */
11185 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11186 }
11187 }
11188
11189 /* VMCS link pointer. */
11190 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11191 AssertRCBreak(rc);
11192 if (u64Val != UINT64_C(0xffffffffffffffff))
11193 {
11194 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11195 /** @todo Bits beyond the processor's physical-address width MBZ. */
11196 /** @todo 32-bit located in memory referenced by value of this field (as a
11197 * physical address) must contain the processor's VMCS revision ID. */
11198 /** @todo SMM checks. */
11199 }
11200
11201 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11202 * not using Nested Paging? */
11203 if ( pVM->hm.s.fNestedPaging
11204 && !fLongModeGuest
11205 && CPUMIsGuestInPAEModeEx(pCtx))
11206 {
11207 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11208 AssertRCBreak(rc);
11209 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11210
11211 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11212 AssertRCBreak(rc);
11213 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11214
11215 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11216 AssertRCBreak(rc);
11217 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11218
11219 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11220 AssertRCBreak(rc);
11221 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11222 }
11223
11224 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11225 if (uError == VMX_IGS_ERROR)
11226 uError = VMX_IGS_REASON_NOT_FOUND;
11227 } while (0);
11228
11229 pVCpu->hm.s.u32HMError = uError;
11230 return uError;
11231
11232#undef HMVMX_ERROR_BREAK
11233#undef HMVMX_CHECK_BREAK
11234}
11235
11236/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11237/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11238/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11239
11240/** @name VM-exit handlers.
11241 * @{
11242 */
11243
11244/**
11245 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11246 */
11247HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11248{
11249 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11250 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11251 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11252 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11253 return VINF_SUCCESS;
11254 return VINF_EM_RAW_INTERRUPT;
11255}
11256
11257
11258/**
11259 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11260 */
11261HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11262{
11263 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11264 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11265
11266 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11267 AssertRCReturn(rc, rc);
11268
11269 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11270 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11271 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11272 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11273
11274 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11275 {
11276 /*
11277 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11278 * anything we inject is not going to cause a VM-exit directly for the event being injected.
11279 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11280 *
11281 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11282 */
11283 VMXDispatchHostNmi();
11284 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11285 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11286 return VINF_SUCCESS;
11287 }
11288
11289 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11290 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11291 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11292 { /* likely */ }
11293 else
11294 {
11295 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11296 rcStrictRc1 = VINF_SUCCESS;
11297 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11298 return rcStrictRc1;
11299 }
11300
11301 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11302 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11303 switch (uIntType)
11304 {
11305 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11306 Assert(uVector == X86_XCPT_DB);
11307 /* no break */
11308 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11309 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11310 /* no break */
11311 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11312 {
11313 /*
11314 * If there's any exception caused as a result of event injection, go back to
11315 * the interpreter. The page-fault case is complicated and we manually handle
11316 * any currently pending event in hmR0VmxExitXcptPF. Nested #ACs are already
11317 * handled in hmR0VmxCheckExitDueToEventDelivery.
11318 */
11319 if (!pVCpu->hm.s.Event.fPending)
11320 { /* likely */ }
11321 else if ( uVector != X86_XCPT_PF
11322 && uVector != X86_XCPT_AC)
11323 {
11324 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
11325 rc = VERR_EM_INTERPRETER;
11326 break;
11327 }
11328
11329 switch (uVector)
11330 {
11331 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11332 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11333 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11334 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11335 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11336 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11337 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11338
11339 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11340 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11341 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11342 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11343 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11344 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11345 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11346 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11347 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11348 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11349 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11350 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11351 default:
11352 {
11353 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11354 AssertRCReturn(rc, rc);
11355
11356 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11357 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11358 {
11359 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11360 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11361 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11362
11363 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11364 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11365 AssertRCReturn(rc, rc);
11366 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11367 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11368 0 /* GCPtrFaultAddress */);
11369 AssertRCReturn(rc, rc);
11370 }
11371 else
11372 {
11373 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11374 pVCpu->hm.s.u32HMError = uVector;
11375 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11376 }
11377 break;
11378 }
11379 }
11380 break;
11381 }
11382
11383 default:
11384 {
11385 pVCpu->hm.s.u32HMError = uExitIntInfo;
11386 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11387 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11388 break;
11389 }
11390 }
11391 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11392 return rc;
11393}
11394
11395
11396/**
11397 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11398 */
11399HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11400{
11401 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11402
11403 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11404 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11405
11406 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11407 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11408 return VINF_SUCCESS;
11409}
11410
11411
11412/**
11413 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11414 */
11415HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11416{
11417 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11418 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11419 {
11420 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11421 HMVMX_RETURN_UNEXPECTED_EXIT();
11422 }
11423
11424 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11425
11426 /*
11427 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11428 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11429 */
11430 uint32_t uIntrState = 0;
11431 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11432 AssertRCReturn(rc, rc);
11433
11434 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11435 if ( fBlockSti
11436 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11437 {
11438 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11439 }
11440
11441 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11442 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11443
11444 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11445 return VINF_SUCCESS;
11446}
11447
11448
11449/**
11450 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11451 */
11452HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11453{
11454 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11455 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11456 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11457}
11458
11459
11460/**
11461 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11462 */
11463HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11464{
11465 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11466 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11467 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11468}
11469
11470
11471/**
11472 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11473 */
11474HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11475{
11476 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11477 PVM pVM = pVCpu->CTX_SUFF(pVM);
11478 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11479 if (RT_LIKELY(rc == VINF_SUCCESS))
11480 {
11481 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11482 Assert(pVmxTransient->cbInstr == 2);
11483 }
11484 else
11485 {
11486 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11487 rc = VERR_EM_INTERPRETER;
11488 }
11489 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11490 return rc;
11491}
11492
11493
11494/**
11495 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11496 */
11497HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11498{
11499 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11500 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11501 AssertRCReturn(rc, rc);
11502
11503 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11504 return VINF_EM_RAW_EMULATE_INSTR;
11505
11506 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11507 HMVMX_RETURN_UNEXPECTED_EXIT();
11508}
11509
11510
11511/**
11512 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11513 */
11514HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11515{
11516 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11517 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11518 AssertRCReturn(rc, rc);
11519
11520 PVM pVM = pVCpu->CTX_SUFF(pVM);
11521 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11522 if (RT_LIKELY(rc == VINF_SUCCESS))
11523 {
11524 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11525 Assert(pVmxTransient->cbInstr == 2);
11526 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11527 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11528 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11529 }
11530 else
11531 rc = VERR_EM_INTERPRETER;
11532 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11533 return rc;
11534}
11535
11536
11537/**
11538 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11539 */
11540HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11541{
11542 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11543 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11544 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11545 AssertRCReturn(rc, rc);
11546
11547 PVM pVM = pVCpu->CTX_SUFF(pVM);
11548 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11549 if (RT_SUCCESS(rc))
11550 {
11551 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11552 Assert(pVmxTransient->cbInstr == 3);
11553 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11554 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11555 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11556 }
11557 else
11558 {
11559 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11560 rc = VERR_EM_INTERPRETER;
11561 }
11562 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11563 return rc;
11564}
11565
11566
11567/**
11568 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11569 */
11570HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11571{
11572 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11573 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11574 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11575 AssertRCReturn(rc, rc);
11576
11577 PVM pVM = pVCpu->CTX_SUFF(pVM);
11578 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11579 if (RT_LIKELY(rc == VINF_SUCCESS))
11580 {
11581 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11582 Assert(pVmxTransient->cbInstr == 2);
11583 }
11584 else
11585 {
11586 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11587 rc = VERR_EM_INTERPRETER;
11588 }
11589 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11590 return rc;
11591}
11592
11593
11594/**
11595 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11596 */
11597HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11598{
11599 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11600 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11601
11602 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11603 if (pVCpu->hm.s.fHypercallsEnabled)
11604 {
11605#if 0
11606 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11607#else
11608 /* Aggressive state sync. for now. */
11609 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11610 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11611 AssertRCReturn(rc, rc);
11612#endif
11613
11614 /* Perform the hypercall. */
11615 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11616 if (rcStrict == VINF_SUCCESS)
11617 {
11618 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11619 AssertRCReturn(rc, rc);
11620 }
11621 else
11622 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11623 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11624 || RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
11625
11626 /* If the hypercall changes anything other than guest's general-purpose registers,
11627 we would need to reload the guest changed bits here before VM-entry. */
11628 }
11629 else
11630 Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11631
11632 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11633 if (RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)))
11634 {
11635 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11636 rcStrict = VINF_SUCCESS;
11637 }
11638
11639 return rcStrict;
11640}
11641
11642
11643/**
11644 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11645 */
11646HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11647{
11648 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11649 PVM pVM = pVCpu->CTX_SUFF(pVM);
11650 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11651
11652 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11653 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11654 AssertRCReturn(rc, rc);
11655
11656 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11657 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11658 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11659 else
11660 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11661 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11662 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
11663 return rcStrict;
11664}
11665
11666
11667/**
11668 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11669 */
11670HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11671{
11672 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11673 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11674 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11675 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11676 AssertRCReturn(rc, rc);
11677
11678 PVM pVM = pVCpu->CTX_SUFF(pVM);
11679 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11680 if (RT_LIKELY(rc == VINF_SUCCESS))
11681 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11682 else
11683 {
11684 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11685 rc = VERR_EM_INTERPRETER;
11686 }
11687 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11688 return rc;
11689}
11690
11691
11692/**
11693 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11694 */
11695HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11696{
11697 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11698 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11699 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11700 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11701 AssertRCReturn(rc, rc);
11702
11703 PVM pVM = pVCpu->CTX_SUFF(pVM);
11704 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11705 rc = VBOXSTRICTRC_VAL(rc2);
11706 if (RT_LIKELY( rc == VINF_SUCCESS
11707 || rc == VINF_EM_HALT))
11708 {
11709 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11710 AssertRCReturn(rc3, rc3);
11711
11712 if ( rc == VINF_EM_HALT
11713 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11714 {
11715 rc = VINF_SUCCESS;
11716 }
11717 }
11718 else
11719 {
11720 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11721 rc = VERR_EM_INTERPRETER;
11722 }
11723 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11724 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11725 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11726 return rc;
11727}
11728
11729
11730/**
11731 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11732 */
11733HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11734{
11735 /*
11736 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
11737 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
11738 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
11739 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
11740 */
11741 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11742 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11743 HMVMX_RETURN_UNEXPECTED_EXIT();
11744}
11745
11746
11747/**
11748 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11749 */
11750HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11751{
11752 /*
11753 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
11754 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
11755 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
11756 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
11757 */
11758 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11759 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11760 HMVMX_RETURN_UNEXPECTED_EXIT();
11761}
11762
11763
11764/**
11765 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11766 */
11767HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11768{
11769 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11770 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11771 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11772 HMVMX_RETURN_UNEXPECTED_EXIT();
11773}
11774
11775
11776/**
11777 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11778 */
11779HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11780{
11781 /*
11782 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
11783 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
11784 * See Intel spec. 25.3 "Other Causes of VM-exits".
11785 */
11786 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11787 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11788 HMVMX_RETURN_UNEXPECTED_EXIT();
11789}
11790
11791
11792/**
11793 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11794 * VM-exit.
11795 */
11796HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11797{
11798 /*
11799 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11800 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11801 *
11802 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11803 * See Intel spec. "23.8 Restrictions on VMX operation".
11804 */
11805 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11806 return VINF_SUCCESS;
11807}
11808
11809
11810/**
11811 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11812 * VM-exit.
11813 */
11814HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11815{
11816 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11817 return VINF_EM_RESET;
11818}
11819
11820
11821/**
11822 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11823 */
11824HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11825{
11826 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11827 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11828
11829 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11830 AssertRCReturn(rc, rc);
11831
11832 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11833 rc = VINF_SUCCESS;
11834 else
11835 rc = VINF_EM_HALT;
11836
11837 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11838 if (rc != VINF_SUCCESS)
11839 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11840 return rc;
11841}
11842
11843
11844/**
11845 * VM-exit handler for instructions that result in a \#UD exception delivered to
11846 * the guest.
11847 */
11848HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11849{
11850 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11851 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11852 return VINF_SUCCESS;
11853}
11854
11855
11856/**
11857 * VM-exit handler for expiry of the VMX preemption timer.
11858 */
11859HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11860{
11861 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11862
11863 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11864 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11865
11866 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11867 PVM pVM = pVCpu->CTX_SUFF(pVM);
11868 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11869 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11870 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11871}
11872
11873
11874/**
11875 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11876 */
11877HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11878{
11879 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11880
11881 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11882 rc |= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, false /*fNeedRsp*/);
11883 rc |= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11884 AssertRCReturn(rc, rc);
11885
11886 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11887 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
11888
11889 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11890
11891 return rcStrict;
11892}
11893
11894
11895/**
11896 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11897 */
11898HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11899{
11900 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11901
11902 /* The guest should not invalidate the host CPU's TLBs, fallback to interpreter. */
11903 /** @todo implement EMInterpretInvpcid() */
11904 return VERR_EM_INTERPRETER;
11905}
11906
11907
11908/**
11909 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11910 * Error VM-exit.
11911 */
11912HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11913{
11914 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11915 AssertRCReturn(rc, rc);
11916
11917 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11918 AssertRCReturn(rc, rc);
11919
11920 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11921 NOREF(uInvalidReason);
11922
11923#ifdef VBOX_STRICT
11924 uint32_t uIntrState;
11925 RTHCUINTREG uHCReg;
11926 uint64_t u64Val;
11927 uint32_t u32Val;
11928
11929 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11930 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11931 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11932 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11933 AssertRCReturn(rc, rc);
11934
11935 Log4(("uInvalidReason %u\n", uInvalidReason));
11936 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11937 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11938 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11939 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
11940
11941 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11942 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11943 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11944 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11945 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11946 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11947 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11948 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11949 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11950 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11951 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11952 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11953#else
11954 NOREF(pVmxTransient);
11955#endif
11956
11957 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11958 return VERR_VMX_INVALID_GUEST_STATE;
11959}
11960
11961
11962/**
11963 * VM-exit handler for VM-entry failure due to an MSR-load
11964 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11965 */
11966HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11967{
11968 NOREF(pVmxTransient);
11969 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11970 HMVMX_RETURN_UNEXPECTED_EXIT();
11971}
11972
11973
11974/**
11975 * VM-exit handler for VM-entry failure due to a machine-check event
11976 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11977 */
11978HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11979{
11980 NOREF(pVmxTransient);
11981 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11982 HMVMX_RETURN_UNEXPECTED_EXIT();
11983}
11984
11985
11986/**
11987 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11988 * theory.
11989 */
11990HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11991{
11992 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
11993 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
11994 return VERR_VMX_UNDEFINED_EXIT_CODE;
11995}
11996
11997
11998/**
11999 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
12000 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
12001 * Conditional VM-exit.
12002 */
12003HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12004{
12005 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12006
12007 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
12008 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
12009 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
12010 return VERR_EM_INTERPRETER;
12011 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12012 HMVMX_RETURN_UNEXPECTED_EXIT();
12013}
12014
12015
12016/**
12017 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
12018 */
12019HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12020{
12021 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12022
12023 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
12024 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
12025 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
12026 return VERR_EM_INTERPRETER;
12027 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12028 HMVMX_RETURN_UNEXPECTED_EXIT();
12029}
12030
12031
12032/**
12033 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
12034 */
12035HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12036{
12037 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12038
12039 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
12040 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12041 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12042 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12043 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12044 {
12045 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12046 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12047 }
12048 AssertRCReturn(rc, rc);
12049 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
12050
12051#ifdef VBOX_STRICT
12052 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
12053 {
12054 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
12055 && pMixedCtx->ecx != MSR_K6_EFER)
12056 {
12057 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12058 pMixedCtx->ecx));
12059 HMVMX_RETURN_UNEXPECTED_EXIT();
12060 }
12061 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12062 {
12063 VMXMSREXITREAD enmRead;
12064 VMXMSREXITWRITE enmWrite;
12065 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12066 AssertRCReturn(rc2, rc2);
12067 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
12068 {
12069 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12070 HMVMX_RETURN_UNEXPECTED_EXIT();
12071 }
12072 }
12073 }
12074#endif
12075
12076 PVM pVM = pVCpu->CTX_SUFF(pVM);
12077 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12078 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
12079 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
12080 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
12081 if (RT_SUCCESS(rc))
12082 {
12083 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12084 Assert(pVmxTransient->cbInstr == 2);
12085 }
12086 return rc;
12087}
12088
12089
12090/**
12091 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12092 */
12093HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12094{
12095 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12096 PVM pVM = pVCpu->CTX_SUFF(pVM);
12097 int rc = VINF_SUCCESS;
12098
12099 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
12100 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12101 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12102 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12103 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12104 {
12105 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12106 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12107 }
12108 AssertRCReturn(rc, rc);
12109 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
12110
12111 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12112 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
12113 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12114
12115 if (RT_SUCCESS(rc))
12116 {
12117 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12118
12119 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
12120 if ( pMixedCtx->ecx == MSR_IA32_APICBASE
12121 || ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
12122 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
12123 {
12124 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12125 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
12126 EMInterpretWrmsr() changes it. */
12127 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12128 }
12129 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12130 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12131 else if (pMixedCtx->ecx == MSR_K6_EFER)
12132 {
12133 /*
12134 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12135 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12136 * the other bits as well, SCE and NXE. See @bugref{7368}.
12137 */
12138 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
12139 }
12140
12141 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12142 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12143 {
12144 switch (pMixedCtx->ecx)
12145 {
12146 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
12147 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
12148 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
12149 case MSR_K8_FS_BASE: /* no break */
12150 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
12151 case MSR_K6_EFER: /* already handled above */ break;
12152 default:
12153 {
12154 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12155 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12156 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12157 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
12158 break;
12159 }
12160 }
12161 }
12162#ifdef VBOX_STRICT
12163 else
12164 {
12165 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12166 switch (pMixedCtx->ecx)
12167 {
12168 case MSR_IA32_SYSENTER_CS:
12169 case MSR_IA32_SYSENTER_EIP:
12170 case MSR_IA32_SYSENTER_ESP:
12171 case MSR_K8_FS_BASE:
12172 case MSR_K8_GS_BASE:
12173 {
12174 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12175 HMVMX_RETURN_UNEXPECTED_EXIT();
12176 }
12177
12178 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12179 default:
12180 {
12181 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12182 {
12183 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12184 if (pMixedCtx->ecx != MSR_K6_EFER)
12185 {
12186 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12187 pMixedCtx->ecx));
12188 HMVMX_RETURN_UNEXPECTED_EXIT();
12189 }
12190 }
12191
12192 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12193 {
12194 VMXMSREXITREAD enmRead;
12195 VMXMSREXITWRITE enmWrite;
12196 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12197 AssertRCReturn(rc2, rc2);
12198 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12199 {
12200 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12201 HMVMX_RETURN_UNEXPECTED_EXIT();
12202 }
12203 }
12204 break;
12205 }
12206 }
12207 }
12208#endif /* VBOX_STRICT */
12209 }
12210 return rc;
12211}
12212
12213
12214/**
12215 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12216 */
12217HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12218{
12219 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12220
12221 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12222 return VINF_EM_RAW_INTERRUPT;
12223}
12224
12225
12226/**
12227 * VM-exit handler for when the TPR value is lowered below the specified
12228 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12229 */
12230HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12231{
12232 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12233 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12234
12235 /*
12236 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
12237 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
12238 * resume guest execution.
12239 */
12240 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12241 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12242 return VINF_SUCCESS;
12243}
12244
12245
12246/**
12247 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12248 * VM-exit.
12249 *
12250 * @retval VINF_SUCCESS when guest execution can continue.
12251 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12252 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12253 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12254 * interpreter.
12255 */
12256HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12257{
12258 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12259 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12260 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12261 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12262 AssertRCReturn(rc, rc);
12263
12264 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12265 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12266 PVM pVM = pVCpu->CTX_SUFF(pVM);
12267 VBOXSTRICTRC rcStrict;
12268 rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, true /*fNeedRsp*/);
12269 switch (uAccessType)
12270 {
12271 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12272 {
12273 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12274 AssertRCReturn(rc, rc);
12275
12276 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12277 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12278 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12279 AssertMsg( rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE
12280 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12281 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12282 {
12283 case 0: /* CR0 */
12284 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12285 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12286 break;
12287 case 2: /* CR2 */
12288 /* Nothing to do here, CR2 it's not part of the VMCS. */
12289 break;
12290 case 3: /* CR3 */
12291 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
12292 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12293 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12294 break;
12295 case 4: /* CR4 */
12296 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12297 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12298 VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12299 break;
12300 case 8: /* CR8 */
12301 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12302 /* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12303 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12304 break;
12305 default:
12306 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12307 break;
12308 }
12309
12310 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12311 break;
12312 }
12313
12314 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12315 {
12316 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12317 AssertRCReturn(rc, rc);
12318
12319 Assert( !pVM->hm.s.fNestedPaging
12320 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12321 || pVCpu->hm.s.fUsingDebugLoop
12322 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12323
12324 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12325 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12326 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12327
12328 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12329 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12330 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12331 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12332 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12333 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12334 VBOXSTRICTRC_VAL(rcStrict)));
12335 break;
12336 }
12337
12338 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12339 {
12340 AssertRCReturn(rc, rc);
12341 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12342 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12343 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12344 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12345 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12346 break;
12347 }
12348
12349 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12350 {
12351 AssertRCReturn(rc, rc);
12352 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12353 VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12354 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE,
12355 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12356 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12357 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12358 break;
12359 }
12360
12361 default:
12362 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12363 VERR_VMX_UNEXPECTED_EXCEPTION);
12364 }
12365
12366 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12367 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12368 NOREF(pVM);
12369 return rcStrict;
12370}
12371
12372
12373/**
12374 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12375 * VM-exit.
12376 */
12377HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12378{
12379 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12380 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12381
12382 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12383 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12384 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12385 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12386 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12387 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12388 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12389 AssertRCReturn(rc2, rc2);
12390
12391 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12392 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12393 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12394 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12395 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12396 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12397 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12398 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12399 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12400
12401 /* I/O operation lookup arrays. */
12402 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12403 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12404
12405 VBOXSTRICTRC rcStrict;
12406 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12407 uint32_t const cbInstr = pVmxTransient->cbInstr;
12408 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12409 PVM pVM = pVCpu->CTX_SUFF(pVM);
12410 if (fIOString)
12411 {
12412#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12413 See @bugref{5752#c158}. Should work now. */
12414 /*
12415 * INS/OUTS - I/O String instruction.
12416 *
12417 * Use instruction-information if available, otherwise fall back on
12418 * interpreting the instruction.
12419 */
12420 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12421 fIOWrite ? 'w' : 'r'));
12422 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12423 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12424 {
12425 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12426 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12427 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12428 AssertRCReturn(rc2, rc2);
12429 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12430 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12431 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12432 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12433 if (fIOWrite)
12434 {
12435 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12436 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12437 }
12438 else
12439 {
12440 /*
12441 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12442 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12443 * See Intel Instruction spec. for "INS".
12444 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12445 */
12446 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12447 }
12448 }
12449 else
12450 {
12451 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12452 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12453 AssertRCReturn(rc2, rc2);
12454 rcStrict = IEMExecOne(pVCpu);
12455 }
12456 /** @todo IEM needs to be setting these flags somehow. */
12457 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12458 fUpdateRipAlready = true;
12459#else
12460 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12461 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12462 if (RT_SUCCESS(rcStrict))
12463 {
12464 if (fIOWrite)
12465 {
12466 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12467 (DISCPUMODE)pDis->uAddrMode, cbValue);
12468 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12469 }
12470 else
12471 {
12472 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12473 (DISCPUMODE)pDis->uAddrMode, cbValue);
12474 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12475 }
12476 }
12477 else
12478 {
12479 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12480 pMixedCtx->rip));
12481 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12482 }
12483#endif
12484 }
12485 else
12486 {
12487 /*
12488 * IN/OUT - I/O instruction.
12489 */
12490 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12491 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12492 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12493 if (fIOWrite)
12494 {
12495 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12496 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12497 }
12498 else
12499 {
12500 uint32_t u32Result = 0;
12501 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12502 if (IOM_SUCCESS(rcStrict))
12503 {
12504 /* Save result of I/O IN instr. in AL/AX/EAX. */
12505 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12506 }
12507 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12508 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12509 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12510 }
12511 }
12512
12513 if (IOM_SUCCESS(rcStrict))
12514 {
12515 if (!fUpdateRipAlready)
12516 {
12517 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12518 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12519 }
12520
12521 /*
12522 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12523 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12524 */
12525 if (fIOString)
12526 {
12527 /** @todo Single-step for INS/OUTS with REP prefix? */
12528 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12529 }
12530 else if ( !fDbgStepping
12531 && fGstStepping)
12532 {
12533 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12534 }
12535
12536 /*
12537 * If any I/O breakpoints are armed, we need to check if one triggered
12538 * and take appropriate action.
12539 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12540 */
12541 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12542 AssertRCReturn(rc2, rc2);
12543
12544 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12545 * execution engines about whether hyper BPs and such are pending. */
12546 uint32_t const uDr7 = pMixedCtx->dr[7];
12547 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12548 && X86_DR7_ANY_RW_IO(uDr7)
12549 && (pMixedCtx->cr4 & X86_CR4_DE))
12550 || DBGFBpIsHwIoArmed(pVM)))
12551 {
12552 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12553
12554 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12555 VMMRZCallRing3Disable(pVCpu);
12556 HM_DISABLE_PREEMPT();
12557
12558 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12559
12560 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12561 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12562 {
12563 /* Raise #DB. */
12564 if (fIsGuestDbgActive)
12565 ASMSetDR6(pMixedCtx->dr[6]);
12566 if (pMixedCtx->dr[7] != uDr7)
12567 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12568
12569 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12570 }
12571 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12572 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12573 else if ( rcStrict2 != VINF_SUCCESS
12574 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12575 rcStrict = rcStrict2;
12576 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12577
12578 HM_RESTORE_PREEMPT();
12579 VMMRZCallRing3Enable(pVCpu);
12580 }
12581 }
12582
12583#ifdef VBOX_STRICT
12584 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12585 Assert(!fIOWrite);
12586 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12587 Assert(fIOWrite);
12588 else
12589 {
12590#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12591 * statuses, that the VMM device and some others may return. See
12592 * IOM_SUCCESS() for guidance. */
12593 AssertMsg( RT_FAILURE(rcStrict)
12594 || rcStrict == VINF_SUCCESS
12595 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12596 || rcStrict == VINF_EM_DBG_BREAKPOINT
12597 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12598 || rcStrict == VINF_EM_RAW_TO_R3
12599 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12600#endif
12601 }
12602#endif
12603
12604 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12605 return rcStrict;
12606}
12607
12608
12609/**
12610 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12611 * VM-exit.
12612 */
12613HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12614{
12615 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12616
12617 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12618 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12619 AssertRCReturn(rc, rc);
12620 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12621 {
12622 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12623 AssertRCReturn(rc, rc);
12624 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12625 {
12626 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12627
12628 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12629 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12630
12631 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
12632 Assert(!pVCpu->hm.s.Event.fPending);
12633 pVCpu->hm.s.Event.fPending = true;
12634 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
12635 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12636 AssertRCReturn(rc, rc);
12637 if (fErrorCodeValid)
12638 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12639 else
12640 pVCpu->hm.s.Event.u32ErrCode = 0;
12641 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12642 && uVector == X86_XCPT_PF)
12643 {
12644 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
12645 }
12646
12647 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12648 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12649 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12650 }
12651 }
12652
12653 /* Fall back to the interpreter to emulate the task-switch. */
12654 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12655 return VERR_EM_INTERPRETER;
12656}
12657
12658
12659/**
12660 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12661 */
12662HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12663{
12664 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12665 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12666 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12667 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12668 AssertRCReturn(rc, rc);
12669 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12670 return VINF_EM_DBG_STEPPED;
12671}
12672
12673
12674/**
12675 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12676 */
12677HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12678{
12679 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12680
12681 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12682
12683 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12684 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12685 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12686 {
12687 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12688 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12689 {
12690 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12691 return VERR_EM_INTERPRETER;
12692 }
12693 }
12694 else
12695 {
12696 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12697 rcStrict1 = VINF_SUCCESS;
12698 return rcStrict1;
12699 }
12700
12701#if 0
12702 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
12703 * just sync the whole thing. */
12704 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12705#else
12706 /* Aggressive state sync. for now. */
12707 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12708 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12709 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12710#endif
12711 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12712 AssertRCReturn(rc, rc);
12713
12714 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12715 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12716 VBOXSTRICTRC rcStrict2;
12717 switch (uAccessType)
12718 {
12719 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12720 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12721 {
12722 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12723 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != 0x80,
12724 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12725
12726 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12727 GCPhys &= PAGE_BASE_GC_MASK;
12728 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12729 PVM pVM = pVCpu->CTX_SUFF(pVM);
12730 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12731 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12732
12733 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12734 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12735 CPUMCTX2CORE(pMixedCtx), GCPhys);
12736 Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
12737 if ( rcStrict2 == VINF_SUCCESS
12738 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12739 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12740 {
12741 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12742 | HM_CHANGED_GUEST_RSP
12743 | HM_CHANGED_GUEST_RFLAGS
12744 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12745 rcStrict2 = VINF_SUCCESS;
12746 }
12747 break;
12748 }
12749
12750 default:
12751 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
12752 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12753 break;
12754 }
12755
12756 if (rcStrict2 != VINF_SUCCESS)
12757 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12758 return rcStrict2;
12759}
12760
12761
12762/**
12763 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12764 * VM-exit.
12765 */
12766HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12767{
12768 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12769
12770 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12771 if (pVmxTransient->fWasGuestDebugStateActive)
12772 {
12773 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12774 HMVMX_RETURN_UNEXPECTED_EXIT();
12775 }
12776
12777 if ( !pVCpu->hm.s.fSingleInstruction
12778 && !pVmxTransient->fWasHyperDebugStateActive)
12779 {
12780 Assert(!DBGFIsStepping(pVCpu));
12781 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12782
12783 /* Don't intercept MOV DRx any more. */
12784 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12785 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12786 AssertRCReturn(rc, rc);
12787
12788 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12789 VMMRZCallRing3Disable(pVCpu);
12790 HM_DISABLE_PREEMPT();
12791
12792 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12793 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12794 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12795
12796 HM_RESTORE_PREEMPT();
12797 VMMRZCallRing3Enable(pVCpu);
12798
12799#ifdef VBOX_WITH_STATISTICS
12800 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12801 AssertRCReturn(rc, rc);
12802 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12803 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12804 else
12805 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12806#endif
12807 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12808 return VINF_SUCCESS;
12809 }
12810
12811 /*
12812 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12813 * Update the segment registers and DR7 from the CPU.
12814 */
12815 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12816 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12817 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12818 AssertRCReturn(rc, rc);
12819 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12820
12821 PVM pVM = pVCpu->CTX_SUFF(pVM);
12822 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12823 {
12824 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12825 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
12826 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
12827 if (RT_SUCCESS(rc))
12828 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12829 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12830 }
12831 else
12832 {
12833 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12834 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
12835 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
12836 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12837 }
12838
12839 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12840 if (RT_SUCCESS(rc))
12841 {
12842 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12843 AssertRCReturn(rc2, rc2);
12844 return VINF_SUCCESS;
12845 }
12846 return rc;
12847}
12848
12849
12850/**
12851 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12852 * Conditional VM-exit.
12853 */
12854HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12855{
12856 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12857 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12858
12859 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12860 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12861 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12862 {
12863 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12864 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12865 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12866 {
12867 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12868 return VERR_EM_INTERPRETER;
12869 }
12870 }
12871 else
12872 {
12873 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12874 rcStrict1 = VINF_SUCCESS;
12875 return rcStrict1;
12876 }
12877
12878 RTGCPHYS GCPhys = 0;
12879 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12880
12881#if 0
12882 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12883#else
12884 /* Aggressive state sync. for now. */
12885 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12886 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12887 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12888#endif
12889 AssertRCReturn(rc, rc);
12890
12891 /*
12892 * If we succeed, resume guest execution.
12893 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12894 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12895 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12896 * weird case. See @bugref{6043}.
12897 */
12898 PVM pVM = pVCpu->CTX_SUFF(pVM);
12899 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12900 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
12901 if ( rcStrict2 == VINF_SUCCESS
12902 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12903 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12904 {
12905 /* Successfully handled MMIO operation. */
12906 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12907 | HM_CHANGED_GUEST_RSP
12908 | HM_CHANGED_GUEST_RFLAGS
12909 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12910 return VINF_SUCCESS;
12911 }
12912 return rcStrict2;
12913}
12914
12915
12916/**
12917 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12918 * VM-exit.
12919 */
12920HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12921{
12922 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12923 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12924
12925 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12926 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12927 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12928 {
12929 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12930 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12931 Log4(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12932 }
12933 else
12934 {
12935 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12936 rcStrict1 = VINF_SUCCESS;
12937 return rcStrict1;
12938 }
12939
12940 RTGCPHYS GCPhys = 0;
12941 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12942 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12943#if 0
12944 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12945#else
12946 /* Aggressive state sync. for now. */
12947 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12948 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12949 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12950#endif
12951 AssertRCReturn(rc, rc);
12952
12953 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12954 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12955
12956 RTGCUINT uErrorCode = 0;
12957 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
12958 uErrorCode |= X86_TRAP_PF_ID;
12959 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
12960 uErrorCode |= X86_TRAP_PF_RW;
12961 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
12962 uErrorCode |= X86_TRAP_PF_P;
12963
12964 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12965
12966 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12967 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12968
12969 /* Handle the pagefault trap for the nested shadow table. */
12970 PVM pVM = pVCpu->CTX_SUFF(pVM);
12971 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12972 TRPMResetTrap(pVCpu);
12973
12974 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12975 if ( rcStrict2 == VINF_SUCCESS
12976 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12977 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12978 {
12979 /* Successfully synced our nested page tables. */
12980 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12981 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12982 | HM_CHANGED_GUEST_RSP
12983 | HM_CHANGED_GUEST_RFLAGS);
12984 return VINF_SUCCESS;
12985 }
12986
12987 Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12988 return rcStrict2;
12989}
12990
12991/** @} */
12992
12993/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12994/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
12995/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12996
12997/** @name VM-exit exception handlers.
12998 * @{
12999 */
13000
13001/**
13002 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
13003 */
13004static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13005{
13006 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13007 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13008
13009 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13010 AssertRCReturn(rc, rc);
13011
13012 if (!(pMixedCtx->cr0 & X86_CR0_NE))
13013 {
13014 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13015 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13016
13017 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13018 * provides VM-exit instruction length. If this causes problem later,
13019 * disassemble the instruction like it's done on AMD-V. */
13020 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13021 AssertRCReturn(rc2, rc2);
13022 return rc;
13023 }
13024
13025 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13026 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13027 return rc;
13028}
13029
13030
13031/**
13032 * VM-exit exception handler for \#BP (Breakpoint exception).
13033 */
13034static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13035{
13036 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13037 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13038
13039 /** @todo Try optimize this by not saving the entire guest state unless
13040 * really needed. */
13041 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13042 AssertRCReturn(rc, rc);
13043
13044 PVM pVM = pVCpu->CTX_SUFF(pVM);
13045 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
13046 if (rc == VINF_EM_RAW_GUEST_TRAP)
13047 {
13048 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13049 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13050 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13051 AssertRCReturn(rc, rc);
13052
13053 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13054 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13055 }
13056
13057 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
13058 return rc;
13059}
13060
13061
13062/**
13063 * VM-exit exception handler for \#AC (alignment check exception).
13064 */
13065static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13066{
13067 RT_NOREF_PV(pMixedCtx);
13068 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13069
13070 /*
13071 * Re-inject it. We'll detect any nesting before getting here.
13072 */
13073 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13074 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13075 AssertRCReturn(rc, rc);
13076 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13077
13078 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13079 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13080 return VINF_SUCCESS;
13081}
13082
13083
13084/**
13085 * VM-exit exception handler for \#DB (Debug exception).
13086 */
13087static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13088{
13089 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13090 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13091 Log6(("XcptDB\n"));
13092
13093 /*
13094 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13095 * for processing.
13096 */
13097 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13098 AssertRCReturn(rc, rc);
13099
13100 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13101 uint64_t uDR6 = X86_DR6_INIT_VAL;
13102 uDR6 |= ( pVmxTransient->uExitQualification
13103 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
13104
13105 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13106 if (rc == VINF_EM_RAW_GUEST_TRAP)
13107 {
13108 /*
13109 * The exception was for the guest. Update DR6, DR7.GD and
13110 * IA32_DEBUGCTL.LBR before forwarding it.
13111 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13112 */
13113 VMMRZCallRing3Disable(pVCpu);
13114 HM_DISABLE_PREEMPT();
13115
13116 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
13117 pMixedCtx->dr[6] |= uDR6;
13118 if (CPUMIsGuestDebugStateActive(pVCpu))
13119 ASMSetDR6(pMixedCtx->dr[6]);
13120
13121 HM_RESTORE_PREEMPT();
13122 VMMRZCallRing3Enable(pVCpu);
13123
13124 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13125 AssertRCReturn(rc, rc);
13126
13127 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13128 pMixedCtx->dr[7] &= ~X86_DR7_GD;
13129
13130 /* Paranoia. */
13131 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13132 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
13133
13134 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
13135 AssertRCReturn(rc, rc);
13136
13137 /*
13138 * Raise #DB in the guest.
13139 *
13140 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
13141 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
13142 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
13143 *
13144 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
13145 */
13146 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13147 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13148 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13149 AssertRCReturn(rc, rc);
13150 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13151 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13152 return VINF_SUCCESS;
13153 }
13154
13155 /*
13156 * Not a guest trap, must be a hypervisor related debug event then.
13157 * Update DR6 in case someone is interested in it.
13158 */
13159 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13160 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13161 CPUMSetHyperDR6(pVCpu, uDR6);
13162
13163 return rc;
13164}
13165
13166
13167/**
13168 * VM-exit exception handler for \#NM (Device-not-available exception: floating
13169 * point exception).
13170 */
13171static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13172{
13173 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13174
13175 /* We require CR0 and EFER. EFER is always up-to-date. */
13176 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13177 AssertRCReturn(rc, rc);
13178
13179 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
13180 VMMRZCallRing3Disable(pVCpu);
13181 HM_DISABLE_PREEMPT();
13182
13183 /* If the guest FPU was active at the time of the #NM VM-exit, then it's a guest fault. */
13184 if (pVmxTransient->fWasGuestFPUStateActive)
13185 {
13186 rc = VINF_EM_RAW_GUEST_TRAP;
13187 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
13188 }
13189 else
13190 {
13191#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13192 Assert(!pVmxTransient->fWasGuestFPUStateActive || pVCpu->hm.s.fUsingDebugLoop);
13193#endif
13194 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu);
13195 Assert( rc == VINF_EM_RAW_GUEST_TRAP
13196 || ((rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
13197 if (rc == VINF_CPUM_HOST_CR0_MODIFIED)
13198 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
13199 }
13200
13201 HM_RESTORE_PREEMPT();
13202 VMMRZCallRing3Enable(pVCpu);
13203
13204 if (rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED)
13205 {
13206 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
13207 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13208 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
13209 pVCpu->hm.s.fPreloadGuestFpu = true;
13210 }
13211 else
13212 {
13213 /* Forward #NM to the guest. */
13214 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
13215 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13216 AssertRCReturn(rc, rc);
13217 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13218 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
13219 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13220 }
13221
13222 return VINF_SUCCESS;
13223}
13224
13225
13226/**
13227 * VM-exit exception handler for \#GP (General-protection exception).
13228 *
13229 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13230 */
13231static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13232{
13233 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13234 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13235
13236 int rc;
13237 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13238 { /* likely */ }
13239 else
13240 {
13241#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13242 Assert(pVCpu->hm.s.fUsingDebugLoop);
13243#endif
13244 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13245 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13246 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13247 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13248 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13249 AssertRCReturn(rc, rc);
13250 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13251 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13252 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13253 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13254 return rc;
13255 }
13256
13257 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13258 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13259
13260 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13261 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13262 AssertRCReturn(rc, rc);
13263
13264 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13265 uint32_t cbOp = 0;
13266 PVM pVM = pVCpu->CTX_SUFF(pVM);
13267 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13268 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13269 if (RT_SUCCESS(rc))
13270 {
13271 rc = VINF_SUCCESS;
13272 Assert(cbOp == pDis->cbInstr);
13273 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13274 switch (pDis->pCurInstr->uOpcode)
13275 {
13276 case OP_CLI:
13277 {
13278 pMixedCtx->eflags.Bits.u1IF = 0;
13279 pMixedCtx->eflags.Bits.u1RF = 0;
13280 pMixedCtx->rip += pDis->cbInstr;
13281 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13282 if ( !fDbgStepping
13283 && pMixedCtx->eflags.Bits.u1TF)
13284 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13285 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13286 break;
13287 }
13288
13289 case OP_STI:
13290 {
13291 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13292 pMixedCtx->eflags.Bits.u1IF = 1;
13293 pMixedCtx->eflags.Bits.u1RF = 0;
13294 pMixedCtx->rip += pDis->cbInstr;
13295 if (!fOldIF)
13296 {
13297 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13298 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13299 }
13300 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13301 if ( !fDbgStepping
13302 && pMixedCtx->eflags.Bits.u1TF)
13303 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13304 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13305 break;
13306 }
13307
13308 case OP_HLT:
13309 {
13310 rc = VINF_EM_HALT;
13311 pMixedCtx->rip += pDis->cbInstr;
13312 pMixedCtx->eflags.Bits.u1RF = 0;
13313 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13314 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13315 break;
13316 }
13317
13318 case OP_POPF:
13319 {
13320 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13321 uint32_t cbParm;
13322 uint32_t uMask;
13323 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13324 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13325 {
13326 cbParm = 4;
13327 uMask = 0xffffffff;
13328 }
13329 else
13330 {
13331 cbParm = 2;
13332 uMask = 0xffff;
13333 }
13334
13335 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13336 RTGCPTR GCPtrStack = 0;
13337 X86EFLAGS Eflags;
13338 Eflags.u32 = 0;
13339 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13340 &GCPtrStack);
13341 if (RT_SUCCESS(rc))
13342 {
13343 Assert(sizeof(Eflags.u32) >= cbParm);
13344 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13345 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13346 }
13347 if (RT_FAILURE(rc))
13348 {
13349 rc = VERR_EM_INTERPRETER;
13350 break;
13351 }
13352 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13353 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13354 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13355 pMixedCtx->esp += cbParm;
13356 pMixedCtx->esp &= uMask;
13357 pMixedCtx->rip += pDis->cbInstr;
13358 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13359 | HM_CHANGED_GUEST_RSP
13360 | HM_CHANGED_GUEST_RFLAGS);
13361 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13362 POPF restores EFLAGS.TF. */
13363 if ( !fDbgStepping
13364 && fGstStepping)
13365 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13366 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13367 break;
13368 }
13369
13370 case OP_PUSHF:
13371 {
13372 uint32_t cbParm;
13373 uint32_t uMask;
13374 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13375 {
13376 cbParm = 4;
13377 uMask = 0xffffffff;
13378 }
13379 else
13380 {
13381 cbParm = 2;
13382 uMask = 0xffff;
13383 }
13384
13385 /* Get the stack pointer & push the contents of eflags onto the stack. */
13386 RTGCPTR GCPtrStack = 0;
13387 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13388 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13389 if (RT_FAILURE(rc))
13390 {
13391 rc = VERR_EM_INTERPRETER;
13392 break;
13393 }
13394 X86EFLAGS Eflags = pMixedCtx->eflags;
13395 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13396 Eflags.Bits.u1RF = 0;
13397 Eflags.Bits.u1VM = 0;
13398
13399 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13400 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13401 {
13402 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13403 rc = VERR_EM_INTERPRETER;
13404 break;
13405 }
13406 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13407 pMixedCtx->esp -= cbParm;
13408 pMixedCtx->esp &= uMask;
13409 pMixedCtx->rip += pDis->cbInstr;
13410 pMixedCtx->eflags.Bits.u1RF = 0;
13411 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13412 | HM_CHANGED_GUEST_RSP
13413 | HM_CHANGED_GUEST_RFLAGS);
13414 if ( !fDbgStepping
13415 && pMixedCtx->eflags.Bits.u1TF)
13416 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13417 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13418 break;
13419 }
13420
13421 case OP_IRET:
13422 {
13423 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13424 * instruction reference. */
13425 RTGCPTR GCPtrStack = 0;
13426 uint32_t uMask = 0xffff;
13427 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13428 uint16_t aIretFrame[3];
13429 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13430 {
13431 rc = VERR_EM_INTERPRETER;
13432 break;
13433 }
13434 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13435 &GCPtrStack);
13436 if (RT_SUCCESS(rc))
13437 {
13438 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13439 PGMACCESSORIGIN_HM));
13440 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13441 }
13442 if (RT_FAILURE(rc))
13443 {
13444 rc = VERR_EM_INTERPRETER;
13445 break;
13446 }
13447 pMixedCtx->eip = 0;
13448 pMixedCtx->ip = aIretFrame[0];
13449 pMixedCtx->cs.Sel = aIretFrame[1];
13450 pMixedCtx->cs.ValidSel = aIretFrame[1];
13451 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13452 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13453 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13454 pMixedCtx->sp += sizeof(aIretFrame);
13455 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13456 | HM_CHANGED_GUEST_SEGMENT_REGS
13457 | HM_CHANGED_GUEST_RSP
13458 | HM_CHANGED_GUEST_RFLAGS);
13459 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13460 if ( !fDbgStepping
13461 && fGstStepping)
13462 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13463 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13464 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13465 break;
13466 }
13467
13468 case OP_INT:
13469 {
13470 uint16_t uVector = pDis->Param1.uValue & 0xff;
13471 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13472 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13473 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13474 break;
13475 }
13476
13477 case OP_INTO:
13478 {
13479 if (pMixedCtx->eflags.Bits.u1OF)
13480 {
13481 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13482 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13483 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13484 }
13485 else
13486 {
13487 pMixedCtx->eflags.Bits.u1RF = 0;
13488 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13489 }
13490 break;
13491 }
13492
13493 default:
13494 {
13495 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13496 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13497 EMCODETYPE_SUPERVISOR);
13498 rc = VBOXSTRICTRC_VAL(rc2);
13499 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13500 /** @todo We have to set pending-debug exceptions here when the guest is
13501 * single-stepping depending on the instruction that was interpreted. */
13502 Log4(("#GP rc=%Rrc\n", rc));
13503 break;
13504 }
13505 }
13506 }
13507 else
13508 rc = VERR_EM_INTERPRETER;
13509
13510 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13511 ("#GP Unexpected rc=%Rrc\n", rc));
13512 return rc;
13513}
13514
13515
13516/**
13517 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13518 * the exception reported in the VMX transient structure back into the VM.
13519 *
13520 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13521 * up-to-date.
13522 */
13523static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13524{
13525 RT_NOREF_PV(pMixedCtx);
13526 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13527#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13528 Assert(pVCpu->hm.s.fUsingDebugLoop);
13529#endif
13530
13531 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13532 hmR0VmxCheckExitDueToEventDelivery(). */
13533 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13534 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13535 AssertRCReturn(rc, rc);
13536 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13537
13538#ifdef DEBUG_ramshankar
13539 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13540 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13541 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13542#endif
13543
13544 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13545 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13546 return VINF_SUCCESS;
13547}
13548
13549
13550/**
13551 * VM-exit exception handler for \#PF (Page-fault exception).
13552 */
13553static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13554{
13555 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13556 PVM pVM = pVCpu->CTX_SUFF(pVM);
13557 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13558 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13559 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13560 AssertRCReturn(rc, rc);
13561
13562 if (!pVM->hm.s.fNestedPaging)
13563 { /* likely */ }
13564 else
13565 {
13566#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13567 Assert(pVCpu->hm.s.fUsingDebugLoop);
13568#endif
13569 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13570 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13571 {
13572 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13573 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13574 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13575 }
13576 else
13577 {
13578 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13579 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13580 Log4(("Pending #DF due to vectoring #PF. NP\n"));
13581 }
13582 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13583 return rc;
13584 }
13585
13586 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13587 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13588 if (pVmxTransient->fVectoringPF)
13589 {
13590 Assert(pVCpu->hm.s.Event.fPending);
13591 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13592 }
13593
13594 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13595 AssertRCReturn(rc, rc);
13596
13597 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13598 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13599
13600 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13601 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13602 (RTGCPTR)pVmxTransient->uExitQualification);
13603
13604 Log4(("#PF: rc=%Rrc\n", rc));
13605 if (rc == VINF_SUCCESS)
13606 {
13607#if 0
13608 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13609 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
13610 * memory? We don't update the whole state here... */
13611 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13612 | HM_CHANGED_GUEST_RSP
13613 | HM_CHANGED_GUEST_RFLAGS
13614 | HM_CHANGED_VMX_GUEST_APIC_STATE);
13615#else
13616 /*
13617 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13618 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13619 */
13620 /** @todo take advantage of CPUM changed flags instead of brute forcing. */
13621 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13622#endif
13623 TRPMResetTrap(pVCpu);
13624 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13625 return rc;
13626 }
13627
13628 if (rc == VINF_EM_RAW_GUEST_TRAP)
13629 {
13630 if (!pVmxTransient->fVectoringDoublePF)
13631 {
13632 /* It's a guest page fault and needs to be reflected to the guest. */
13633 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13634 TRPMResetTrap(pVCpu);
13635 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13636 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13637 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13638 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13639 }
13640 else
13641 {
13642 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13643 TRPMResetTrap(pVCpu);
13644 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13645 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13646 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
13647 }
13648
13649 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13650 return VINF_SUCCESS;
13651 }
13652
13653 TRPMResetTrap(pVCpu);
13654 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13655 return rc;
13656}
13657
13658/** @} */
13659
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette