VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 65276

Last change on this file since 65276 was 65276, checked in by vboxsync, 8 years ago

VMM/HMVMXR0: Update comment.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 584.5 KB
Line 
1/* $Id: HMVMXR0.cpp 65276 2017-01-13 08:52:51Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2016 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#include <iprt/x86.h>
24#include <iprt/asm-amd64-x86.h>
25#include <iprt/thread.h>
26
27#include <VBox/vmm/pdmapi.h>
28#include <VBox/vmm/dbgf.h>
29#include <VBox/vmm/iem.h>
30#include <VBox/vmm/iom.h>
31#include <VBox/vmm/selm.h>
32#include <VBox/vmm/tm.h>
33#include <VBox/vmm/gim.h>
34#include <VBox/vmm/apic.h>
35#ifdef VBOX_WITH_REM
36# include <VBox/vmm/rem.h>
37#endif
38#include "HMInternal.h"
39#include <VBox/vmm/vm.h>
40#include "HMVMXR0.h"
41#include "dtrace/VBoxVMM.h"
42
43#ifdef DEBUG_ramshankar
44# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
45# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
46# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
47# define HMVMX_ALWAYS_CHECK_GUEST_STATE
48# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
49# define HMVMX_ALWAYS_TRAP_PF
50# define HMVMX_ALWAYS_SWAP_FPU_STATE
51# define HMVMX_ALWAYS_FLUSH_TLB
52# define HMVMX_ALWAYS_SWAP_EFER
53#endif
54
55
56/*********************************************************************************************************************************
57* Defined Constants And Macros *
58*********************************************************************************************************************************/
59/** Use the function table. */
60#define HMVMX_USE_FUNCTION_TABLE
61
62/** Determine which tagged-TLB flush handler to use. */
63#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68/** @name Updated-guest-state flags.
69 * @{ */
70#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
71#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
72#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
73#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
74#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
75#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
76#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
77#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
78#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
79#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
80#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
81#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
82#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
83#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
84#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
85#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
86#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
87#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
88#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
89#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
90#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
91 | HMVMX_UPDATED_GUEST_RSP \
92 | HMVMX_UPDATED_GUEST_RFLAGS \
93 | HMVMX_UPDATED_GUEST_CR0 \
94 | HMVMX_UPDATED_GUEST_CR3 \
95 | HMVMX_UPDATED_GUEST_CR4 \
96 | HMVMX_UPDATED_GUEST_GDTR \
97 | HMVMX_UPDATED_GUEST_IDTR \
98 | HMVMX_UPDATED_GUEST_LDTR \
99 | HMVMX_UPDATED_GUEST_TR \
100 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
101 | HMVMX_UPDATED_GUEST_DEBUG \
102 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
103 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
104 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
105 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
106 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
107 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
108 | HMVMX_UPDATED_GUEST_INTR_STATE \
109 | HMVMX_UPDATED_GUEST_APIC_STATE)
110/** @} */
111
112/** @name
113 * Flags to skip redundant reads of some common VMCS fields that are not part of
114 * the guest-CPU state but are in the transient structure.
115 */
116#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
117#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
118#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
119#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
120#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
123/** @} */
124
125/** @name
126 * States of the VMCS.
127 *
128 * This does not reflect all possible VMCS states but currently only those
129 * needed for maintaining the VMCS consistently even when thread-context hooks
130 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
131 */
132#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
133#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
134#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
135/** @} */
136
137/**
138 * Exception bitmap mask for real-mode guests (real-on-v86).
139 *
140 * We need to intercept all exceptions manually except:
141 * - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
142 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
143 * due to bugs in Intel CPUs.
144 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
145 * support.
146 */
147#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
148 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
149 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
150 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
151 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
152 /* RT_BIT(X86_XCPT_MF) always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
153 | RT_BIT(X86_XCPT_XF))
154
155/**
156 * Exception bitmap mask for all contributory exceptions.
157 *
158 * Page fault is deliberately excluded here as it's conditional as to whether
159 * it's contributory or benign. Page faults are handled separately.
160 */
161#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
162 | RT_BIT(X86_XCPT_DE))
163
164/** Maximum VM-instruction error number. */
165#define HMVMX_INSTR_ERROR_MAX 28
166
167/** Profiling macro. */
168#ifdef HM_PROFILE_EXIT_DISPATCH
169# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
170# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
171#else
172# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
173# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
174#endif
175
176/** Assert that preemption is disabled or covered by thread-context hooks. */
177#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
178 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
179
180/** Assert that we haven't migrated CPUs when thread-context hooks are not
181 * used. */
182#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
183 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
184 ("Illegal migration! Entered on CPU %u Current %u\n", \
185 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
186
187/** Helper macro for VM-exit handlers called unexpectedly. */
188#define HMVMX_RETURN_UNEXPECTED_EXIT() \
189 do { \
190 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
191 return VERR_VMX_UNEXPECTED_EXIT; \
192 } while (0)
193
194
195/*********************************************************************************************************************************
196* Structures and Typedefs *
197*********************************************************************************************************************************/
198/**
199 * VMX transient state.
200 *
201 * A state structure for holding miscellaneous information across
202 * VMX non-root operation and restored after the transition.
203 */
204typedef struct VMXTRANSIENT
205{
206 /** The host's rflags/eflags. */
207 RTCCUINTREG fEFlags;
208#if HC_ARCH_BITS == 32
209 uint32_t u32Alignment0;
210#endif
211 /** The guest's TPR value used for TPR shadowing. */
212 uint8_t u8GuestTpr;
213 /** Alignment. */
214 uint8_t abAlignment0[7];
215
216 /** The basic VM-exit reason. */
217 uint16_t uExitReason;
218 /** Alignment. */
219 uint16_t u16Alignment0;
220 /** The VM-exit interruption error code. */
221 uint32_t uExitIntErrorCode;
222 /** The VM-exit exit code qualification. */
223 uint64_t uExitQualification;
224
225 /** The VM-exit interruption-information field. */
226 uint32_t uExitIntInfo;
227 /** The VM-exit instruction-length field. */
228 uint32_t cbInstr;
229 /** The VM-exit instruction-information field. */
230 union
231 {
232 /** Plain unsigned int representation. */
233 uint32_t u;
234 /** INS and OUTS information. */
235 struct
236 {
237 uint32_t u7Reserved0 : 7;
238 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
239 uint32_t u3AddrSize : 3;
240 uint32_t u5Reserved1 : 5;
241 /** The segment register (X86_SREG_XXX). */
242 uint32_t iSegReg : 3;
243 uint32_t uReserved2 : 14;
244 } StrIo;
245 } ExitInstrInfo;
246 /** Whether the VM-entry failed or not. */
247 bool fVMEntryFailed;
248 /** Alignment. */
249 uint8_t abAlignment1[3];
250
251 /** The VM-entry interruption-information field. */
252 uint32_t uEntryIntInfo;
253 /** The VM-entry exception error code field. */
254 uint32_t uEntryXcptErrorCode;
255 /** The VM-entry instruction length field. */
256 uint32_t cbEntryInstr;
257
258 /** IDT-vectoring information field. */
259 uint32_t uIdtVectoringInfo;
260 /** IDT-vectoring error code. */
261 uint32_t uIdtVectoringErrorCode;
262
263 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
264 uint32_t fVmcsFieldsRead;
265
266 /** Whether the guest FPU was active at the time of VM-exit. */
267 bool fWasGuestFPUStateActive;
268 /** Whether the guest debug state was active at the time of VM-exit. */
269 bool fWasGuestDebugStateActive;
270 /** Whether the hyper debug state was active at the time of VM-exit. */
271 bool fWasHyperDebugStateActive;
272 /** Whether TSC-offsetting should be setup before VM-entry. */
273 bool fUpdateTscOffsettingAndPreemptTimer;
274 /** Whether the VM-exit was caused by a page-fault during delivery of a
275 * contributory exception or a page-fault. */
276 bool fVectoringDoublePF;
277 /** Whether the VM-exit was caused by a page-fault during delivery of an
278 * external interrupt or NMI. */
279 bool fVectoringPF;
280} VMXTRANSIENT;
281AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
282AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
283AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
284AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
285AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
286/** Pointer to VMX transient state. */
287typedef VMXTRANSIENT *PVMXTRANSIENT;
288
289
290/**
291 * MSR-bitmap read permissions.
292 */
293typedef enum VMXMSREXITREAD
294{
295 /** Reading this MSR causes a VM-exit. */
296 VMXMSREXIT_INTERCEPT_READ = 0xb,
297 /** Reading this MSR does not cause a VM-exit. */
298 VMXMSREXIT_PASSTHRU_READ
299} VMXMSREXITREAD;
300/** Pointer to MSR-bitmap read permissions. */
301typedef VMXMSREXITREAD* PVMXMSREXITREAD;
302
303/**
304 * MSR-bitmap write permissions.
305 */
306typedef enum VMXMSREXITWRITE
307{
308 /** Writing to this MSR causes a VM-exit. */
309 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
310 /** Writing to this MSR does not cause a VM-exit. */
311 VMXMSREXIT_PASSTHRU_WRITE
312} VMXMSREXITWRITE;
313/** Pointer to MSR-bitmap write permissions. */
314typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
315
316
317/**
318 * VMX VM-exit handler.
319 *
320 * @returns Strict VBox status code (i.e. informational status codes too).
321 * @param pVCpu The cross context virtual CPU structure.
322 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
323 * out-of-sync. Make sure to update the required
324 * fields before using them.
325 * @param pVmxTransient Pointer to the VMX-transient structure.
326 */
327#ifndef HMVMX_USE_FUNCTION_TABLE
328typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
329#else
330typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
331/** Pointer to VM-exit handler. */
332typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
333#endif
334
335/**
336 * VMX VM-exit handler, non-strict status code.
337 *
338 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
339 *
340 * @returns VBox status code, no informational status code returned.
341 * @param pVCpu The cross context virtual CPU structure.
342 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
343 * out-of-sync. Make sure to update the required
344 * fields before using them.
345 * @param pVmxTransient Pointer to the VMX-transient structure.
346 *
347 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
348 * use of that status code will be replaced with VINF_EM_SOMETHING
349 * later when switching over to IEM.
350 */
351#ifndef HMVMX_USE_FUNCTION_TABLE
352typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
353#else
354typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
355#endif
356
357
358/*********************************************************************************************************************************
359* Internal Functions *
360*********************************************************************************************************************************/
361static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
362static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
363static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
364static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
365 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
366 bool fStepping, uint32_t *puIntState);
367#if HC_ARCH_BITS == 32
368static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
369#endif
370#ifndef HMVMX_USE_FUNCTION_TABLE
371DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
372# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
373# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
374#else
375# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
376# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
377#endif
378
379
380/** @name VM-exit handlers.
381 * @{
382 */
383static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
384static FNVMXEXITHANDLER hmR0VmxExitExtInt;
385static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
386static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
387static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
388static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
389static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
390static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
391static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
392static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
393static FNVMXEXITHANDLER hmR0VmxExitCpuid;
394static FNVMXEXITHANDLER hmR0VmxExitGetsec;
395static FNVMXEXITHANDLER hmR0VmxExitHlt;
396static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
397static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
398static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
399static FNVMXEXITHANDLER hmR0VmxExitVmcall;
400static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
401static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
402static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
403static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
404static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
405static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
406static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
407static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
408static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
409static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
410static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
411static FNVMXEXITHANDLER hmR0VmxExitMwait;
412static FNVMXEXITHANDLER hmR0VmxExitMtf;
413static FNVMXEXITHANDLER hmR0VmxExitMonitor;
414static FNVMXEXITHANDLER hmR0VmxExitPause;
415static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
416static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
417static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
418static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
419static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
420static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
421static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
422static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
423static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
424static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
425static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
426static FNVMXEXITHANDLER hmR0VmxExitRdrand;
427static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
428/** @} */
429
430static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
431static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
432static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
433static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
434static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
435static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
436static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
437static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
438static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
439
440
441/*********************************************************************************************************************************
442* Global Variables *
443*********************************************************************************************************************************/
444#ifdef HMVMX_USE_FUNCTION_TABLE
445
446/**
447 * VMX_EXIT dispatch table.
448 */
449static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
450{
451 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
452 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
453 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
454 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
455 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
456 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
457 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
458 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
459 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
460 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
461 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
462 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
463 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
464 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
465 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
466 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
467 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
468 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
469 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
470 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
471 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
472 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
473 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
474 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
475 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
476 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
477 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
478 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
479 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
480 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
481 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
482 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
483 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
484 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
485 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
486 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
487 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
488 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
489 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
490 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
491 /* 40 UNDEFINED */ hmR0VmxExitPause,
492 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
493 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
494 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
495 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
496 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
497 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
498 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
499 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
500 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
501 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
502 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
503 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
504 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
505 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
506 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
507 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
508 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
509 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
510 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
511 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
512 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
513 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
514 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
515 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
516};
517#endif /* HMVMX_USE_FUNCTION_TABLE */
518
519#ifdef VBOX_STRICT
520static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
521{
522 /* 0 */ "(Not Used)",
523 /* 1 */ "VMCALL executed in VMX root operation.",
524 /* 2 */ "VMCLEAR with invalid physical address.",
525 /* 3 */ "VMCLEAR with VMXON pointer.",
526 /* 4 */ "VMLAUNCH with non-clear VMCS.",
527 /* 5 */ "VMRESUME with non-launched VMCS.",
528 /* 6 */ "VMRESUME after VMXOFF",
529 /* 7 */ "VM-entry with invalid control fields.",
530 /* 8 */ "VM-entry with invalid host state fields.",
531 /* 9 */ "VMPTRLD with invalid physical address.",
532 /* 10 */ "VMPTRLD with VMXON pointer.",
533 /* 11 */ "VMPTRLD with incorrect revision identifier.",
534 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
535 /* 13 */ "VMWRITE to read-only VMCS component.",
536 /* 14 */ "(Not Used)",
537 /* 15 */ "VMXON executed in VMX root operation.",
538 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
539 /* 17 */ "VM-entry with non-launched executing VMCS.",
540 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
541 /* 19 */ "VMCALL with non-clear VMCS.",
542 /* 20 */ "VMCALL with invalid VM-exit control fields.",
543 /* 21 */ "(Not Used)",
544 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
545 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
546 /* 24 */ "VMCALL with invalid SMM-monitor features.",
547 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
548 /* 26 */ "VM-entry with events blocked by MOV SS.",
549 /* 27 */ "(Not Used)",
550 /* 28 */ "Invalid operand to INVEPT/INVVPID."
551};
552#endif /* VBOX_STRICT */
553
554
555
556/**
557 * Updates the VM's last error record.
558 *
559 * If there was a VMX instruction error, reads the error data from the VMCS and
560 * updates VCPU's last error record as well.
561 *
562 * @param pVM The cross context VM structure.
563 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
564 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
565 * VERR_VMX_INVALID_VMCS_FIELD.
566 * @param rc The error code.
567 */
568static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
569{
570 AssertPtr(pVM);
571 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
572 || rc == VERR_VMX_UNABLE_TO_START_VM)
573 {
574 AssertPtrReturnVoid(pVCpu);
575 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
576 }
577 pVM->hm.s.lLastError = rc;
578}
579
580
581/**
582 * Reads the VM-entry interruption-information field from the VMCS into the VMX
583 * transient structure.
584 *
585 * @returns VBox status code.
586 * @param pVmxTransient Pointer to the VMX transient structure.
587 *
588 * @remarks No-long-jump zone!!!
589 */
590DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
591{
592 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
593 AssertRCReturn(rc, rc);
594 return VINF_SUCCESS;
595}
596
597
598#ifdef VBOX_STRICT
599/**
600 * Reads the VM-entry exception error code field from the VMCS into
601 * the VMX transient structure.
602 *
603 * @returns VBox status code.
604 * @param pVmxTransient Pointer to the VMX transient structure.
605 *
606 * @remarks No-long-jump zone!!!
607 */
608DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
609{
610 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
611 AssertRCReturn(rc, rc);
612 return VINF_SUCCESS;
613}
614#endif /* VBOX_STRICT */
615
616
617#ifdef VBOX_STRICT
618/**
619 * Reads the VM-entry exception error code field from the VMCS into
620 * the VMX transient structure.
621 *
622 * @returns VBox status code.
623 * @param pVmxTransient Pointer to the VMX transient structure.
624 *
625 * @remarks No-long-jump zone!!!
626 */
627DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
628{
629 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
630 AssertRCReturn(rc, rc);
631 return VINF_SUCCESS;
632}
633#endif /* VBOX_STRICT */
634
635
636/**
637 * Reads the VM-exit interruption-information field from the VMCS into the VMX
638 * transient structure.
639 *
640 * @returns VBox status code.
641 * @param pVmxTransient Pointer to the VMX transient structure.
642 */
643DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
644{
645 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
646 {
647 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
648 AssertRCReturn(rc, rc);
649 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
650 }
651 return VINF_SUCCESS;
652}
653
654
655/**
656 * Reads the VM-exit interruption error code from the VMCS into the VMX
657 * transient structure.
658 *
659 * @returns VBox status code.
660 * @param pVmxTransient Pointer to the VMX transient structure.
661 */
662DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
663{
664 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
665 {
666 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
667 AssertRCReturn(rc, rc);
668 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
669 }
670 return VINF_SUCCESS;
671}
672
673
674/**
675 * Reads the VM-exit instruction length field from the VMCS into the VMX
676 * transient structure.
677 *
678 * @returns VBox status code.
679 * @param pVmxTransient Pointer to the VMX transient structure.
680 */
681DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
682{
683 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
684 {
685 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
686 AssertRCReturn(rc, rc);
687 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
688 }
689 return VINF_SUCCESS;
690}
691
692
693/**
694 * Reads the VM-exit instruction-information field from the VMCS into
695 * the VMX transient structure.
696 *
697 * @returns VBox status code.
698 * @param pVmxTransient Pointer to the VMX transient structure.
699 */
700DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
701{
702 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
703 {
704 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
705 AssertRCReturn(rc, rc);
706 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
707 }
708 return VINF_SUCCESS;
709}
710
711
712/**
713 * Reads the exit code qualification from the VMCS into the VMX transient
714 * structure.
715 *
716 * @returns VBox status code.
717 * @param pVCpu The cross context virtual CPU structure of the
718 * calling EMT. (Required for the VMCS cache case.)
719 * @param pVmxTransient Pointer to the VMX transient structure.
720 */
721DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
722{
723 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
724 {
725 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
726 AssertRCReturn(rc, rc);
727 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
728 }
729 return VINF_SUCCESS;
730}
731
732
733/**
734 * Reads the IDT-vectoring information field from the VMCS into the VMX
735 * transient structure.
736 *
737 * @returns VBox status code.
738 * @param pVmxTransient Pointer to the VMX transient structure.
739 *
740 * @remarks No-long-jump zone!!!
741 */
742DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
743{
744 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
745 {
746 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
747 AssertRCReturn(rc, rc);
748 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
749 }
750 return VINF_SUCCESS;
751}
752
753
754/**
755 * Reads the IDT-vectoring error code from the VMCS into the VMX
756 * transient structure.
757 *
758 * @returns VBox status code.
759 * @param pVmxTransient Pointer to the VMX transient structure.
760 */
761DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
762{
763 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
764 {
765 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
766 AssertRCReturn(rc, rc);
767 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
768 }
769 return VINF_SUCCESS;
770}
771
772
773/**
774 * Enters VMX root mode operation on the current CPU.
775 *
776 * @returns VBox status code.
777 * @param pVM The cross context VM structure. Can be
778 * NULL, after a resume.
779 * @param HCPhysCpuPage Physical address of the VMXON region.
780 * @param pvCpuPage Pointer to the VMXON region.
781 */
782static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
783{
784 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
785 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
786 Assert(pvCpuPage);
787 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
788
789 if (pVM)
790 {
791 /* Write the VMCS revision dword to the VMXON region. */
792 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
793 }
794
795 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
796 RTCCUINTREG fEFlags = ASMIntDisableFlags();
797
798 /* Enable the VMX bit in CR4 if necessary. */
799 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
800
801 /* Enter VMX root mode. */
802 int rc = VMXEnable(HCPhysCpuPage);
803 if (RT_FAILURE(rc))
804 {
805 if (!(uOldCr4 & X86_CR4_VMXE))
806 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
807
808 if (pVM)
809 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
810 }
811
812 /* Restore interrupts. */
813 ASMSetFlags(fEFlags);
814 return rc;
815}
816
817
818/**
819 * Exits VMX root mode operation on the current CPU.
820 *
821 * @returns VBox status code.
822 */
823static int hmR0VmxLeaveRootMode(void)
824{
825 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
826
827 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
828 RTCCUINTREG fEFlags = ASMIntDisableFlags();
829
830 /* If we're for some reason not in VMX root mode, then don't leave it. */
831 RTCCUINTREG uHostCR4 = ASMGetCR4();
832
833 int rc;
834 if (uHostCR4 & X86_CR4_VMXE)
835 {
836 /* Exit VMX root mode and clear the VMX bit in CR4. */
837 VMXDisable();
838 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
839 rc = VINF_SUCCESS;
840 }
841 else
842 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
843
844 /* Restore interrupts. */
845 ASMSetFlags(fEFlags);
846 return rc;
847}
848
849
850/**
851 * Allocates and maps one physically contiguous page. The allocated page is
852 * zero'd out. (Used by various VT-x structures).
853 *
854 * @returns IPRT status code.
855 * @param pMemObj Pointer to the ring-0 memory object.
856 * @param ppVirt Where to store the virtual address of the
857 * allocation.
858 * @param pHCPhys Where to store the physical address of the
859 * allocation.
860 */
861DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
862{
863 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
864 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
865 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
866
867 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
868 if (RT_FAILURE(rc))
869 return rc;
870 *ppVirt = RTR0MemObjAddress(*pMemObj);
871 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
872 ASMMemZero32(*ppVirt, PAGE_SIZE);
873 return VINF_SUCCESS;
874}
875
876
877/**
878 * Frees and unmaps an allocated physical page.
879 *
880 * @param pMemObj Pointer to the ring-0 memory object.
881 * @param ppVirt Where to re-initialize the virtual address of
882 * allocation as 0.
883 * @param pHCPhys Where to re-initialize the physical address of the
884 * allocation as 0.
885 */
886DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
887{
888 AssertPtr(pMemObj);
889 AssertPtr(ppVirt);
890 AssertPtr(pHCPhys);
891 if (*pMemObj != NIL_RTR0MEMOBJ)
892 {
893 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
894 AssertRC(rc);
895 *pMemObj = NIL_RTR0MEMOBJ;
896 *ppVirt = 0;
897 *pHCPhys = 0;
898 }
899}
900
901
902/**
903 * Worker function to free VT-x related structures.
904 *
905 * @returns IPRT status code.
906 * @param pVM The cross context VM structure.
907 */
908static void hmR0VmxStructsFree(PVM pVM)
909{
910 for (VMCPUID i = 0; i < pVM->cCpus; i++)
911 {
912 PVMCPU pVCpu = &pVM->aCpus[i];
913 AssertPtr(pVCpu);
914
915 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
916 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
917
918 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
919 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
920
921 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
922 }
923
924 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
925#ifdef VBOX_WITH_CRASHDUMP_MAGIC
926 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
927#endif
928}
929
930
931/**
932 * Worker function to allocate VT-x related VM structures.
933 *
934 * @returns IPRT status code.
935 * @param pVM The cross context VM structure.
936 */
937static int hmR0VmxStructsAlloc(PVM pVM)
938{
939 /*
940 * Initialize members up-front so we can cleanup properly on allocation failure.
941 */
942#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
943 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
944 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
945 pVM->hm.s.vmx.HCPhys##a_Name = 0;
946
947#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
948 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
949 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
950 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
951
952#ifdef VBOX_WITH_CRASHDUMP_MAGIC
953 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
954#endif
955 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
956
957 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
958 for (VMCPUID i = 0; i < pVM->cCpus; i++)
959 {
960 PVMCPU pVCpu = &pVM->aCpus[i];
961 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
962 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
963 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
964 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
965 }
966#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
967#undef VMXLOCAL_INIT_VM_MEMOBJ
968
969 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
970 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
971 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
972 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
973
974 /*
975 * Allocate all the VT-x structures.
976 */
977 int rc = VINF_SUCCESS;
978#ifdef VBOX_WITH_CRASHDUMP_MAGIC
979 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
980 if (RT_FAILURE(rc))
981 goto cleanup;
982 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
983 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
984#endif
985
986 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
987 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
988 {
989 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
990 &pVM->hm.s.vmx.HCPhysApicAccess);
991 if (RT_FAILURE(rc))
992 goto cleanup;
993 }
994
995 /*
996 * Initialize per-VCPU VT-x structures.
997 */
998 for (VMCPUID i = 0; i < pVM->cCpus; i++)
999 {
1000 PVMCPU pVCpu = &pVM->aCpus[i];
1001 AssertPtr(pVCpu);
1002
1003 /* Allocate the VM control structure (VMCS). */
1004 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1005 if (RT_FAILURE(rc))
1006 goto cleanup;
1007
1008 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
1009 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1010 {
1011 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1012 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1013 if (RT_FAILURE(rc))
1014 goto cleanup;
1015 }
1016
1017 /*
1018 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1019 * transparent accesses of specific MSRs.
1020 *
1021 * If the condition for enabling MSR bitmaps changes here, don't forget to
1022 * update HMAreMsrBitmapsAvailable().
1023 */
1024 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1025 {
1026 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1027 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1028 if (RT_FAILURE(rc))
1029 goto cleanup;
1030 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1031 }
1032
1033 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1034 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1035 if (RT_FAILURE(rc))
1036 goto cleanup;
1037
1038 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1039 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1040 if (RT_FAILURE(rc))
1041 goto cleanup;
1042 }
1043
1044 return VINF_SUCCESS;
1045
1046cleanup:
1047 hmR0VmxStructsFree(pVM);
1048 return rc;
1049}
1050
1051
1052/**
1053 * Does global VT-x initialization (called during module initialization).
1054 *
1055 * @returns VBox status code.
1056 */
1057VMMR0DECL(int) VMXR0GlobalInit(void)
1058{
1059#ifdef HMVMX_USE_FUNCTION_TABLE
1060 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1061# ifdef VBOX_STRICT
1062 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1063 Assert(g_apfnVMExitHandlers[i]);
1064# endif
1065#endif
1066 return VINF_SUCCESS;
1067}
1068
1069
1070/**
1071 * Does global VT-x termination (called during module termination).
1072 */
1073VMMR0DECL(void) VMXR0GlobalTerm()
1074{
1075 /* Nothing to do currently. */
1076}
1077
1078
1079/**
1080 * Sets up and activates VT-x on the current CPU.
1081 *
1082 * @returns VBox status code.
1083 * @param pCpu Pointer to the global CPU info struct.
1084 * @param pVM The cross context VM structure. Can be
1085 * NULL after a host resume operation.
1086 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1087 * fEnabledByHost is @c true).
1088 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1089 * @a fEnabledByHost is @c true).
1090 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1091 * enable VT-x on the host.
1092 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1093 */
1094VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1095 void *pvMsrs)
1096{
1097 Assert(pCpu);
1098 Assert(pvMsrs);
1099 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1100
1101 /* Enable VT-x if it's not already enabled by the host. */
1102 if (!fEnabledByHost)
1103 {
1104 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1105 if (RT_FAILURE(rc))
1106 return rc;
1107 }
1108
1109 /*
1110 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1111 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1112 */
1113 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1114 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1115 {
1116 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1117 pCpu->fFlushAsidBeforeUse = false;
1118 }
1119 else
1120 pCpu->fFlushAsidBeforeUse = true;
1121
1122 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1123 ++pCpu->cTlbFlushes;
1124
1125 return VINF_SUCCESS;
1126}
1127
1128
1129/**
1130 * Deactivates VT-x on the current CPU.
1131 *
1132 * @returns VBox status code.
1133 * @param pCpu Pointer to the global CPU info struct.
1134 * @param pvCpuPage Pointer to the VMXON region.
1135 * @param HCPhysCpuPage Physical address of the VMXON region.
1136 *
1137 * @remarks This function should never be called when SUPR0EnableVTx() or
1138 * similar was used to enable VT-x on the host.
1139 */
1140VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1141{
1142 NOREF(pCpu);
1143 NOREF(pvCpuPage);
1144 NOREF(HCPhysCpuPage);
1145
1146 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1147 return hmR0VmxLeaveRootMode();
1148}
1149
1150
1151/**
1152 * Sets the permission bits for the specified MSR in the MSR bitmap.
1153 *
1154 * @param pVCpu The cross context virtual CPU structure.
1155 * @param uMsr The MSR value.
1156 * @param enmRead Whether reading this MSR causes a VM-exit.
1157 * @param enmWrite Whether writing this MSR causes a VM-exit.
1158 */
1159static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1160{
1161 int32_t iBit;
1162 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1163
1164 /*
1165 * Layout:
1166 * 0x000 - 0x3ff - Low MSR read bits
1167 * 0x400 - 0x7ff - High MSR read bits
1168 * 0x800 - 0xbff - Low MSR write bits
1169 * 0xc00 - 0xfff - High MSR write bits
1170 */
1171 if (uMsr <= 0x00001FFF)
1172 iBit = uMsr;
1173 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1174 {
1175 iBit = uMsr - UINT32_C(0xC0000000);
1176 pbMsrBitmap += 0x400;
1177 }
1178 else
1179 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1180
1181 Assert(iBit <= 0x1fff);
1182 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1183 ASMBitSet(pbMsrBitmap, iBit);
1184 else
1185 ASMBitClear(pbMsrBitmap, iBit);
1186
1187 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1188 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1189 else
1190 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1191}
1192
1193
1194#ifdef VBOX_STRICT
1195/**
1196 * Gets the permission bits for the specified MSR in the MSR bitmap.
1197 *
1198 * @returns VBox status code.
1199 * @retval VINF_SUCCESS if the specified MSR is found.
1200 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1201 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1202 *
1203 * @param pVCpu The cross context virtual CPU structure.
1204 * @param uMsr The MSR.
1205 * @param penmRead Where to store the read permissions.
1206 * @param penmWrite Where to store the write permissions.
1207 */
1208static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1209{
1210 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1211 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1212 int32_t iBit;
1213 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1214
1215 /* See hmR0VmxSetMsrPermission() for the layout. */
1216 if (uMsr <= 0x00001FFF)
1217 iBit = uMsr;
1218 else if ( uMsr >= 0xC0000000
1219 && uMsr <= 0xC0001FFF)
1220 {
1221 iBit = (uMsr - 0xC0000000);
1222 pbMsrBitmap += 0x400;
1223 }
1224 else
1225 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1226
1227 Assert(iBit <= 0x1fff);
1228 if (ASMBitTest(pbMsrBitmap, iBit))
1229 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1230 else
1231 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1232
1233 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1234 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1235 else
1236 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1237 return VINF_SUCCESS;
1238}
1239#endif /* VBOX_STRICT */
1240
1241
1242/**
1243 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1244 * area.
1245 *
1246 * @returns VBox status code.
1247 * @param pVCpu The cross context virtual CPU structure.
1248 * @param cMsrs The number of MSRs.
1249 */
1250DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1251{
1252 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1253 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1254 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1255 {
1256 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1257 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1258 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1259 }
1260
1261 /* Update number of guest MSRs to load/store across the world-switch. */
1262 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1263 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1264
1265 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1266 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1267 AssertRCReturn(rc, rc);
1268
1269 /* Update the VCPU's copy of the MSR count. */
1270 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1271
1272 return VINF_SUCCESS;
1273}
1274
1275
1276/**
1277 * Adds a new (or updates the value of an existing) guest/host MSR
1278 * pair to be swapped during the world-switch as part of the
1279 * auto-load/store MSR area in the VMCS.
1280 *
1281 * @returns VBox status code.
1282 * @param pVCpu The cross context virtual CPU structure.
1283 * @param uMsr The MSR.
1284 * @param uGuestMsrValue Value of the guest MSR.
1285 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1286 * necessary.
1287 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1288 * its value was updated. Optional, can be NULL.
1289 */
1290static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1291 bool *pfAddedAndUpdated)
1292{
1293 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1294 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1295 uint32_t i;
1296 for (i = 0; i < cMsrs; i++)
1297 {
1298 if (pGuestMsr->u32Msr == uMsr)
1299 break;
1300 pGuestMsr++;
1301 }
1302
1303 bool fAdded = false;
1304 if (i == cMsrs)
1305 {
1306 ++cMsrs;
1307 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1308 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1309
1310 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1311 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1312 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1313
1314 fAdded = true;
1315 }
1316
1317 /* Update the MSR values in the auto-load/store MSR area. */
1318 pGuestMsr->u32Msr = uMsr;
1319 pGuestMsr->u64Value = uGuestMsrValue;
1320
1321 /* Create/update the MSR slot in the host MSR area. */
1322 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1323 pHostMsr += i;
1324 pHostMsr->u32Msr = uMsr;
1325
1326 /*
1327 * Update the host MSR only when requested by the caller AND when we're
1328 * adding it to the auto-load/store area. Otherwise, it would have been
1329 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1330 */
1331 bool fUpdatedMsrValue = false;
1332 if ( fAdded
1333 && fUpdateHostMsr)
1334 {
1335 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1336 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1337 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1338 fUpdatedMsrValue = true;
1339 }
1340
1341 if (pfAddedAndUpdated)
1342 *pfAddedAndUpdated = fUpdatedMsrValue;
1343 return VINF_SUCCESS;
1344}
1345
1346
1347/**
1348 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1349 * auto-load/store MSR area in the VMCS.
1350 *
1351 * @returns VBox status code.
1352 * @param pVCpu The cross context virtual CPU structure.
1353 * @param uMsr The MSR.
1354 */
1355static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1356{
1357 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1358 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1359 for (uint32_t i = 0; i < cMsrs; i++)
1360 {
1361 /* Find the MSR. */
1362 if (pGuestMsr->u32Msr == uMsr)
1363 {
1364 /* If it's the last MSR, simply reduce the count. */
1365 if (i == cMsrs - 1)
1366 {
1367 --cMsrs;
1368 break;
1369 }
1370
1371 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1372 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1373 pLastGuestMsr += cMsrs - 1;
1374 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1375 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1376
1377 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1378 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1379 pLastHostMsr += cMsrs - 1;
1380 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1381 pHostMsr->u64Value = pLastHostMsr->u64Value;
1382 --cMsrs;
1383 break;
1384 }
1385 pGuestMsr++;
1386 }
1387
1388 /* Update the VMCS if the count changed (meaning the MSR was found). */
1389 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1390 {
1391 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1392 AssertRCReturn(rc, rc);
1393
1394 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1395 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1396 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1397
1398 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1399 return VINF_SUCCESS;
1400 }
1401
1402 return VERR_NOT_FOUND;
1403}
1404
1405
1406/**
1407 * Checks if the specified guest MSR is part of the auto-load/store area in
1408 * the VMCS.
1409 *
1410 * @returns true if found, false otherwise.
1411 * @param pVCpu The cross context virtual CPU structure.
1412 * @param uMsr The MSR to find.
1413 */
1414static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1415{
1416 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1417 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1418
1419 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1420 {
1421 if (pGuestMsr->u32Msr == uMsr)
1422 return true;
1423 }
1424 return false;
1425}
1426
1427
1428/**
1429 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1430 *
1431 * @param pVCpu The cross context virtual CPU structure.
1432 *
1433 * @remarks No-long-jump zone!!!
1434 */
1435static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1436{
1437 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1438 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1439 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1440 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1441
1442 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1443 {
1444 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1445
1446 /*
1447 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1448 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1449 */
1450 if (pHostMsr->u32Msr == MSR_K6_EFER)
1451 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1452 else
1453 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1454 }
1455
1456 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1457}
1458
1459
1460/**
1461 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1462 * perform lazy restoration of the host MSRs while leaving VT-x.
1463 *
1464 * @param pVCpu The cross context virtual CPU structure.
1465 *
1466 * @remarks No-long-jump zone!!!
1467 */
1468static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1469{
1470 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1471
1472 /*
1473 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1474 */
1475 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1476 {
1477 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1478#if HC_ARCH_BITS == 64
1479 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1480 {
1481 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1482 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1483 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1484 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1485 }
1486#endif
1487 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1488 }
1489}
1490
1491
1492/**
1493 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1494 * lazily while leaving VT-x.
1495 *
1496 * @returns true if it does, false otherwise.
1497 * @param pVCpu The cross context virtual CPU structure.
1498 * @param uMsr The MSR to check.
1499 */
1500static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1501{
1502 NOREF(pVCpu);
1503#if HC_ARCH_BITS == 64
1504 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1505 {
1506 switch (uMsr)
1507 {
1508 case MSR_K8_LSTAR:
1509 case MSR_K6_STAR:
1510 case MSR_K8_SF_MASK:
1511 case MSR_K8_KERNEL_GS_BASE:
1512 return true;
1513 }
1514 }
1515#else
1516 RT_NOREF(pVCpu, uMsr);
1517#endif
1518 return false;
1519}
1520
1521
1522/**
1523 * Saves a set of guest MSRs back into the guest-CPU context.
1524 *
1525 * @param pVCpu The cross context virtual CPU structure.
1526 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1527 * out-of-sync. Make sure to update the required fields
1528 * before using them.
1529 *
1530 * @remarks No-long-jump zone!!!
1531 */
1532static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1533{
1534 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1535 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1536
1537 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1538 {
1539 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1540#if HC_ARCH_BITS == 64
1541 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1542 {
1543 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1544 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1545 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1546 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1547 }
1548#else
1549 NOREF(pMixedCtx);
1550#endif
1551 }
1552}
1553
1554
1555/**
1556 * Loads a set of guests MSRs to allow read/passthru to the guest.
1557 *
1558 * The name of this function is slightly confusing. This function does NOT
1559 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1560 * common prefix for functions dealing with "lazy restoration" of the shared
1561 * MSRs.
1562 *
1563 * @param pVCpu The cross context virtual CPU structure.
1564 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1565 * out-of-sync. Make sure to update the required fields
1566 * before using them.
1567 *
1568 * @remarks No-long-jump zone!!!
1569 */
1570static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1571{
1572 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1573 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1574
1575#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1576 do { \
1577 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1578 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1579 else \
1580 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1581 } while (0)
1582
1583 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1584 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1585 {
1586#if HC_ARCH_BITS == 64
1587 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1588 {
1589 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1590 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1591 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1592 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1593 }
1594#else
1595 RT_NOREF(pMixedCtx);
1596#endif
1597 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1598 }
1599
1600#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1601}
1602
1603
1604/**
1605 * Performs lazy restoration of the set of host MSRs if they were previously
1606 * loaded with guest MSR values.
1607 *
1608 * @param pVCpu The cross context virtual CPU structure.
1609 *
1610 * @remarks No-long-jump zone!!!
1611 * @remarks The guest MSRs should have been saved back into the guest-CPU
1612 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1613 */
1614static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1615{
1616 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1617 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1618
1619 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1620 {
1621 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1622#if HC_ARCH_BITS == 64
1623 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1624 {
1625 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1626 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1627 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1628 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1629 }
1630#endif
1631 }
1632 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1633}
1634
1635
1636/**
1637 * Verifies that our cached values of the VMCS controls are all
1638 * consistent with what's actually present in the VMCS.
1639 *
1640 * @returns VBox status code.
1641 * @param pVCpu The cross context virtual CPU structure.
1642 */
1643static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1644{
1645 uint32_t u32Val;
1646 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1647 AssertRCReturn(rc, rc);
1648 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1649 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1650
1651 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1652 AssertRCReturn(rc, rc);
1653 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1654 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1655
1656 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1657 AssertRCReturn(rc, rc);
1658 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1659 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1660
1661 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1662 AssertRCReturn(rc, rc);
1663 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1664 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1665
1666 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1667 {
1668 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1669 AssertRCReturn(rc, rc);
1670 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1671 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1672 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1673 }
1674
1675 return VINF_SUCCESS;
1676}
1677
1678
1679#ifdef VBOX_STRICT
1680/**
1681 * Verifies that our cached host EFER value has not changed
1682 * since we cached it.
1683 *
1684 * @param pVCpu The cross context virtual CPU structure.
1685 */
1686static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1687{
1688 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1689
1690 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1691 {
1692 uint64_t u64Val;
1693 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1694 AssertRC(rc);
1695
1696 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1697 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1698 }
1699}
1700
1701
1702/**
1703 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1704 * VMCS are correct.
1705 *
1706 * @param pVCpu The cross context virtual CPU structure.
1707 */
1708static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1709{
1710 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1711
1712 /* Verify MSR counts in the VMCS are what we think it should be. */
1713 uint32_t cMsrs;
1714 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1715 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1716
1717 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1718 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1719
1720 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1721 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1722
1723 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1724 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1725 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1726 {
1727 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1728 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1729 pGuestMsr->u32Msr, cMsrs));
1730
1731 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1732 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1733 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1734
1735 /* Verify that the permissions are as expected in the MSR bitmap. */
1736 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1737 {
1738 VMXMSREXITREAD enmRead;
1739 VMXMSREXITWRITE enmWrite;
1740 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1741 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1742 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1743 {
1744 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1745 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1746 }
1747 else
1748 {
1749 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1750 pGuestMsr->u32Msr, cMsrs));
1751 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1752 pGuestMsr->u32Msr, cMsrs));
1753 }
1754 }
1755 }
1756}
1757#endif /* VBOX_STRICT */
1758
1759
1760/**
1761 * Flushes the TLB using EPT.
1762 *
1763 * @returns VBox status code.
1764 * @param pVCpu The cross context virtual CPU structure of the calling
1765 * EMT. Can be NULL depending on @a enmFlush.
1766 * @param enmFlush Type of flush.
1767 *
1768 * @remarks Caller is responsible for making sure this function is called only
1769 * when NestedPaging is supported and providing @a enmFlush that is
1770 * supported by the CPU.
1771 * @remarks Can be called with interrupts disabled.
1772 */
1773static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1774{
1775 uint64_t au64Descriptor[2];
1776 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1777 au64Descriptor[0] = 0;
1778 else
1779 {
1780 Assert(pVCpu);
1781 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1782 }
1783 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1784
1785 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1786 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1787 rc));
1788 if ( RT_SUCCESS(rc)
1789 && pVCpu)
1790 {
1791 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1792 }
1793}
1794
1795
1796/**
1797 * Flushes the TLB using VPID.
1798 *
1799 * @returns VBox status code.
1800 * @param pVM The cross context VM structure.
1801 * @param pVCpu The cross context virtual CPU structure of the calling
1802 * EMT. Can be NULL depending on @a enmFlush.
1803 * @param enmFlush Type of flush.
1804 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1805 * on @a enmFlush).
1806 *
1807 * @remarks Can be called with interrupts disabled.
1808 */
1809static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1810{
1811 NOREF(pVM);
1812 AssertPtr(pVM);
1813 Assert(pVM->hm.s.vmx.fVpid);
1814
1815 uint64_t au64Descriptor[2];
1816 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1817 {
1818 au64Descriptor[0] = 0;
1819 au64Descriptor[1] = 0;
1820 }
1821 else
1822 {
1823 AssertPtr(pVCpu);
1824 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1825 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1826 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1827 au64Descriptor[1] = GCPtr;
1828 }
1829
1830 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1831 AssertMsg(rc == VINF_SUCCESS,
1832 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1833 if ( RT_SUCCESS(rc)
1834 && pVCpu)
1835 {
1836 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1837 }
1838}
1839
1840
1841/**
1842 * Invalidates a guest page by guest virtual address. Only relevant for
1843 * EPT/VPID, otherwise there is nothing really to invalidate.
1844 *
1845 * @returns VBox status code.
1846 * @param pVM The cross context VM structure.
1847 * @param pVCpu The cross context virtual CPU structure.
1848 * @param GCVirt Guest virtual address of the page to invalidate.
1849 */
1850VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1851{
1852 AssertPtr(pVM);
1853 AssertPtr(pVCpu);
1854 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1855
1856 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1857 if (!fFlushPending)
1858 {
1859 /*
1860 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1861 * See @bugref{6043} and @bugref{6177}.
1862 *
1863 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1864 * function maybe called in a loop with individual addresses.
1865 */
1866 if (pVM->hm.s.vmx.fVpid)
1867 {
1868 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1869 {
1870 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1871 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1872 }
1873 else
1874 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1875 }
1876 else if (pVM->hm.s.fNestedPaging)
1877 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1878 }
1879
1880 return VINF_SUCCESS;
1881}
1882
1883
1884/**
1885 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1886 * otherwise there is nothing really to invalidate.
1887 *
1888 * @returns VBox status code.
1889 * @param pVM The cross context VM structure.
1890 * @param pVCpu The cross context virtual CPU structure.
1891 * @param GCPhys Guest physical address of the page to invalidate.
1892 */
1893VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1894{
1895 NOREF(pVM); NOREF(GCPhys);
1896 LogFlowFunc(("%RGp\n", GCPhys));
1897
1898 /*
1899 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1900 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1901 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1902 */
1903 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1904 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1905 return VINF_SUCCESS;
1906}
1907
1908
1909/**
1910 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1911 * case where neither EPT nor VPID is supported by the CPU.
1912 *
1913 * @param pVM The cross context VM structure.
1914 * @param pVCpu The cross context virtual CPU structure.
1915 * @param pCpu Pointer to the global HM struct.
1916 *
1917 * @remarks Called with interrupts disabled.
1918 */
1919static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1920{
1921 AssertPtr(pVCpu);
1922 AssertPtr(pCpu);
1923 NOREF(pVM);
1924
1925 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1926
1927 Assert(pCpu->idCpu != NIL_RTCPUID);
1928 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1929 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1930 pVCpu->hm.s.fForceTLBFlush = false;
1931 return;
1932}
1933
1934
1935/**
1936 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1937 *
1938 * @param pVM The cross context VM structure.
1939 * @param pVCpu The cross context virtual CPU structure.
1940 * @param pCpu Pointer to the global HM CPU struct.
1941 * @remarks All references to "ASID" in this function pertains to "VPID" in
1942 * Intel's nomenclature. The reason is, to avoid confusion in compare
1943 * statements since the host-CPU copies are named "ASID".
1944 *
1945 * @remarks Called with interrupts disabled.
1946 */
1947static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1948{
1949#ifdef VBOX_WITH_STATISTICS
1950 bool fTlbFlushed = false;
1951# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1952# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1953 if (!fTlbFlushed) \
1954 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1955 } while (0)
1956#else
1957# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1958# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1959#endif
1960
1961 AssertPtr(pVM);
1962 AssertPtr(pCpu);
1963 AssertPtr(pVCpu);
1964 Assert(pCpu->idCpu != NIL_RTCPUID);
1965
1966 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1967 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1968 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1969
1970 /*
1971 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1972 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1973 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1974 */
1975 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1976 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1977 {
1978 ++pCpu->uCurrentAsid;
1979 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1980 {
1981 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1982 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1983 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1984 }
1985
1986 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1987 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1988 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1989
1990 /*
1991 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1992 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1993 */
1994 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1995 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1996 HMVMX_SET_TAGGED_TLB_FLUSHED();
1997 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1998 }
1999
2000 /* Check for explicit TLB flushes. */
2001 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2002 {
2003 /*
2004 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
2005 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2006 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2007 * but not guest-physical mappings.
2008 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2009 */
2010 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2011 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2012 HMVMX_SET_TAGGED_TLB_FLUSHED();
2013 }
2014
2015 pVCpu->hm.s.fForceTLBFlush = false;
2016 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2017
2018 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2019 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2020 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2021 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2022 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2023 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2024 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2025 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2026 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2027
2028 /* Update VMCS with the VPID. */
2029 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2030 AssertRC(rc);
2031
2032#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2033}
2034
2035
2036/**
2037 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2038 *
2039 * @returns VBox status code.
2040 * @param pVM The cross context VM structure.
2041 * @param pVCpu The cross context virtual CPU structure.
2042 * @param pCpu Pointer to the global HM CPU struct.
2043 *
2044 * @remarks Called with interrupts disabled.
2045 */
2046static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2047{
2048 AssertPtr(pVM);
2049 AssertPtr(pVCpu);
2050 AssertPtr(pCpu);
2051 Assert(pCpu->idCpu != NIL_RTCPUID);
2052 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2053 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2054
2055 /*
2056 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2057 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2058 */
2059 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2060 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2061 {
2062 pVCpu->hm.s.fForceTLBFlush = true;
2063 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2064 }
2065
2066 /* Check for explicit TLB flushes. */
2067 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2068 {
2069 pVCpu->hm.s.fForceTLBFlush = true;
2070 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2071 }
2072
2073 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2074 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2075
2076 if (pVCpu->hm.s.fForceTLBFlush)
2077 {
2078 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2079 pVCpu->hm.s.fForceTLBFlush = false;
2080 }
2081}
2082
2083
2084/**
2085 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2086 *
2087 * @returns VBox status code.
2088 * @param pVM The cross context VM structure.
2089 * @param pVCpu The cross context virtual CPU structure.
2090 * @param pCpu Pointer to the global HM CPU struct.
2091 *
2092 * @remarks Called with interrupts disabled.
2093 */
2094static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2095{
2096 AssertPtr(pVM);
2097 AssertPtr(pVCpu);
2098 AssertPtr(pCpu);
2099 Assert(pCpu->idCpu != NIL_RTCPUID);
2100 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2101 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2102
2103 /*
2104 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2105 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2106 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2107 */
2108 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2109 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2110 {
2111 pVCpu->hm.s.fForceTLBFlush = true;
2112 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2113 }
2114
2115 /* Check for explicit TLB flushes. */
2116 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2117 {
2118 /*
2119 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2120 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2121 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2122 */
2123 pVCpu->hm.s.fForceTLBFlush = true;
2124 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2125 }
2126
2127 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2128 if (pVCpu->hm.s.fForceTLBFlush)
2129 {
2130 ++pCpu->uCurrentAsid;
2131 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2132 {
2133 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2134 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2135 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2136 }
2137
2138 pVCpu->hm.s.fForceTLBFlush = false;
2139 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2140 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2141 if (pCpu->fFlushAsidBeforeUse)
2142 {
2143 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2144 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2145 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2146 {
2147 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2148 pCpu->fFlushAsidBeforeUse = false;
2149 }
2150 else
2151 {
2152 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2153 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2154 }
2155 }
2156 }
2157
2158 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2159 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2160 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2161 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2162 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2163 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2164 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2165
2166 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2167 AssertRC(rc);
2168}
2169
2170
2171/**
2172 * Flushes the guest TLB entry based on CPU capabilities.
2173 *
2174 * @param pVCpu The cross context virtual CPU structure.
2175 * @param pCpu Pointer to the global HM CPU struct.
2176 */
2177DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2178{
2179#ifdef HMVMX_ALWAYS_FLUSH_TLB
2180 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2181#endif
2182 PVM pVM = pVCpu->CTX_SUFF(pVM);
2183 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2184 {
2185 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2186 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2187 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2188 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2189 default:
2190 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2191 break;
2192 }
2193
2194 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2195}
2196
2197
2198/**
2199 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2200 * TLB entries from the host TLB before VM-entry.
2201 *
2202 * @returns VBox status code.
2203 * @param pVM The cross context VM structure.
2204 */
2205static int hmR0VmxSetupTaggedTlb(PVM pVM)
2206{
2207 /*
2208 * Determine optimal flush type for Nested Paging.
2209 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2210 * guest execution (see hmR3InitFinalizeR0()).
2211 */
2212 if (pVM->hm.s.fNestedPaging)
2213 {
2214 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2215 {
2216 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2217 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2218 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2219 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2220 else
2221 {
2222 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2223 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2224 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2225 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2226 }
2227
2228 /* Make sure the write-back cacheable memory type for EPT is supported. */
2229 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2230 {
2231 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2232 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2233 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2234 }
2235
2236 /* EPT requires a page-walk length of 4. */
2237 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2238 {
2239 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2240 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2241 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2242 }
2243 }
2244 else
2245 {
2246 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2247 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2248 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2249 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2250 }
2251 }
2252
2253 /*
2254 * Determine optimal flush type for VPID.
2255 */
2256 if (pVM->hm.s.vmx.fVpid)
2257 {
2258 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2259 {
2260 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2261 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2262 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2263 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2264 else
2265 {
2266 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2267 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2268 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2269 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2270 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2271 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2272 pVM->hm.s.vmx.fVpid = false;
2273 }
2274 }
2275 else
2276 {
2277 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2278 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2279 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2280 pVM->hm.s.vmx.fVpid = false;
2281 }
2282 }
2283
2284 /*
2285 * Setup the handler for flushing tagged-TLBs.
2286 */
2287 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2288 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2289 else if (pVM->hm.s.fNestedPaging)
2290 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2291 else if (pVM->hm.s.vmx.fVpid)
2292 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2293 else
2294 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2295 return VINF_SUCCESS;
2296}
2297
2298
2299/**
2300 * Sets up pin-based VM-execution controls in the VMCS.
2301 *
2302 * @returns VBox status code.
2303 * @param pVM The cross context VM structure.
2304 * @param pVCpu The cross context virtual CPU structure.
2305 */
2306static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2307{
2308 AssertPtr(pVM);
2309 AssertPtr(pVCpu);
2310
2311 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2312 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2313
2314 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2315 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2316
2317 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2318 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2319
2320 /* Enable the VMX preemption timer. */
2321 if (pVM->hm.s.vmx.fUsePreemptTimer)
2322 {
2323 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2324 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2325 }
2326
2327#if 0
2328 /* Enable posted-interrupt processing. */
2329 if (pVM->hm.s.fPostedIntrs)
2330 {
2331 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2332 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2333 val |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2334 }
2335#endif
2336
2337 if ((val & zap) != val)
2338 {
2339 LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2340 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2341 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2342 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2343 }
2344
2345 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2346 AssertRCReturn(rc, rc);
2347
2348 pVCpu->hm.s.vmx.u32PinCtls = val;
2349 return rc;
2350}
2351
2352
2353/**
2354 * Sets up processor-based VM-execution controls in the VMCS.
2355 *
2356 * @returns VBox status code.
2357 * @param pVM The cross context VM structure.
2358 * @param pVCpu The cross context virtual CPU structure.
2359 */
2360static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2361{
2362 AssertPtr(pVM);
2363 AssertPtr(pVCpu);
2364
2365 int rc = VERR_INTERNAL_ERROR_5;
2366 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2367 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2368
2369 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2370 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2371 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2372 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2373 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2374 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2375 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2376
2377 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2378 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2379 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2380 {
2381 LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2382 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2383 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2384 }
2385
2386 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2387 if (!pVM->hm.s.fNestedPaging)
2388 {
2389 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2390 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2391 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2392 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2393 }
2394
2395 /* Use TPR shadowing if supported by the CPU. */
2396 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2397 {
2398 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2399 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2400 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2401 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2402 AssertRCReturn(rc, rc);
2403
2404 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2405 /* CR8 writes cause a VM-exit based on TPR threshold. */
2406 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2407 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2408 }
2409 else
2410 {
2411 /*
2412 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2413 * Set this control only for 64-bit guests.
2414 */
2415 if (pVM->hm.s.fAllow64BitGuests)
2416 {
2417 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2418 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2419 }
2420 }
2421
2422 /* Use MSR-bitmaps if supported by the CPU. */
2423 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2424 {
2425 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2426
2427 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2428 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2429 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2430 AssertRCReturn(rc, rc);
2431
2432 /*
2433 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2434 * automatically using dedicated fields in the VMCS.
2435 */
2436 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2437 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2438 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2439 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2440 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2441
2442#if HC_ARCH_BITS == 64
2443 /*
2444 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2445 */
2446 if (pVM->hm.s.fAllow64BitGuests)
2447 {
2448 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2449 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2450 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2451 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2452 }
2453#endif
2454 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2455 }
2456
2457 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2458 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2459 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2460
2461 if ((val & zap) != val)
2462 {
2463 LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2464 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2465 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2466 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2467 }
2468
2469 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2470 AssertRCReturn(rc, rc);
2471
2472 pVCpu->hm.s.vmx.u32ProcCtls = val;
2473
2474 /*
2475 * Secondary processor-based VM-execution controls.
2476 */
2477 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2478 {
2479 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2480 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2481
2482 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2483 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2484
2485 if (pVM->hm.s.fNestedPaging)
2486 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2487 else
2488 {
2489 /*
2490 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2491 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2492 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2493 */
2494 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2495 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2496 }
2497
2498 if (pVM->hm.s.vmx.fVpid)
2499 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2500
2501 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2502 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2503
2504#if 0
2505 if (pVM->hm.s.fVirtApicRegs)
2506 {
2507 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2508 val |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2509
2510 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2511 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2512 }
2513#endif
2514
2515 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2516 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2517 * done dynamically. */
2518 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2519 {
2520 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2521 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2522 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2523 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2524 AssertRCReturn(rc, rc);
2525 }
2526
2527 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2528 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2529
2530 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2531 && pVM->hm.s.vmx.cPleGapTicks
2532 && pVM->hm.s.vmx.cPleWindowTicks)
2533 {
2534 val |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2535
2536 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2537 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2538 AssertRCReturn(rc, rc);
2539 }
2540
2541 if ((val & zap) != val)
2542 {
2543 LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2544 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2545 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2546 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2547 }
2548
2549 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2550 AssertRCReturn(rc, rc);
2551
2552 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2553 }
2554 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2555 {
2556 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2557 "available\n"));
2558 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2559 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2560 }
2561
2562 return VINF_SUCCESS;
2563}
2564
2565
2566/**
2567 * Sets up miscellaneous (everything other than Pin & Processor-based
2568 * VM-execution) control fields in the VMCS.
2569 *
2570 * @returns VBox status code.
2571 * @param pVM The cross context VM structure.
2572 * @param pVCpu The cross context virtual CPU structure.
2573 */
2574static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2575{
2576 NOREF(pVM);
2577 AssertPtr(pVM);
2578 AssertPtr(pVCpu);
2579
2580 int rc = VERR_GENERAL_FAILURE;
2581
2582 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2583#if 0
2584 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2585 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2586 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2587
2588 /*
2589 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2590 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2591 * We thus use the exception bitmap to control it rather than use both.
2592 */
2593 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2594 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2595
2596 /** @todo Explore possibility of using IO-bitmaps. */
2597 /* All IO & IOIO instructions cause VM-exits. */
2598 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2599 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2600
2601 /* Initialize the MSR-bitmap area. */
2602 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2603 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2604 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2605 AssertRCReturn(rc, rc);
2606#endif
2607
2608 /* Setup MSR auto-load/store area. */
2609 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2610 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2611 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2612 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2613 AssertRCReturn(rc, rc);
2614
2615 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2616 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2617 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2618 AssertRCReturn(rc, rc);
2619
2620 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2621 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2622 AssertRCReturn(rc, rc);
2623
2624 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2625#if 0
2626 /* Setup debug controls */
2627 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2628 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2629 AssertRCReturn(rc, rc);
2630#endif
2631
2632 return rc;
2633}
2634
2635
2636/**
2637 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2638 *
2639 * We shall setup those exception intercepts that don't change during the
2640 * lifetime of the VM here. The rest are done dynamically while loading the
2641 * guest state.
2642 *
2643 * @returns VBox status code.
2644 * @param pVM The cross context VM structure.
2645 * @param pVCpu The cross context virtual CPU structure.
2646 */
2647static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2648{
2649 AssertPtr(pVM);
2650 AssertPtr(pVCpu);
2651
2652 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2653
2654 uint32_t u32XcptBitmap = 0;
2655
2656 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2657 u32XcptBitmap |= RT_BIT_32(X86_XCPT_AC);
2658
2659 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2660 and writes, and because recursive #DBs can cause the CPU hang, we must always
2661 intercept #DB. */
2662 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2663
2664 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2665 if (!pVM->hm.s.fNestedPaging)
2666 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2667
2668 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2669 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2670 AssertRCReturn(rc, rc);
2671 return rc;
2672}
2673
2674
2675/**
2676 * Sets up the initial guest-state mask. The guest-state mask is consulted
2677 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2678 * for the nested virtualization case (as it would cause a VM-exit).
2679 *
2680 * @param pVCpu The cross context virtual CPU structure.
2681 */
2682static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2683{
2684 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2685 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2686 return VINF_SUCCESS;
2687}
2688
2689
2690/**
2691 * Does per-VM VT-x initialization.
2692 *
2693 * @returns VBox status code.
2694 * @param pVM The cross context VM structure.
2695 */
2696VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2697{
2698 LogFlowFunc(("pVM=%p\n", pVM));
2699
2700 int rc = hmR0VmxStructsAlloc(pVM);
2701 if (RT_FAILURE(rc))
2702 {
2703 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2704 return rc;
2705 }
2706
2707 return VINF_SUCCESS;
2708}
2709
2710
2711/**
2712 * Does per-VM VT-x termination.
2713 *
2714 * @returns VBox status code.
2715 * @param pVM The cross context VM structure.
2716 */
2717VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2718{
2719 LogFlowFunc(("pVM=%p\n", pVM));
2720
2721#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2722 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2723 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2724#endif
2725 hmR0VmxStructsFree(pVM);
2726 return VINF_SUCCESS;
2727}
2728
2729
2730/**
2731 * Sets up the VM for execution under VT-x.
2732 * This function is only called once per-VM during initialization.
2733 *
2734 * @returns VBox status code.
2735 * @param pVM The cross context VM structure.
2736 */
2737VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2738{
2739 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2740 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2741
2742 LogFlowFunc(("pVM=%p\n", pVM));
2743
2744 /*
2745 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2746 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2747 */
2748 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2749 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2750 || !pVM->hm.s.vmx.pRealModeTSS))
2751 {
2752 LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2753 return VERR_INTERNAL_ERROR;
2754 }
2755
2756 /* Initialize these always, see hmR3InitFinalizeR0().*/
2757 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2758 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2759
2760 /* Setup the tagged-TLB flush handlers. */
2761 int rc = hmR0VmxSetupTaggedTlb(pVM);
2762 if (RT_FAILURE(rc))
2763 {
2764 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2765 return rc;
2766 }
2767
2768 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2769 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2770#if HC_ARCH_BITS == 64
2771 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2772 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2773 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2774 {
2775 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2776 }
2777#endif
2778
2779 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2780 RTCCUINTREG uHostCR4 = ASMGetCR4();
2781 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2782 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2783
2784 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2785 {
2786 PVMCPU pVCpu = &pVM->aCpus[i];
2787 AssertPtr(pVCpu);
2788 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2789
2790 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2791 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2792
2793 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2794 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2795 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2796
2797 /* Set revision dword at the beginning of the VMCS structure. */
2798 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2799
2800 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2801 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2802 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2803 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2804
2805 /* Load this VMCS as the current VMCS. */
2806 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2807 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2808 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2809
2810 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2811 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2812 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2813
2814 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2815 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2816 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2817
2818 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2819 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2820 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2821
2822 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2823 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2824 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2825
2826 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2827 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2828 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2829
2830#if HC_ARCH_BITS == 32
2831 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2832 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2833 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2834#endif
2835
2836 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2837 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2838 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2839 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2840
2841 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2842
2843 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2844 }
2845
2846 return VINF_SUCCESS;
2847}
2848
2849
2850/**
2851 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2852 * the VMCS.
2853 *
2854 * @returns VBox status code.
2855 * @param pVM The cross context VM structure.
2856 * @param pVCpu The cross context virtual CPU structure.
2857 */
2858DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2859{
2860 NOREF(pVM); NOREF(pVCpu);
2861
2862 RTCCUINTREG uReg = ASMGetCR0();
2863 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2864 AssertRCReturn(rc, rc);
2865
2866 uReg = ASMGetCR3();
2867 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2868 AssertRCReturn(rc, rc);
2869
2870 uReg = ASMGetCR4();
2871 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2872 AssertRCReturn(rc, rc);
2873 return rc;
2874}
2875
2876
2877#if HC_ARCH_BITS == 64
2878/**
2879 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2880 * requirements. See hmR0VmxSaveHostSegmentRegs().
2881 */
2882# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2883 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2884 { \
2885 bool fValidSelector = true; \
2886 if ((selValue) & X86_SEL_LDT) \
2887 { \
2888 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2889 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2890 } \
2891 if (fValidSelector) \
2892 { \
2893 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2894 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2895 } \
2896 (selValue) = 0; \
2897 }
2898#endif
2899
2900
2901/**
2902 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2903 * the host-state area in the VMCS.
2904 *
2905 * @returns VBox status code.
2906 * @param pVM The cross context VM structure.
2907 * @param pVCpu The cross context virtual CPU structure.
2908 */
2909DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2910{
2911 int rc = VERR_INTERNAL_ERROR_5;
2912
2913#if HC_ARCH_BITS == 64
2914 /*
2915 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2916 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2917 *
2918 * This apparently can happen (most likely the FPU changes), deal with it rather than asserting.
2919 * Was observed booting Solaris10u10 32-bit guest.
2920 */
2921 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2922 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2923 {
2924 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2925 pVCpu->idCpu));
2926 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2927 }
2928 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2929#else
2930 RT_NOREF(pVCpu);
2931#endif
2932
2933 /*
2934 * Host DS, ES, FS and GS segment registers.
2935 */
2936#if HC_ARCH_BITS == 64
2937 RTSEL uSelDS = ASMGetDS();
2938 RTSEL uSelES = ASMGetES();
2939 RTSEL uSelFS = ASMGetFS();
2940 RTSEL uSelGS = ASMGetGS();
2941#else
2942 RTSEL uSelDS = 0;
2943 RTSEL uSelES = 0;
2944 RTSEL uSelFS = 0;
2945 RTSEL uSelGS = 0;
2946#endif
2947
2948 /*
2949 * Host CS and SS segment registers.
2950 */
2951 RTSEL uSelCS = ASMGetCS();
2952 RTSEL uSelSS = ASMGetSS();
2953
2954 /*
2955 * Host TR segment register.
2956 */
2957 RTSEL uSelTR = ASMGetTR();
2958
2959#if HC_ARCH_BITS == 64
2960 /*
2961 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2962 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2963 */
2964 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2965 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2966 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2967 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2968# undef VMXLOCAL_ADJUST_HOST_SEG
2969#endif
2970
2971 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2972 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2973 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2974 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2975 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2976 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2977 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2978 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2979 Assert(uSelCS);
2980 Assert(uSelTR);
2981
2982 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2983#if 0
2984 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2985 Assert(uSelSS != 0);
2986#endif
2987
2988 /* Write these host selector fields into the host-state area in the VMCS. */
2989 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2990 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2991#if HC_ARCH_BITS == 64
2992 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2993 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2994 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2995 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2996#else
2997 NOREF(uSelDS);
2998 NOREF(uSelES);
2999 NOREF(uSelFS);
3000 NOREF(uSelGS);
3001#endif
3002 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3003 AssertRCReturn(rc, rc);
3004
3005 /*
3006 * Host GDTR and IDTR.
3007 */
3008 RTGDTR Gdtr;
3009 RTIDTR Idtr;
3010 RT_ZERO(Gdtr);
3011 RT_ZERO(Idtr);
3012 ASMGetGDTR(&Gdtr);
3013 ASMGetIDTR(&Idtr);
3014 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3015 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3016 AssertRCReturn(rc, rc);
3017
3018#if HC_ARCH_BITS == 64
3019 /*
3020 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3021 * maximum limit (0xffff) on every VM-exit.
3022 */
3023 if (Gdtr.cbGdt != 0xffff)
3024 {
3025 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3026 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3027 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3028 }
3029
3030 /*
3031 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3032 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3033 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3034 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3035 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3036 * hosts where we are pretty sure it won't cause trouble.
3037 */
3038# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3039 if (Idtr.cbIdt < 0x0fff)
3040# else
3041 if (Idtr.cbIdt != 0xffff)
3042# endif
3043 {
3044 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3045 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3046 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3047 }
3048#endif
3049
3050 /*
3051 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3052 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3053 */
3054 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3055 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3056 VERR_VMX_INVALID_HOST_STATE);
3057
3058 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3059#if HC_ARCH_BITS == 64
3060 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3061
3062 /*
3063 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3064 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3065 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3066 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3067 *
3068 * [1] See Intel spec. 3.5 "System Descriptor Types".
3069 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3070 */
3071 Assert(pDesc->System.u4Type == 11);
3072 if ( pDesc->System.u16LimitLow != 0x67
3073 || pDesc->System.u4LimitHigh)
3074 {
3075 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3076 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3077 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3078 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3079 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3080
3081 /* Store the GDTR here as we need it while restoring TR. */
3082 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3083 }
3084#else
3085 NOREF(pVM);
3086 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3087#endif
3088 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3089 AssertRCReturn(rc, rc);
3090
3091 /*
3092 * Host FS base and GS base.
3093 */
3094#if HC_ARCH_BITS == 64
3095 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3096 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3097 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3098 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3099 AssertRCReturn(rc, rc);
3100
3101 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3102 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3103 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3104 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3105 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3106#endif
3107 return rc;
3108}
3109
3110
3111/**
3112 * Saves certain host MSRs in the VM-exit MSR-load area and some in the
3113 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3114 * the host after every successful VM-exit.
3115 *
3116 * @returns VBox status code.
3117 * @param pVM The cross context VM structure.
3118 * @param pVCpu The cross context virtual CPU structure.
3119 *
3120 * @remarks No-long-jump zone!!!
3121 */
3122DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3123{
3124 NOREF(pVM);
3125
3126 AssertPtr(pVCpu);
3127 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3128
3129 /*
3130 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3131 * rather than swapping them on every VM-entry.
3132 */
3133 hmR0VmxLazySaveHostMsrs(pVCpu);
3134
3135 /*
3136 * Host Sysenter MSRs.
3137 */
3138 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3139#if HC_ARCH_BITS == 32
3140 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3141 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3142#else
3143 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3144 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3145#endif
3146 AssertRCReturn(rc, rc);
3147
3148 /*
3149 * Host EFER MSR.
3150 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3151 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3152 */
3153 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3154 {
3155 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3156 AssertRCReturn(rc, rc);
3157 }
3158
3159 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3160 * hmR0VmxLoadGuestExitCtls() !! */
3161
3162 return rc;
3163}
3164
3165
3166/**
3167 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3168 *
3169 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3170 * these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3171 * hmR0VMxLoadGuestEntryCtls().
3172 *
3173 * @returns true if we need to load guest EFER, false otherwise.
3174 * @param pVCpu The cross context virtual CPU structure.
3175 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3176 * out-of-sync. Make sure to update the required fields
3177 * before using them.
3178 *
3179 * @remarks Requires EFER, CR4.
3180 * @remarks No-long-jump zone!!!
3181 */
3182static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3183{
3184#ifdef HMVMX_ALWAYS_SWAP_EFER
3185 return true;
3186#endif
3187
3188#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3189 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3190 if (CPUMIsGuestInLongMode(pVCpu))
3191 return false;
3192#endif
3193
3194 PVM pVM = pVCpu->CTX_SUFF(pVM);
3195 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3196 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3197
3198 /*
3199 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3200 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3201 */
3202 if ( CPUMIsGuestInLongMode(pVCpu)
3203 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3204 {
3205 return true;
3206 }
3207
3208 /*
3209 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3210 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3211 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3212 */
3213 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3214 && (pMixedCtx->cr0 & X86_CR0_PG)
3215 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3216 {
3217 /* Assert that host is PAE capable. */
3218 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3219 return true;
3220 }
3221
3222 /** @todo Check the latest Intel spec. for any other bits,
3223 * like SMEP/SMAP? */
3224 return false;
3225}
3226
3227
3228/**
3229 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3230 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3231 * controls".
3232 *
3233 * @returns VBox status code.
3234 * @param pVCpu The cross context virtual CPU structure.
3235 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3236 * out-of-sync. Make sure to update the required fields
3237 * before using them.
3238 *
3239 * @remarks Requires EFER.
3240 * @remarks No-long-jump zone!!!
3241 */
3242DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3243{
3244 int rc = VINF_SUCCESS;
3245 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3246 {
3247 PVM pVM = pVCpu->CTX_SUFF(pVM);
3248 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3249 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3250
3251 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3252 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3253
3254 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3255 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3256 {
3257 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3258 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3259 }
3260 else
3261 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3262
3263 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3264 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3265 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3266 {
3267 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3268 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3269 }
3270
3271 /*
3272 * The following should -not- be set (since we're not in SMM mode):
3273 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3274 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3275 */
3276
3277 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3278 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3279
3280 if ((val & zap) != val)
3281 {
3282 LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3283 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3284 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3285 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3286 }
3287
3288 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3289 AssertRCReturn(rc, rc);
3290
3291 pVCpu->hm.s.vmx.u32EntryCtls = val;
3292 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3293 }
3294 return rc;
3295}
3296
3297
3298/**
3299 * Sets up the VM-exit controls in the VMCS.
3300 *
3301 * @returns VBox status code.
3302 * @param pVCpu The cross context virtual CPU structure.
3303 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3304 * out-of-sync. Make sure to update the required fields
3305 * before using them.
3306 *
3307 * @remarks Requires EFER.
3308 */
3309DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3310{
3311 NOREF(pMixedCtx);
3312
3313 int rc = VINF_SUCCESS;
3314 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3315 {
3316 PVM pVM = pVCpu->CTX_SUFF(pVM);
3317 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3318 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3319
3320 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3321 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3322
3323 /*
3324 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3325 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3326 */
3327#if HC_ARCH_BITS == 64
3328 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3329 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3330#else
3331 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3332 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3333 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3334 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3335 {
3336 /* The switcher returns to long mode, EFER is managed by the switcher. */
3337 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3338 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3339 }
3340 else
3341 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3342#endif
3343
3344 /* If the newer VMCS fields for managing EFER exists, use it. */
3345 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3346 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3347 {
3348 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3349 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3350 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3351 }
3352
3353 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3354 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3355
3356 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3357 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3358 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3359
3360 if ( pVM->hm.s.vmx.fUsePreemptTimer
3361 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3362 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3363
3364 if ((val & zap) != val)
3365 {
3366 LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3367 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3368 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3369 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3370 }
3371
3372 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3373 AssertRCReturn(rc, rc);
3374
3375 pVCpu->hm.s.vmx.u32ExitCtls = val;
3376 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3377 }
3378 return rc;
3379}
3380
3381
3382/**
3383 * Sets the TPR threshold in the VMCS.
3384 *
3385 * @returns VBox status code.
3386 * @param pVCpu The cross context virtual CPU structure.
3387 * @param u32TprThreshold The TPR threshold (task-priority class only).
3388 */
3389DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3390{
3391 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3392 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3393 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3394}
3395
3396
3397/**
3398 * Loads the guest APIC and related state.
3399 *
3400 * @returns VBox status code.
3401 * @param pVCpu The cross context virtual CPU structure.
3402 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3403 * out-of-sync. Make sure to update the required fields
3404 * before using them.
3405 *
3406 * @remarks No-long-jump zone!!!
3407 */
3408DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3409{
3410 NOREF(pMixedCtx);
3411
3412 int rc = VINF_SUCCESS;
3413 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3414 {
3415 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3416 && APICIsEnabled(pVCpu))
3417 {
3418 /*
3419 * Setup TPR shadowing.
3420 */
3421 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3422 {
3423 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3424
3425 bool fPendingIntr = false;
3426 uint8_t u8Tpr = 0;
3427 uint8_t u8PendingIntr = 0;
3428 rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3429 AssertRCReturn(rc, rc);
3430
3431 /*
3432 * If there are interrupts pending but masked by the TPR, instruct VT-x to cause a TPR-below-threshold VM-exit
3433 * when the guest lowers its TPR below the priority of the pending interrupt so we can deliver the interrupt.
3434 * If there are no interrupts pending, set threshold to 0 to not cause any TPR-below-threshold VM-exits.
3435 */
3436 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3437 uint32_t u32TprThreshold = 0;
3438 if (fPendingIntr)
3439 {
3440 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3441 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3442 const uint8_t u8TprPriority = u8Tpr >> 4;
3443 if (u8PendingPriority <= u8TprPriority)
3444 u32TprThreshold = u8PendingPriority;
3445 }
3446
3447 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3448 AssertRCReturn(rc, rc);
3449 }
3450 }
3451 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3452 }
3453
3454 return rc;
3455}
3456
3457
3458/**
3459 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3460 *
3461 * @returns Guest's interruptibility-state.
3462 * @param pVCpu The cross context virtual CPU structure.
3463 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3464 * out-of-sync. Make sure to update the required fields
3465 * before using them.
3466 *
3467 * @remarks No-long-jump zone!!!
3468 */
3469DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3470{
3471 /*
3472 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3473 */
3474 uint32_t uIntrState = 0;
3475 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3476 {
3477 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3478 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3479 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3480 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3481 {
3482 if (pMixedCtx->eflags.Bits.u1IF)
3483 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3484 else
3485 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3486 }
3487 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3488 {
3489 /*
3490 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3491 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3492 */
3493 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3494 }
3495 }
3496
3497 /*
3498 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3499 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3500 * setting this would block host-NMIs and IRET will not clear the blocking.
3501 *
3502 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3503 */
3504 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3505 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3506 {
3507 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3508 }
3509
3510 return uIntrState;
3511}
3512
3513
3514/**
3515 * Loads the guest's interruptibility-state into the guest-state area in the
3516 * VMCS.
3517 *
3518 * @returns VBox status code.
3519 * @param pVCpu The cross context virtual CPU structure.
3520 * @param uIntrState The interruptibility-state to set.
3521 */
3522static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3523{
3524 NOREF(pVCpu);
3525 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3526 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3527 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3528 AssertRC(rc);
3529 return rc;
3530}
3531
3532
3533/**
3534 * Loads the exception intercepts required for guest execution in the VMCS.
3535 *
3536 * @returns VBox status code.
3537 * @param pVCpu The cross context virtual CPU structure.
3538 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3539 * out-of-sync. Make sure to update the required fields
3540 * before using them.
3541 */
3542static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3543{
3544 NOREF(pMixedCtx);
3545 int rc = VINF_SUCCESS;
3546 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3547 {
3548 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3549 if (pVCpu->hm.s.fGIMTrapXcptUD)
3550 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3551#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3552 else
3553 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3554#endif
3555
3556 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3557 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3558
3559 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3560 AssertRCReturn(rc, rc);
3561
3562 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3563 Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3564 pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3565 }
3566 return rc;
3567}
3568
3569
3570/**
3571 * Loads the guest's RIP into the guest-state area in the VMCS.
3572 *
3573 * @returns VBox status code.
3574 * @param pVCpu The cross context virtual CPU structure.
3575 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3576 * out-of-sync. Make sure to update the required fields
3577 * before using them.
3578 *
3579 * @remarks No-long-jump zone!!!
3580 */
3581static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3582{
3583 int rc = VINF_SUCCESS;
3584 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3585 {
3586 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3587 AssertRCReturn(rc, rc);
3588
3589 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3590 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3591 HMCPU_CF_VALUE(pVCpu)));
3592 }
3593 return rc;
3594}
3595
3596
3597/**
3598 * Loads the guest's RSP into the guest-state area in the VMCS.
3599 *
3600 * @returns VBox status code.
3601 * @param pVCpu The cross context virtual CPU structure.
3602 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3603 * out-of-sync. Make sure to update the required fields
3604 * before using them.
3605 *
3606 * @remarks No-long-jump zone!!!
3607 */
3608static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3609{
3610 int rc = VINF_SUCCESS;
3611 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3612 {
3613 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3614 AssertRCReturn(rc, rc);
3615
3616 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3617 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3618 }
3619 return rc;
3620}
3621
3622
3623/**
3624 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3625 *
3626 * @returns VBox status code.
3627 * @param pVCpu The cross context virtual CPU structure.
3628 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3629 * out-of-sync. Make sure to update the required fields
3630 * before using them.
3631 *
3632 * @remarks No-long-jump zone!!!
3633 */
3634static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3635{
3636 int rc = VINF_SUCCESS;
3637 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3638 {
3639 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3640 Let us assert it as such and use 32-bit VMWRITE. */
3641 Assert(!(pMixedCtx->rflags.u64 >> 32));
3642 X86EFLAGS Eflags = pMixedCtx->eflags;
3643 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3644 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3645 * These will never be cleared/set, unless some other part of the VMM
3646 * code is buggy - in which case we're better of finding and fixing
3647 * those bugs than hiding them. */
3648 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3649 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3650 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3651 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3652
3653 /*
3654 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3655 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3656 */
3657 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3658 {
3659 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3660 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3661 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3662 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3663 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3664 }
3665
3666 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3667 AssertRCReturn(rc, rc);
3668
3669 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3670 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3671 }
3672 return rc;
3673}
3674
3675
3676/**
3677 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3678 *
3679 * @returns VBox status code.
3680 * @param pVCpu The cross context virtual CPU structure.
3681 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3682 * out-of-sync. Make sure to update the required fields
3683 * before using them.
3684 *
3685 * @remarks No-long-jump zone!!!
3686 */
3687DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3688{
3689 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3690 rc |= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3691 rc |= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3692 AssertRCReturn(rc, rc);
3693 return rc;
3694}
3695
3696
3697/**
3698 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3699 * CR0 is partially shared with the host and we have to consider the FPU bits.
3700 *
3701 * @returns VBox status code.
3702 * @param pVCpu The cross context virtual CPU structure.
3703 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3704 * out-of-sync. Make sure to update the required fields
3705 * before using them.
3706 *
3707 * @remarks No-long-jump zone!!!
3708 */
3709static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3710{
3711 /*
3712 * Guest CR0.
3713 * Guest FPU.
3714 */
3715 int rc = VINF_SUCCESS;
3716 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3717 {
3718 Assert(!(pMixedCtx->cr0 >> 32));
3719 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3720 PVM pVM = pVCpu->CTX_SUFF(pVM);
3721
3722 /* The guest's view (read access) of its CR0 is unblemished. */
3723 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3724 AssertRCReturn(rc, rc);
3725 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3726
3727 /* Setup VT-x's view of the guest CR0. */
3728 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3729 if (pVM->hm.s.fNestedPaging)
3730 {
3731 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3732 {
3733 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3734 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3735 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3736 }
3737 else
3738 {
3739 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3740 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3741 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3742 }
3743
3744 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3745 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3746 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3747
3748 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3749 AssertRCReturn(rc, rc);
3750 }
3751 else
3752 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3753
3754 /*
3755 * Guest FPU bits.
3756 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3757 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3758 */
3759 u32GuestCR0 |= X86_CR0_NE;
3760 bool fInterceptNM = false;
3761 if (CPUMIsGuestFPUStateActive(pVCpu))
3762 {
3763 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3764 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3765 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3766 }
3767 else
3768 {
3769 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3770 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3771 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3772 }
3773
3774 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3775 bool fInterceptMF = false;
3776 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3777 fInterceptMF = true;
3778
3779 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3780 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3781 {
3782 Assert(PDMVmmDevHeapIsEnabled(pVM));
3783 Assert(pVM->hm.s.vmx.pRealModeTSS);
3784 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3785 fInterceptNM = true;
3786 fInterceptMF = true;
3787 }
3788 else
3789 {
3790 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3791 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3792 }
3793 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3794
3795 if (fInterceptNM)
3796 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3797 else
3798 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3799
3800 if (fInterceptMF)
3801 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3802 else
3803 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3804
3805 /* Additional intercepts for debugging, define these yourself explicitly. */
3806#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3807 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3808 | RT_BIT(X86_XCPT_BP)
3809 | RT_BIT(X86_XCPT_DE)
3810 | RT_BIT(X86_XCPT_NM)
3811 | RT_BIT(X86_XCPT_TS)
3812 | RT_BIT(X86_XCPT_UD)
3813 | RT_BIT(X86_XCPT_NP)
3814 | RT_BIT(X86_XCPT_SS)
3815 | RT_BIT(X86_XCPT_GP)
3816 | RT_BIT(X86_XCPT_PF)
3817 | RT_BIT(X86_XCPT_MF)
3818 ;
3819#elif defined(HMVMX_ALWAYS_TRAP_PF)
3820 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3821#endif
3822
3823 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3824
3825 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3826 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3827 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3828 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3829 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3830 else
3831 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3832
3833 u32GuestCR0 |= uSetCR0;
3834 u32GuestCR0 &= uZapCR0;
3835 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3836
3837 /* Write VT-x's view of the guest CR0 into the VMCS. */
3838 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3839 AssertRCReturn(rc, rc);
3840 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3841 uZapCR0));
3842
3843 /*
3844 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3845 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3846 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3847 */
3848 uint32_t u32CR0Mask = 0;
3849 u32CR0Mask = X86_CR0_PE
3850 | X86_CR0_NE
3851 | X86_CR0_WP
3852 | X86_CR0_PG
3853 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3854 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3855 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3856
3857 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3858 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3859 * and @bugref{6944}. */
3860#if 0
3861 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3862 u32CR0Mask &= ~X86_CR0_PE;
3863#endif
3864 if (pVM->hm.s.fNestedPaging)
3865 u32CR0Mask &= ~X86_CR0_WP;
3866
3867 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3868 if (fInterceptNM)
3869 {
3870 u32CR0Mask |= X86_CR0_TS
3871 | X86_CR0_MP;
3872 }
3873
3874 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3875 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3876 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3877 AssertRCReturn(rc, rc);
3878 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3879
3880 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3881 }
3882 return rc;
3883}
3884
3885
3886/**
3887 * Loads the guest control registers (CR3, CR4) into the guest-state area
3888 * in the VMCS.
3889 *
3890 * @returns VBox strict status code.
3891 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3892 * without unrestricted guest access and the VMMDev is not presently
3893 * mapped (e.g. EFI32).
3894 *
3895 * @param pVCpu The cross context virtual CPU structure.
3896 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3897 * out-of-sync. Make sure to update the required fields
3898 * before using them.
3899 *
3900 * @remarks No-long-jump zone!!!
3901 */
3902static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3903{
3904 int rc = VINF_SUCCESS;
3905 PVM pVM = pVCpu->CTX_SUFF(pVM);
3906
3907 /*
3908 * Guest CR2.
3909 * It's always loaded in the assembler code. Nothing to do here.
3910 */
3911
3912 /*
3913 * Guest CR3.
3914 */
3915 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3916 {
3917 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3918 if (pVM->hm.s.fNestedPaging)
3919 {
3920 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3921
3922 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3923 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3924 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3925 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3926
3927 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3928 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3929 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3930
3931 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3932 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3933 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3934 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3935 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3936 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3937 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3938
3939 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3940 AssertRCReturn(rc, rc);
3941 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3942
3943 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3944 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3945 {
3946 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3947 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3948 {
3949 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3950 AssertRCReturn(rc, rc);
3951 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3952 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3953 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3954 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3955 AssertRCReturn(rc, rc);
3956 }
3957
3958 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3959 have Unrestricted Execution to handle the guest when it's not using paging. */
3960 GCPhysGuestCR3 = pMixedCtx->cr3;
3961 }
3962 else
3963 {
3964 /*
3965 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3966 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3967 * EPT takes care of translating it to host-physical addresses.
3968 */
3969 RTGCPHYS GCPhys;
3970 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3971
3972 /* We obtain it here every time as the guest could have relocated this PCI region. */
3973 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3974 if (RT_SUCCESS(rc))
3975 { /* likely */ }
3976 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3977 {
3978 Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
3979 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3980 }
3981 else
3982 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3983
3984 GCPhysGuestCR3 = GCPhys;
3985 }
3986
3987 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3988 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3989 }
3990 else
3991 {
3992 /* Non-nested paging case, just use the hypervisor's CR3. */
3993 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3994
3995 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
3996 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3997 }
3998 AssertRCReturn(rc, rc);
3999
4000 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
4001 }
4002
4003 /*
4004 * Guest CR4.
4005 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
4006 */
4007 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
4008 {
4009 Assert(!(pMixedCtx->cr4 >> 32));
4010 uint32_t u32GuestCR4 = pMixedCtx->cr4;
4011
4012 /* The guest's view of its CR4 is unblemished. */
4013 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4014 AssertRCReturn(rc, rc);
4015 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4016
4017 /* Setup VT-x's view of the guest CR4. */
4018 /*
4019 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4020 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4021 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4022 */
4023 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4024 {
4025 Assert(pVM->hm.s.vmx.pRealModeTSS);
4026 Assert(PDMVmmDevHeapIsEnabled(pVM));
4027 u32GuestCR4 &= ~X86_CR4_VME;
4028 }
4029
4030 if (pVM->hm.s.fNestedPaging)
4031 {
4032 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4033 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4034 {
4035 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4036 u32GuestCR4 |= X86_CR4_PSE;
4037 /* Our identity mapping is a 32-bit page directory. */
4038 u32GuestCR4 &= ~X86_CR4_PAE;
4039 }
4040 /* else use guest CR4.*/
4041 }
4042 else
4043 {
4044 /*
4045 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4046 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4047 */
4048 switch (pVCpu->hm.s.enmShadowMode)
4049 {
4050 case PGMMODE_REAL: /* Real-mode. */
4051 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4052 case PGMMODE_32_BIT: /* 32-bit paging. */
4053 {
4054 u32GuestCR4 &= ~X86_CR4_PAE;
4055 break;
4056 }
4057
4058 case PGMMODE_PAE: /* PAE paging. */
4059 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4060 {
4061 u32GuestCR4 |= X86_CR4_PAE;
4062 break;
4063 }
4064
4065 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4066 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4067#ifdef VBOX_ENABLE_64_BITS_GUESTS
4068 break;
4069#endif
4070 default:
4071 AssertFailed();
4072 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4073 }
4074 }
4075
4076 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4077 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4078 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4079 u32GuestCR4 |= uSetCR4;
4080 u32GuestCR4 &= uZapCR4;
4081
4082 /* Write VT-x's view of the guest CR4 into the VMCS. */
4083 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4084 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4085 AssertRCReturn(rc, rc);
4086
4087 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4088 uint32_t u32CR4Mask = X86_CR4_VME
4089 | X86_CR4_PAE
4090 | X86_CR4_PGE
4091 | X86_CR4_PSE
4092 | X86_CR4_VMXE;
4093 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4094 u32CR4Mask |= X86_CR4_OSXSAVE;
4095 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4096 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4097 AssertRCReturn(rc, rc);
4098
4099 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4100 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4101
4102 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4103 }
4104 return rc;
4105}
4106
4107
4108/**
4109 * Loads the guest debug registers into the guest-state area in the VMCS.
4110 *
4111 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4112 *
4113 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4114 *
4115 * @returns VBox status code.
4116 * @param pVCpu The cross context virtual CPU structure.
4117 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4118 * out-of-sync. Make sure to update the required fields
4119 * before using them.
4120 *
4121 * @remarks No-long-jump zone!!!
4122 */
4123static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4124{
4125 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4126 return VINF_SUCCESS;
4127
4128#ifdef VBOX_STRICT
4129 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4130 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4131 {
4132 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4133 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4134 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4135 }
4136#endif
4137
4138 int rc;
4139 PVM pVM = pVCpu->CTX_SUFF(pVM);
4140 bool fSteppingDB = false;
4141 bool fInterceptMovDRx = false;
4142 if (pVCpu->hm.s.fSingleInstruction)
4143 {
4144 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4145 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4146 {
4147 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4148 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4149 AssertRCReturn(rc, rc);
4150 Assert(fSteppingDB == false);
4151 }
4152 else
4153 {
4154 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4155 pVCpu->hm.s.fClearTrapFlag = true;
4156 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4157 fSteppingDB = true;
4158 }
4159 }
4160
4161 if ( fSteppingDB
4162 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4163 {
4164 /*
4165 * Use the combined guest and host DRx values found in the hypervisor
4166 * register set because the debugger has breakpoints active or someone
4167 * is single stepping on the host side without a monitor trap flag.
4168 *
4169 * Note! DBGF expects a clean DR6 state before executing guest code.
4170 */
4171#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4172 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4173 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4174 {
4175 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4176 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4177 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4178 }
4179 else
4180#endif
4181 if (!CPUMIsHyperDebugStateActive(pVCpu))
4182 {
4183 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4184 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4185 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4186 }
4187
4188 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4189 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4190 AssertRCReturn(rc, rc);
4191
4192 pVCpu->hm.s.fUsingHyperDR7 = true;
4193 fInterceptMovDRx = true;
4194 }
4195 else
4196 {
4197 /*
4198 * If the guest has enabled debug registers, we need to load them prior to
4199 * executing guest code so they'll trigger at the right time.
4200 */
4201 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4202 {
4203#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4204 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4205 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4206 {
4207 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4208 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4209 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4210 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4211 }
4212 else
4213#endif
4214 if (!CPUMIsGuestDebugStateActive(pVCpu))
4215 {
4216 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4217 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4218 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4219 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4220 }
4221 Assert(!fInterceptMovDRx);
4222 }
4223 /*
4224 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4225 * must intercept #DB in order to maintain a correct DR6 guest value, and
4226 * because we need to intercept it to prevent nested #DBs from hanging the
4227 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4228 */
4229#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4230 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4231 && !CPUMIsGuestDebugStateActive(pVCpu))
4232#else
4233 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4234#endif
4235 {
4236 fInterceptMovDRx = true;
4237 }
4238
4239 /* Update guest DR7. */
4240 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4241 AssertRCReturn(rc, rc);
4242
4243 pVCpu->hm.s.fUsingHyperDR7 = false;
4244 }
4245
4246 /*
4247 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4248 */
4249 if (fInterceptMovDRx)
4250 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4251 else
4252 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4253 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4254 AssertRCReturn(rc, rc);
4255
4256 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4257 return VINF_SUCCESS;
4258}
4259
4260
4261#ifdef VBOX_STRICT
4262/**
4263 * Strict function to validate segment registers.
4264 *
4265 * @remarks ASSUMES CR0 is up to date.
4266 */
4267static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4268{
4269 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4270 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4271 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4272 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4273 && ( !CPUMIsGuestInRealModeEx(pCtx)
4274 && !CPUMIsGuestInV86ModeEx(pCtx)))
4275 {
4276 /* Protected mode checks */
4277 /* CS */
4278 Assert(pCtx->cs.Attr.n.u1Present);
4279 Assert(!(pCtx->cs.Attr.u & 0xf00));
4280 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4281 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4282 || !(pCtx->cs.Attr.n.u1Granularity));
4283 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4284 || (pCtx->cs.Attr.n.u1Granularity));
4285 /* CS cannot be loaded with NULL in protected mode. */
4286 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4287 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4288 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4289 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4290 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4291 else
4292 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4293 /* SS */
4294 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4295 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4296 if ( !(pCtx->cr0 & X86_CR0_PE)
4297 || pCtx->cs.Attr.n.u4Type == 3)
4298 {
4299 Assert(!pCtx->ss.Attr.n.u2Dpl);
4300 }
4301 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4302 {
4303 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4304 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4305 Assert(pCtx->ss.Attr.n.u1Present);
4306 Assert(!(pCtx->ss.Attr.u & 0xf00));
4307 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4308 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4309 || !(pCtx->ss.Attr.n.u1Granularity));
4310 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4311 || (pCtx->ss.Attr.n.u1Granularity));
4312 }
4313 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4314 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4315 {
4316 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4317 Assert(pCtx->ds.Attr.n.u1Present);
4318 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4319 Assert(!(pCtx->ds.Attr.u & 0xf00));
4320 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4321 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4322 || !(pCtx->ds.Attr.n.u1Granularity));
4323 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4324 || (pCtx->ds.Attr.n.u1Granularity));
4325 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4326 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4327 }
4328 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4329 {
4330 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4331 Assert(pCtx->es.Attr.n.u1Present);
4332 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4333 Assert(!(pCtx->es.Attr.u & 0xf00));
4334 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4335 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4336 || !(pCtx->es.Attr.n.u1Granularity));
4337 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4338 || (pCtx->es.Attr.n.u1Granularity));
4339 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4340 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4341 }
4342 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4343 {
4344 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4345 Assert(pCtx->fs.Attr.n.u1Present);
4346 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4347 Assert(!(pCtx->fs.Attr.u & 0xf00));
4348 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4349 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4350 || !(pCtx->fs.Attr.n.u1Granularity));
4351 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4352 || (pCtx->fs.Attr.n.u1Granularity));
4353 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4354 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4355 }
4356 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4357 {
4358 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4359 Assert(pCtx->gs.Attr.n.u1Present);
4360 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4361 Assert(!(pCtx->gs.Attr.u & 0xf00));
4362 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4363 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4364 || !(pCtx->gs.Attr.n.u1Granularity));
4365 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4366 || (pCtx->gs.Attr.n.u1Granularity));
4367 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4368 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4369 }
4370 /* 64-bit capable CPUs. */
4371# if HC_ARCH_BITS == 64
4372 Assert(!(pCtx->cs.u64Base >> 32));
4373 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4374 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4375 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4376# endif
4377 }
4378 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4379 || ( CPUMIsGuestInRealModeEx(pCtx)
4380 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4381 {
4382 /* Real and v86 mode checks. */
4383 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4384 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4385 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4386 {
4387 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4388 }
4389 else
4390 {
4391 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4392 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4393 }
4394
4395 /* CS */
4396 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4397 Assert(pCtx->cs.u32Limit == 0xffff);
4398 Assert(u32CSAttr == 0xf3);
4399 /* SS */
4400 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4401 Assert(pCtx->ss.u32Limit == 0xffff);
4402 Assert(u32SSAttr == 0xf3);
4403 /* DS */
4404 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4405 Assert(pCtx->ds.u32Limit == 0xffff);
4406 Assert(u32DSAttr == 0xf3);
4407 /* ES */
4408 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4409 Assert(pCtx->es.u32Limit == 0xffff);
4410 Assert(u32ESAttr == 0xf3);
4411 /* FS */
4412 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4413 Assert(pCtx->fs.u32Limit == 0xffff);
4414 Assert(u32FSAttr == 0xf3);
4415 /* GS */
4416 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4417 Assert(pCtx->gs.u32Limit == 0xffff);
4418 Assert(u32GSAttr == 0xf3);
4419 /* 64-bit capable CPUs. */
4420# if HC_ARCH_BITS == 64
4421 Assert(!(pCtx->cs.u64Base >> 32));
4422 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4423 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4424 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4425# endif
4426 }
4427}
4428#endif /* VBOX_STRICT */
4429
4430
4431/**
4432 * Writes a guest segment register into the guest-state area in the VMCS.
4433 *
4434 * @returns VBox status code.
4435 * @param pVCpu The cross context virtual CPU structure.
4436 * @param idxSel Index of the selector in the VMCS.
4437 * @param idxLimit Index of the segment limit in the VMCS.
4438 * @param idxBase Index of the segment base in the VMCS.
4439 * @param idxAccess Index of the access rights of the segment in the VMCS.
4440 * @param pSelReg Pointer to the segment selector.
4441 *
4442 * @remarks No-long-jump zone!!!
4443 */
4444static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4445 uint32_t idxAccess, PCPUMSELREG pSelReg)
4446{
4447 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4448 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4449 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4450 AssertRCReturn(rc, rc);
4451
4452 uint32_t u32Access = pSelReg->Attr.u;
4453 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4454 {
4455 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4456 u32Access = 0xf3;
4457 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4458 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4459 }
4460 else
4461 {
4462 /*
4463 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4464 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4465 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4466 * loaded in protected-mode have their attribute as 0.
4467 */
4468 if (!u32Access)
4469 u32Access = X86DESCATTR_UNUSABLE;
4470 }
4471
4472 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4473 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4474 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4475
4476 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4477 AssertRCReturn(rc, rc);
4478 return rc;
4479}
4480
4481
4482/**
4483 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4484 * into the guest-state area in the VMCS.
4485 *
4486 * @returns VBox status code.
4487 * @param pVCpu The cross context virtual CPU structure.
4488 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4489 * out-of-sync. Make sure to update the required fields
4490 * before using them.
4491 *
4492 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4493 * @remarks No-long-jump zone!!!
4494 */
4495static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4496{
4497 int rc = VERR_INTERNAL_ERROR_5;
4498 PVM pVM = pVCpu->CTX_SUFF(pVM);
4499
4500 /*
4501 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4502 */
4503 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4504 {
4505 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4506 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4507 {
4508 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4509 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4510 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4511 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4512 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4513 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4514 }
4515
4516#ifdef VBOX_WITH_REM
4517 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4518 {
4519 Assert(pVM->hm.s.vmx.pRealModeTSS);
4520 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4521 if ( pVCpu->hm.s.vmx.fWasInRealMode
4522 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4523 {
4524 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4525 in real-mode (e.g. OpenBSD 4.0) */
4526 REMFlushTBs(pVM);
4527 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4528 pVCpu->hm.s.vmx.fWasInRealMode = false;
4529 }
4530 }
4531#endif
4532 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4533 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4534 AssertRCReturn(rc, rc);
4535 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4536 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4537 AssertRCReturn(rc, rc);
4538 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4539 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4540 AssertRCReturn(rc, rc);
4541 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4542 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4543 AssertRCReturn(rc, rc);
4544 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4545 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4546 AssertRCReturn(rc, rc);
4547 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4548 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4549 AssertRCReturn(rc, rc);
4550
4551#ifdef VBOX_STRICT
4552 /* Validate. */
4553 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4554#endif
4555
4556 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4557 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4558 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4559 }
4560
4561 /*
4562 * Guest TR.
4563 */
4564 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4565 {
4566 /*
4567 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4568 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4569 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4570 */
4571 uint16_t u16Sel = 0;
4572 uint32_t u32Limit = 0;
4573 uint64_t u64Base = 0;
4574 uint32_t u32AccessRights = 0;
4575
4576 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4577 {
4578 u16Sel = pMixedCtx->tr.Sel;
4579 u32Limit = pMixedCtx->tr.u32Limit;
4580 u64Base = pMixedCtx->tr.u64Base;
4581 u32AccessRights = pMixedCtx->tr.Attr.u;
4582 }
4583 else
4584 {
4585 Assert(pVM->hm.s.vmx.pRealModeTSS);
4586 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4587
4588 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4589 RTGCPHYS GCPhys;
4590 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4591 AssertRCReturn(rc, rc);
4592
4593 X86DESCATTR DescAttr;
4594 DescAttr.u = 0;
4595 DescAttr.n.u1Present = 1;
4596 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4597
4598 u16Sel = 0;
4599 u32Limit = HM_VTX_TSS_SIZE;
4600 u64Base = GCPhys; /* in real-mode phys = virt. */
4601 u32AccessRights = DescAttr.u;
4602 }
4603
4604 /* Validate. */
4605 Assert(!(u16Sel & RT_BIT(2)));
4606 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4607 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4608 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4609 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4610 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4611 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4612 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4613 Assert( (u32Limit & 0xfff) == 0xfff
4614 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4615 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4616 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4617
4618 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4619 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4620 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4621 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4622 AssertRCReturn(rc, rc);
4623
4624 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4625 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4626 }
4627
4628 /*
4629 * Guest GDTR.
4630 */
4631 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4632 {
4633 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4634 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4635 AssertRCReturn(rc, rc);
4636
4637 /* Validate. */
4638 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4639
4640 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4641 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4642 }
4643
4644 /*
4645 * Guest LDTR.
4646 */
4647 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4648 {
4649 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4650 uint32_t u32Access = 0;
4651 if (!pMixedCtx->ldtr.Attr.u)
4652 u32Access = X86DESCATTR_UNUSABLE;
4653 else
4654 u32Access = pMixedCtx->ldtr.Attr.u;
4655
4656 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4657 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4658 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4659 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4660 AssertRCReturn(rc, rc);
4661
4662 /* Validate. */
4663 if (!(u32Access & X86DESCATTR_UNUSABLE))
4664 {
4665 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4666 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4667 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4668 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4669 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4670 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4671 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4672 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4673 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4674 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4675 }
4676
4677 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4678 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4679 }
4680
4681 /*
4682 * Guest IDTR.
4683 */
4684 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4685 {
4686 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4687 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4688 AssertRCReturn(rc, rc);
4689
4690 /* Validate. */
4691 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4692
4693 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4694 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4695 }
4696
4697 return VINF_SUCCESS;
4698}
4699
4700
4701/**
4702 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4703 * areas.
4704 *
4705 * These MSRs will automatically be loaded to the host CPU on every successful
4706 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4707 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4708 * -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4709 *
4710 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4711 *
4712 * @returns VBox status code.
4713 * @param pVCpu The cross context virtual CPU structure.
4714 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4715 * out-of-sync. Make sure to update the required fields
4716 * before using them.
4717 *
4718 * @remarks No-long-jump zone!!!
4719 */
4720static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4721{
4722 AssertPtr(pVCpu);
4723 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4724
4725 /*
4726 * MSRs that we use the auto-load/store MSR area in the VMCS.
4727 */
4728 PVM pVM = pVCpu->CTX_SUFF(pVM);
4729 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4730 {
4731 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4732#if HC_ARCH_BITS == 32
4733 if (pVM->hm.s.fAllow64BitGuests)
4734 {
4735 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4736 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4737 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4738 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4739 AssertRCReturn(rc, rc);
4740# ifdef LOG_ENABLED
4741 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4742 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4743 {
4744 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4745 pMsr->u64Value));
4746 }
4747# endif
4748 }
4749#endif
4750 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4751 }
4752
4753 /*
4754 * Guest Sysenter MSRs.
4755 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4756 * VM-exits on WRMSRs for these MSRs.
4757 */
4758 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4759 {
4760 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4761 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4762 }
4763
4764 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4765 {
4766 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4767 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4768 }
4769
4770 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4771 {
4772 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4773 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4774 }
4775
4776 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4777 {
4778 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4779 {
4780 /*
4781 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4782 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4783 */
4784 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4785 {
4786 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4787 AssertRCReturn(rc,rc);
4788 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4789 }
4790 else
4791 {
4792 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4793 NULL /* pfAddedAndUpdated */);
4794 AssertRCReturn(rc, rc);
4795
4796 /* We need to intercept reads too, see @bugref{7386#c16}. */
4797 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4798 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4799 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4800 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4801 }
4802 }
4803 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4804 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4805 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4806 }
4807
4808 return VINF_SUCCESS;
4809}
4810
4811
4812/**
4813 * Loads the guest activity state into the guest-state area in the VMCS.
4814 *
4815 * @returns VBox status code.
4816 * @param pVCpu The cross context virtual CPU structure.
4817 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4818 * out-of-sync. Make sure to update the required fields
4819 * before using them.
4820 *
4821 * @remarks No-long-jump zone!!!
4822 */
4823static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4824{
4825 NOREF(pMixedCtx);
4826 /** @todo See if we can make use of other states, e.g.
4827 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4828 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4829 {
4830 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4831 AssertRCReturn(rc, rc);
4832
4833 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4834 }
4835 return VINF_SUCCESS;
4836}
4837
4838
4839#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4840/**
4841 * Check if guest state allows safe use of 32-bit switcher again.
4842 *
4843 * Segment bases and protected mode structures must be 32-bit addressable
4844 * because the 32-bit switcher will ignore high dword when writing these VMCS
4845 * fields. See @bugref{8432} for details.
4846 *
4847 * @returns true if safe, false if must continue to use the 64-bit switcher.
4848 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4849 * out-of-sync. Make sure to update the required fields
4850 * before using them.
4851 *
4852 * @remarks No-long-jump zone!!!
4853 */
4854static bool hmR0VmxIs32BitSwitcherSafe(PCPUMCTX pMixedCtx)
4855{
4856 if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000))
4857 return false;
4858 if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000))
4859 return false;
4860 if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000))
4861 return false;
4862 if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000))
4863 return false;
4864 if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000))
4865 return false;
4866 if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000))
4867 return false;
4868 if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000))
4869 return false;
4870 if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000))
4871 return false;
4872 if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000))
4873 return false;
4874 if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000))
4875 return false;
4876 /* All good, bases are 32-bit. */
4877 return true;
4878}
4879#endif
4880
4881
4882/**
4883 * Sets up the appropriate function to run guest code.
4884 *
4885 * @returns VBox status code.
4886 * @param pVCpu The cross context virtual CPU structure.
4887 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4888 * out-of-sync. Make sure to update the required fields
4889 * before using them.
4890 *
4891 * @remarks No-long-jump zone!!!
4892 */
4893static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4894{
4895 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4896 {
4897#ifndef VBOX_ENABLE_64_BITS_GUESTS
4898 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4899#endif
4900 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4901#if HC_ARCH_BITS == 32
4902 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4903 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4904 {
4905 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4906 {
4907 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4908 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4909 | HM_CHANGED_VMX_ENTRY_CTLS
4910 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4911 }
4912 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4913
4914 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4915 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4916 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4917 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 64-bit switcher\n", pVCpu->idCpu));
4918 }
4919#else
4920 /* 64-bit host. */
4921 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4922#endif
4923 }
4924 else
4925 {
4926 /* Guest is not in long mode, use the 32-bit handler. */
4927#if HC_ARCH_BITS == 32
4928 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4929 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4930 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4931 {
4932 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4933 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4934 | HM_CHANGED_VMX_ENTRY_CTLS
4935 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4936 }
4937# ifdef VBOX_ENABLE_64_BITS_GUESTS
4938 /*
4939 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel design, see @bugref{8432#c7}.
4940 * If real-on-v86 mode is active, clear the 64-bit switcher flag because now we know the guest is in a sane
4941 * state where it's safe to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4942 * the much faster 32-bit switcher again.
4943 */
4944 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4945 {
4946 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4947 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher\n", pVCpu->idCpu));
4948 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4949 }
4950 else
4951 {
4952 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4953 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4954 || hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
4955 {
4956 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4957 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4958 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR
4959 | HM_CHANGED_VMX_ENTRY_CTLS
4960 | HM_CHANGED_VMX_EXIT_CTLS
4961 | HM_CHANGED_HOST_CONTEXT);
4962 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher (safe)\n", pVCpu->idCpu));
4963 }
4964 }
4965# else
4966 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4967# endif
4968#else
4969 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4970#endif
4971 }
4972 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4973 return VINF_SUCCESS;
4974}
4975
4976
4977/**
4978 * Wrapper for running the guest code in VT-x.
4979 *
4980 * @returns VBox status code, no informational status codes.
4981 * @param pVM The cross context VM structure.
4982 * @param pVCpu The cross context virtual CPU structure.
4983 * @param pCtx Pointer to the guest-CPU context.
4984 *
4985 * @remarks No-long-jump zone!!!
4986 */
4987DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4988{
4989 /*
4990 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4991 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4992 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4993 */
4994 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4995 /** @todo Add stats for resume vs launch. */
4996#ifdef VBOX_WITH_KERNEL_USING_XMM
4997 int rc = HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4998#else
4999 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
5000#endif
5001 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
5002 return rc;
5003}
5004
5005
5006/**
5007 * Reports world-switch error and dumps some useful debug info.
5008 *
5009 * @param pVM The cross context VM structure.
5010 * @param pVCpu The cross context virtual CPU structure.
5011 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5012 * @param pCtx Pointer to the guest-CPU context.
5013 * @param pVmxTransient Pointer to the VMX transient structure (only
5014 * exitReason updated).
5015 */
5016static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
5017{
5018 Assert(pVM);
5019 Assert(pVCpu);
5020 Assert(pCtx);
5021 Assert(pVmxTransient);
5022 HMVMX_ASSERT_PREEMPT_SAFE();
5023
5024 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
5025 switch (rcVMRun)
5026 {
5027 case VERR_VMX_INVALID_VMXON_PTR:
5028 AssertFailed();
5029 break;
5030 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5031 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5032 {
5033 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5034 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5035 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5036 AssertRC(rc);
5037
5038 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5039 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5040 Cannot do it here as we may have been long preempted. */
5041
5042#ifdef VBOX_STRICT
5043 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5044 pVmxTransient->uExitReason));
5045 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5046 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5047 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5048 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5049 else
5050 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5051 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5052 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5053
5054 /* VMX control bits. */
5055 uint32_t u32Val;
5056 uint64_t u64Val;
5057 RTHCUINTREG uHCReg;
5058 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5059 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5060 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5061 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5062 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5063 {
5064 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5065 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5066 }
5067 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5068 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5069 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5070 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5071 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5072 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5073 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5074 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5075 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5076 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5077 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5078 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5079 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5080 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5081 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5082 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5083 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5084 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5085 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5086 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5087 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5088 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5089 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5090 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5091 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5092 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5093 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5094 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5095 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5096 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5097 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5098 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5099 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5100 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5101 if (pVM->hm.s.fNestedPaging)
5102 {
5103 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5104 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5105 }
5106
5107 /* Guest bits. */
5108 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5109 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5110 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5111 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5112 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5113 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5114 if (pVM->hm.s.vmx.fVpid)
5115 {
5116 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5117 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5118 }
5119
5120 /* Host bits. */
5121 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5122 Log4(("Host CR0 %#RHr\n", uHCReg));
5123 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5124 Log4(("Host CR3 %#RHr\n", uHCReg));
5125 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5126 Log4(("Host CR4 %#RHr\n", uHCReg));
5127
5128 RTGDTR HostGdtr;
5129 PCX86DESCHC pDesc;
5130 ASMGetGDTR(&HostGdtr);
5131 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5132 Log4(("Host CS %#08x\n", u32Val));
5133 if (u32Val < HostGdtr.cbGdt)
5134 {
5135 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5136 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5137 }
5138
5139 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5140 Log4(("Host DS %#08x\n", u32Val));
5141 if (u32Val < HostGdtr.cbGdt)
5142 {
5143 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5144 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5145 }
5146
5147 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5148 Log4(("Host ES %#08x\n", u32Val));
5149 if (u32Val < HostGdtr.cbGdt)
5150 {
5151 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5152 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5153 }
5154
5155 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5156 Log4(("Host FS %#08x\n", u32Val));
5157 if (u32Val < HostGdtr.cbGdt)
5158 {
5159 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5160 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5161 }
5162
5163 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5164 Log4(("Host GS %#08x\n", u32Val));
5165 if (u32Val < HostGdtr.cbGdt)
5166 {
5167 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5168 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5169 }
5170
5171 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5172 Log4(("Host SS %#08x\n", u32Val));
5173 if (u32Val < HostGdtr.cbGdt)
5174 {
5175 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5176 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5177 }
5178
5179 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5180 Log4(("Host TR %#08x\n", u32Val));
5181 if (u32Val < HostGdtr.cbGdt)
5182 {
5183 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5184 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5185 }
5186
5187 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5188 Log4(("Host TR Base %#RHv\n", uHCReg));
5189 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5190 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5191 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5192 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5193 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5194 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5195 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5196 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5197 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5198 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5199 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5200 Log4(("Host RSP %#RHv\n", uHCReg));
5201 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5202 Log4(("Host RIP %#RHv\n", uHCReg));
5203# if HC_ARCH_BITS == 64
5204 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5205 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5206 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5207 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5208 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5209 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5210# endif
5211#endif /* VBOX_STRICT */
5212 break;
5213 }
5214
5215 default:
5216 /* Impossible */
5217 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5218 break;
5219 }
5220 NOREF(pVM); NOREF(pCtx);
5221}
5222
5223
5224#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5225#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5226# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5227#endif
5228#ifdef VBOX_STRICT
5229static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5230{
5231 switch (idxField)
5232 {
5233 case VMX_VMCS_GUEST_RIP:
5234 case VMX_VMCS_GUEST_RSP:
5235 case VMX_VMCS_GUEST_SYSENTER_EIP:
5236 case VMX_VMCS_GUEST_SYSENTER_ESP:
5237 case VMX_VMCS_GUEST_GDTR_BASE:
5238 case VMX_VMCS_GUEST_IDTR_BASE:
5239 case VMX_VMCS_GUEST_CS_BASE:
5240 case VMX_VMCS_GUEST_DS_BASE:
5241 case VMX_VMCS_GUEST_ES_BASE:
5242 case VMX_VMCS_GUEST_FS_BASE:
5243 case VMX_VMCS_GUEST_GS_BASE:
5244 case VMX_VMCS_GUEST_SS_BASE:
5245 case VMX_VMCS_GUEST_LDTR_BASE:
5246 case VMX_VMCS_GUEST_TR_BASE:
5247 case VMX_VMCS_GUEST_CR3:
5248 return true;
5249 }
5250 return false;
5251}
5252
5253static bool hmR0VmxIsValidReadField(uint32_t idxField)
5254{
5255 switch (idxField)
5256 {
5257 /* Read-only fields. */
5258 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5259 return true;
5260 }
5261 /* Remaining readable fields should also be writable. */
5262 return hmR0VmxIsValidWriteField(idxField);
5263}
5264#endif /* VBOX_STRICT */
5265
5266
5267/**
5268 * Executes the specified handler in 64-bit mode.
5269 *
5270 * @returns VBox status code (no informational status codes).
5271 * @param pVM The cross context VM structure.
5272 * @param pVCpu The cross context virtual CPU structure.
5273 * @param pCtx Pointer to the guest CPU context.
5274 * @param enmOp The operation to perform.
5275 * @param cParams Number of parameters.
5276 * @param paParam Array of 32-bit parameters.
5277 */
5278VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5279 uint32_t cParams, uint32_t *paParam)
5280{
5281 NOREF(pCtx);
5282
5283 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5284 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5285 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5286 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5287
5288#ifdef VBOX_STRICT
5289 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5290 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5291
5292 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5293 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5294#endif
5295
5296 /* Disable interrupts. */
5297 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5298
5299#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5300 RTCPUID idHostCpu = RTMpCpuId();
5301 CPUMR0SetLApic(pVCpu, idHostCpu);
5302#endif
5303
5304 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5305 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5306
5307 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5308 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5309 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5310
5311 /* Leave VMX Root Mode. */
5312 VMXDisable();
5313
5314 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5315
5316 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5317 CPUMSetHyperEIP(pVCpu, enmOp);
5318 for (int i = (int)cParams - 1; i >= 0; i--)
5319 CPUMPushHyper(pVCpu, paParam[i]);
5320
5321 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5322
5323 /* Call the switcher. */
5324 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5325 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5326
5327 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5328 /* Make sure the VMX instructions don't cause #UD faults. */
5329 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5330
5331 /* Re-enter VMX Root Mode */
5332 int rc2 = VMXEnable(HCPhysCpuPage);
5333 if (RT_FAILURE(rc2))
5334 {
5335 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5336 ASMSetFlags(fOldEFlags);
5337 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5338 return rc2;
5339 }
5340
5341 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5342 AssertRC(rc2);
5343 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5344 Assert(!(ASMGetFlags() & X86_EFL_IF));
5345 ASMSetFlags(fOldEFlags);
5346 return rc;
5347}
5348
5349
5350/**
5351 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5352 * supporting 64-bit guests.
5353 *
5354 * @returns VBox status code.
5355 * @param fResume Whether to VMLAUNCH or VMRESUME.
5356 * @param pCtx Pointer to the guest-CPU context.
5357 * @param pCache Pointer to the VMCS cache.
5358 * @param pVM The cross context VM structure.
5359 * @param pVCpu The cross context virtual CPU structure.
5360 */
5361DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5362{
5363 NOREF(fResume);
5364
5365 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5366 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5367
5368#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5369 pCache->uPos = 1;
5370 pCache->interPD = PGMGetInterPaeCR3(pVM);
5371 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5372#endif
5373
5374#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5375 pCache->TestIn.HCPhysCpuPage = 0;
5376 pCache->TestIn.HCPhysVmcs = 0;
5377 pCache->TestIn.pCache = 0;
5378 pCache->TestOut.HCPhysVmcs = 0;
5379 pCache->TestOut.pCache = 0;
5380 pCache->TestOut.pCtx = 0;
5381 pCache->TestOut.eflags = 0;
5382#else
5383 NOREF(pCache);
5384#endif
5385
5386 uint32_t aParam[10];
5387 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5388 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5389 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5390 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5391 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5392 aParam[5] = 0;
5393 aParam[6] = VM_RC_ADDR(pVM, pVM);
5394 aParam[7] = 0;
5395 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5396 aParam[9] = 0;
5397
5398#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5399 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5400 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5401#endif
5402 int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5403
5404#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5405 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5406 Assert(pCtx->dr[4] == 10);
5407 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5408#endif
5409
5410#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5411 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5412 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5413 pVCpu->hm.s.vmx.HCPhysVmcs));
5414 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5415 pCache->TestOut.HCPhysVmcs));
5416 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5417 pCache->TestOut.pCache));
5418 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5419 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5420 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5421 pCache->TestOut.pCtx));
5422 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5423#endif
5424 return rc;
5425}
5426
5427
5428/**
5429 * Initialize the VMCS-Read cache.
5430 *
5431 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5432 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5433 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5434 * (those that have a 32-bit FULL & HIGH part).
5435 *
5436 * @returns VBox status code.
5437 * @param pVM The cross context VM structure.
5438 * @param pVCpu The cross context virtual CPU structure.
5439 */
5440static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5441{
5442#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5443{ \
5444 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5445 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5446 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5447 ++cReadFields; \
5448}
5449
5450 AssertPtr(pVM);
5451 AssertPtr(pVCpu);
5452 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5453 uint32_t cReadFields = 0;
5454
5455 /*
5456 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5457 * and serve to indicate exceptions to the rules.
5458 */
5459
5460 /* Guest-natural selector base fields. */
5461#if 0
5462 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5463 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5464 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5465#endif
5466 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5467 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5468 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5469 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5470 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5471 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5472 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5473 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5474 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5475 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5476 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5477 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5478#if 0
5479 /* Unused natural width guest-state fields. */
5480 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5481 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5482#endif
5483 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5484 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5485
5486 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5487#if 0
5488 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5489 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5490 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5491 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5492 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5493 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5494 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5495 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5496 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5497#endif
5498
5499 /* Natural width guest-state fields. */
5500 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5501#if 0
5502 /* Currently unused field. */
5503 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5504#endif
5505
5506 if (pVM->hm.s.fNestedPaging)
5507 {
5508 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5509 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5510 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5511 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5512 }
5513 else
5514 {
5515 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5516 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5517 }
5518
5519#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5520 return VINF_SUCCESS;
5521}
5522
5523
5524/**
5525 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5526 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5527 * darwin, running 64-bit guests).
5528 *
5529 * @returns VBox status code.
5530 * @param pVCpu The cross context virtual CPU structure.
5531 * @param idxField The VMCS field encoding.
5532 * @param u64Val 16, 32 or 64-bit value.
5533 */
5534VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5535{
5536 int rc;
5537 switch (idxField)
5538 {
5539 /*
5540 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5541 */
5542 /* 64-bit Control fields. */
5543 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5544 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5545 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5546 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5547 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5548 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5549 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5550 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5551 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5552 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5553 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5554 case VMX_VMCS64_CTRL_EPTP_FULL:
5555 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5556 /* 64-bit Guest-state fields. */
5557 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5558 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5559 case VMX_VMCS64_GUEST_PAT_FULL:
5560 case VMX_VMCS64_GUEST_EFER_FULL:
5561 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5562 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5563 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5564 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5565 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5566 /* 64-bit Host-state fields. */
5567 case VMX_VMCS64_HOST_PAT_FULL:
5568 case VMX_VMCS64_HOST_EFER_FULL:
5569 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5570 {
5571 rc = VMXWriteVmcs32(idxField, u64Val);
5572 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5573 break;
5574 }
5575
5576 /*
5577 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5578 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5579 */
5580 /* Natural-width Guest-state fields. */
5581 case VMX_VMCS_GUEST_CR3:
5582 case VMX_VMCS_GUEST_ES_BASE:
5583 case VMX_VMCS_GUEST_CS_BASE:
5584 case VMX_VMCS_GUEST_SS_BASE:
5585 case VMX_VMCS_GUEST_DS_BASE:
5586 case VMX_VMCS_GUEST_FS_BASE:
5587 case VMX_VMCS_GUEST_GS_BASE:
5588 case VMX_VMCS_GUEST_LDTR_BASE:
5589 case VMX_VMCS_GUEST_TR_BASE:
5590 case VMX_VMCS_GUEST_GDTR_BASE:
5591 case VMX_VMCS_GUEST_IDTR_BASE:
5592 case VMX_VMCS_GUEST_RSP:
5593 case VMX_VMCS_GUEST_RIP:
5594 case VMX_VMCS_GUEST_SYSENTER_ESP:
5595 case VMX_VMCS_GUEST_SYSENTER_EIP:
5596 {
5597 if (!(u64Val >> 32))
5598 {
5599 /* If this field is 64-bit, VT-x will zero out the top bits. */
5600 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5601 }
5602 else
5603 {
5604 /* Assert that only the 32->64 switcher case should ever come here. */
5605 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5606 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5607 }
5608 break;
5609 }
5610
5611 default:
5612 {
5613 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5614 rc = VERR_INVALID_PARAMETER;
5615 break;
5616 }
5617 }
5618 AssertRCReturn(rc, rc);
5619 return rc;
5620}
5621
5622
5623/**
5624 * Queue up a VMWRITE by using the VMCS write cache.
5625 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5626 *
5627 * @param pVCpu The cross context virtual CPU structure.
5628 * @param idxField The VMCS field encoding.
5629 * @param u64Val 16, 32 or 64-bit value.
5630 */
5631VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5632{
5633 AssertPtr(pVCpu);
5634 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5635
5636 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5637 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5638
5639 /* Make sure there are no duplicates. */
5640 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5641 {
5642 if (pCache->Write.aField[i] == idxField)
5643 {
5644 pCache->Write.aFieldVal[i] = u64Val;
5645 return VINF_SUCCESS;
5646 }
5647 }
5648
5649 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5650 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5651 pCache->Write.cValidEntries++;
5652 return VINF_SUCCESS;
5653}
5654#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5655
5656
5657/**
5658 * Sets up the usage of TSC-offsetting and updates the VMCS.
5659 *
5660 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5661 * VMX preemption timer.
5662 *
5663 * @returns VBox status code.
5664 * @param pVM The cross context VM structure.
5665 * @param pVCpu The cross context virtual CPU structure.
5666 *
5667 * @remarks No-long-jump zone!!!
5668 */
5669static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5670{
5671 int rc;
5672 bool fOffsettedTsc;
5673 bool fParavirtTsc;
5674 if (pVM->hm.s.vmx.fUsePreemptTimer)
5675 {
5676 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5677 &fOffsettedTsc, &fParavirtTsc);
5678
5679 /* Make sure the returned values have sane upper and lower boundaries. */
5680 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5681 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5682 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5683 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5684
5685 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5686 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5687 }
5688 else
5689 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5690
5691 /** @todo later optimize this to be done elsewhere and not before every
5692 * VM-entry. */
5693 if (fParavirtTsc)
5694 {
5695 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5696 information before every VM-entry, hence disable it for performance sake. */
5697#if 0
5698 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5699 AssertRC(rc);
5700#endif
5701 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5702 }
5703
5704 if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5705 {
5706 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5707 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5708
5709 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5710 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5711 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5712 }
5713 else
5714 {
5715 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5716 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5717 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5718 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5719 }
5720}
5721
5722
5723/**
5724 * Determines if an exception is a contributory exception.
5725 *
5726 * Contributory exceptions are ones which can cause double-faults unless the
5727 * original exception was a benign exception. Page-fault is intentionally not
5728 * included here as it's a conditional contributory exception.
5729 *
5730 * @returns true if the exception is contributory, false otherwise.
5731 * @param uVector The exception vector.
5732 */
5733DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5734{
5735 switch (uVector)
5736 {
5737 case X86_XCPT_GP:
5738 case X86_XCPT_SS:
5739 case X86_XCPT_NP:
5740 case X86_XCPT_TS:
5741 case X86_XCPT_DE:
5742 return true;
5743 default:
5744 break;
5745 }
5746 return false;
5747}
5748
5749
5750/**
5751 * Sets an event as a pending event to be injected into the guest.
5752 *
5753 * @param pVCpu The cross context virtual CPU structure.
5754 * @param u32IntInfo The VM-entry interruption-information field.
5755 * @param cbInstr The VM-entry instruction length in bytes (for software
5756 * interrupts, exceptions and privileged software
5757 * exceptions).
5758 * @param u32ErrCode The VM-entry exception error code.
5759 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5760 * page-fault.
5761 *
5762 * @remarks Statistics counter assumes this is a guest event being injected or
5763 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5764 * always incremented.
5765 */
5766DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5767 RTGCUINTPTR GCPtrFaultAddress)
5768{
5769 Assert(!pVCpu->hm.s.Event.fPending);
5770 pVCpu->hm.s.Event.fPending = true;
5771 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5772 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5773 pVCpu->hm.s.Event.cbInstr = cbInstr;
5774 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5775}
5776
5777
5778/**
5779 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5780 *
5781 * @param pVCpu The cross context virtual CPU structure.
5782 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5783 * out-of-sync. Make sure to update the required fields
5784 * before using them.
5785 */
5786DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5787{
5788 NOREF(pMixedCtx);
5789 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5790 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5791 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5792 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5793}
5794
5795
5796/**
5797 * Handle a condition that occurred while delivering an event through the guest
5798 * IDT.
5799 *
5800 * @returns Strict VBox status code (i.e. informational status codes too).
5801 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5802 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5803 * to continue execution of the guest which will delivery the \#DF.
5804 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5805 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5806 *
5807 * @param pVCpu The cross context virtual CPU structure.
5808 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5809 * out-of-sync. Make sure to update the required fields
5810 * before using them.
5811 * @param pVmxTransient Pointer to the VMX transient structure.
5812 *
5813 * @remarks No-long-jump zone!!!
5814 */
5815static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5816{
5817 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5818
5819 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5820 rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5821
5822 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5823 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5824 {
5825 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5826 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5827
5828 typedef enum
5829 {
5830 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5831 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5832 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5833 VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
5834 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5835 } VMXREFLECTXCPT;
5836
5837 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5838 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5839 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5840 {
5841 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5842 {
5843 enmReflect = VMXREFLECTXCPT_XCPT;
5844#ifdef VBOX_STRICT
5845 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5846 && uExitVector == X86_XCPT_PF)
5847 {
5848 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5849 }
5850#endif
5851 if ( uExitVector == X86_XCPT_PF
5852 && uIdtVector == X86_XCPT_PF)
5853 {
5854 pVmxTransient->fVectoringDoublePF = true;
5855 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5856 }
5857 else if ( uExitVector == X86_XCPT_AC
5858 && uIdtVector == X86_XCPT_AC)
5859 {
5860 enmReflect = VMXREFLECTXCPT_HANG;
5861 Log4(("IDT: Nested #AC - Bad guest\n"));
5862 }
5863 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5864 && hmR0VmxIsContributoryXcpt(uExitVector)
5865 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5866 || uIdtVector == X86_XCPT_PF))
5867 {
5868 enmReflect = VMXREFLECTXCPT_DF;
5869 }
5870 else if (uIdtVector == X86_XCPT_DF)
5871 enmReflect = VMXREFLECTXCPT_TF;
5872 }
5873 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5874 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5875 {
5876 /*
5877 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5878 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5879 */
5880 enmReflect = VMXREFLECTXCPT_XCPT;
5881
5882 if (uExitVector == X86_XCPT_PF)
5883 {
5884 pVmxTransient->fVectoringPF = true;
5885 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5886 }
5887 }
5888 }
5889 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5890 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5891 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5892 {
5893 /*
5894 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5895 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5896 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5897 */
5898 enmReflect = VMXREFLECTXCPT_XCPT;
5899 }
5900
5901 /*
5902 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5903 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5904 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5905 *
5906 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5907 */
5908 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5909 && enmReflect == VMXREFLECTXCPT_XCPT
5910 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5911 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5912 {
5913 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5914 }
5915
5916 switch (enmReflect)
5917 {
5918 case VMXREFLECTXCPT_XCPT:
5919 {
5920 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5921 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5922 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5923
5924 uint32_t u32ErrCode = 0;
5925 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5926 {
5927 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5928 AssertRCReturn(rc2, rc2);
5929 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5930 }
5931
5932 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5933 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5934 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5935 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5936 rcStrict = VINF_SUCCESS;
5937 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5938 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5939
5940 break;
5941 }
5942
5943 case VMXREFLECTXCPT_DF:
5944 {
5945 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5946 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5947 rcStrict = VINF_HM_DOUBLE_FAULT;
5948 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5949 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5950
5951 break;
5952 }
5953
5954 case VMXREFLECTXCPT_TF:
5955 {
5956 rcStrict = VINF_EM_RESET;
5957 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5958 uExitVector));
5959 break;
5960 }
5961
5962 case VMXREFLECTXCPT_HANG:
5963 {
5964 rcStrict = VERR_EM_GUEST_CPU_HANG;
5965 break;
5966 }
5967
5968 default:
5969 Assert(rcStrict == VINF_SUCCESS);
5970 break;
5971 }
5972 }
5973 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5974 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5975 && uExitVector != X86_XCPT_DF
5976 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5977 {
5978 /*
5979 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5980 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5981 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5982 */
5983 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5984 {
5985 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
5986 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5987 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5988 }
5989 }
5990
5991 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
5992 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
5993 return rcStrict;
5994}
5995
5996
5997/**
5998 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5999 *
6000 * @returns VBox status code.
6001 * @param pVCpu The cross context virtual CPU structure.
6002 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6003 * out-of-sync. Make sure to update the required fields
6004 * before using them.
6005 *
6006 * @remarks No-long-jump zone!!!
6007 */
6008static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6009{
6010 NOREF(pMixedCtx);
6011
6012 /*
6013 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
6014 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
6015 */
6016 VMMRZCallRing3Disable(pVCpu);
6017 HM_DISABLE_PREEMPT();
6018
6019 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
6020 {
6021 uint32_t uVal = 0;
6022 uint32_t uShadow = 0;
6023 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
6024 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
6025 AssertRCReturn(rc, rc);
6026
6027 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
6028 CPUMSetGuestCR0(pVCpu, uVal);
6029 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
6030 }
6031
6032 HM_RESTORE_PREEMPT();
6033 VMMRZCallRing3Enable(pVCpu);
6034 return VINF_SUCCESS;
6035}
6036
6037
6038/**
6039 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
6040 *
6041 * @returns VBox status code.
6042 * @param pVCpu The cross context virtual CPU structure.
6043 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6044 * out-of-sync. Make sure to update the required fields
6045 * before using them.
6046 *
6047 * @remarks No-long-jump zone!!!
6048 */
6049static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6050{
6051 NOREF(pMixedCtx);
6052
6053 int rc = VINF_SUCCESS;
6054 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
6055 {
6056 uint32_t uVal = 0;
6057 uint32_t uShadow = 0;
6058 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
6059 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
6060 AssertRCReturn(rc, rc);
6061
6062 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
6063 CPUMSetGuestCR4(pVCpu, uVal);
6064 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6065 }
6066 return rc;
6067}
6068
6069
6070/**
6071 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
6072 *
6073 * @returns VBox status code.
6074 * @param pVCpu The cross context virtual CPU structure.
6075 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6076 * out-of-sync. Make sure to update the required fields
6077 * before using them.
6078 *
6079 * @remarks No-long-jump zone!!!
6080 */
6081static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6082{
6083 int rc = VINF_SUCCESS;
6084 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6085 {
6086 uint64_t u64Val = 0;
6087 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6088 AssertRCReturn(rc, rc);
6089
6090 pMixedCtx->rip = u64Val;
6091 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6092 }
6093 return rc;
6094}
6095
6096
6097/**
6098 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6099 *
6100 * @returns VBox status code.
6101 * @param pVCpu The cross context virtual CPU structure.
6102 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6103 * out-of-sync. Make sure to update the required fields
6104 * before using them.
6105 *
6106 * @remarks No-long-jump zone!!!
6107 */
6108static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6109{
6110 int rc = VINF_SUCCESS;
6111 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6112 {
6113 uint64_t u64Val = 0;
6114 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6115 AssertRCReturn(rc, rc);
6116
6117 pMixedCtx->rsp = u64Val;
6118 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6119 }
6120 return rc;
6121}
6122
6123
6124/**
6125 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6126 *
6127 * @returns VBox status code.
6128 * @param pVCpu The cross context virtual CPU structure.
6129 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6130 * out-of-sync. Make sure to update the required fields
6131 * before using them.
6132 *
6133 * @remarks No-long-jump zone!!!
6134 */
6135static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6136{
6137 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6138 {
6139 uint32_t uVal = 0;
6140 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6141 AssertRCReturn(rc, rc);
6142
6143 pMixedCtx->eflags.u32 = uVal;
6144 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6145 {
6146 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6147 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6148
6149 pMixedCtx->eflags.Bits.u1VM = 0;
6150 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6151 }
6152
6153 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6154 }
6155 return VINF_SUCCESS;
6156}
6157
6158
6159/**
6160 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6161 * guest-CPU context.
6162 */
6163DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6164{
6165 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6166 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6167 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6168 return rc;
6169}
6170
6171
6172/**
6173 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6174 * from the guest-state area in the VMCS.
6175 *
6176 * @param pVCpu The cross context virtual CPU structure.
6177 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6178 * out-of-sync. Make sure to update the required fields
6179 * before using them.
6180 *
6181 * @remarks No-long-jump zone!!!
6182 */
6183static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6184{
6185 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6186 {
6187 uint32_t uIntrState = 0;
6188 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6189 AssertRC(rc);
6190
6191 if (!uIntrState)
6192 {
6193 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6194 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6195
6196 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6197 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6198 }
6199 else
6200 {
6201 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6202 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6203 {
6204 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6205 AssertRC(rc);
6206 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6207 AssertRC(rc);
6208
6209 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6210 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6211 }
6212 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6213 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6214
6215 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6216 {
6217 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6218 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6219 }
6220 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6221 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6222 }
6223
6224 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6225 }
6226}
6227
6228
6229/**
6230 * Saves the guest's activity state.
6231 *
6232 * @returns VBox status code.
6233 * @param pVCpu The cross context virtual CPU structure.
6234 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6235 * out-of-sync. Make sure to update the required fields
6236 * before using them.
6237 *
6238 * @remarks No-long-jump zone!!!
6239 */
6240static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6241{
6242 NOREF(pMixedCtx);
6243 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6244 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6245 return VINF_SUCCESS;
6246}
6247
6248
6249/**
6250 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6251 * the current VMCS into the guest-CPU context.
6252 *
6253 * @returns VBox status code.
6254 * @param pVCpu The cross context virtual CPU structure.
6255 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6256 * out-of-sync. Make sure to update the required fields
6257 * before using them.
6258 *
6259 * @remarks No-long-jump zone!!!
6260 */
6261static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6262{
6263 int rc = VINF_SUCCESS;
6264 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6265 {
6266 uint32_t u32Val = 0;
6267 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6268 pMixedCtx->SysEnter.cs = u32Val;
6269 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6270 }
6271
6272 uint64_t u64Val = 0;
6273 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6274 {
6275 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6276 pMixedCtx->SysEnter.eip = u64Val;
6277 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6278 }
6279 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6280 {
6281 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6282 pMixedCtx->SysEnter.esp = u64Val;
6283 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6284 }
6285 return rc;
6286}
6287
6288
6289/**
6290 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6291 * the CPU back into the guest-CPU context.
6292 *
6293 * @returns VBox status code.
6294 * @param pVCpu The cross context virtual CPU structure.
6295 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6296 * out-of-sync. Make sure to update the required fields
6297 * before using them.
6298 *
6299 * @remarks No-long-jump zone!!!
6300 */
6301static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6302{
6303 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6304 VMMRZCallRing3Disable(pVCpu);
6305 HM_DISABLE_PREEMPT();
6306
6307 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6308 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6309 {
6310 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6311 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6312 }
6313
6314 HM_RESTORE_PREEMPT();
6315 VMMRZCallRing3Enable(pVCpu);
6316
6317 return VINF_SUCCESS;
6318}
6319
6320
6321/**
6322 * Saves the auto load/store'd guest MSRs from the current VMCS into
6323 * the guest-CPU context.
6324 *
6325 * @returns VBox status code.
6326 * @param pVCpu The cross context virtual CPU structure.
6327 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6328 * out-of-sync. Make sure to update the required fields
6329 * before using them.
6330 *
6331 * @remarks No-long-jump zone!!!
6332 */
6333static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6334{
6335 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6336 return VINF_SUCCESS;
6337
6338 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6339 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6340 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6341 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6342 {
6343 switch (pMsr->u32Msr)
6344 {
6345 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6346 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6347 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6348 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6349 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6350 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6351 break;
6352
6353 default:
6354 {
6355 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6356 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6357 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6358 }
6359 }
6360 }
6361
6362 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6363 return VINF_SUCCESS;
6364}
6365
6366
6367/**
6368 * Saves the guest control registers from the current VMCS into the guest-CPU
6369 * context.
6370 *
6371 * @returns VBox status code.
6372 * @param pVCpu The cross context virtual CPU structure.
6373 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6374 * out-of-sync. Make sure to update the required fields
6375 * before using them.
6376 *
6377 * @remarks No-long-jump zone!!!
6378 */
6379static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6380{
6381 /* Guest CR0. Guest FPU. */
6382 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6383 AssertRCReturn(rc, rc);
6384
6385 /* Guest CR4. */
6386 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6387 AssertRCReturn(rc, rc);
6388
6389 /* Guest CR2 - updated always during the world-switch or in #PF. */
6390 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6391 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6392 {
6393 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6394 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6395
6396 PVM pVM = pVCpu->CTX_SUFF(pVM);
6397 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6398 || ( pVM->hm.s.fNestedPaging
6399 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6400 {
6401 uint64_t u64Val = 0;
6402 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6403 if (pMixedCtx->cr3 != u64Val)
6404 {
6405 CPUMSetGuestCR3(pVCpu, u64Val);
6406 if (VMMRZCallRing3IsEnabled(pVCpu))
6407 {
6408 PGMUpdateCR3(pVCpu, u64Val);
6409 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6410 }
6411 else
6412 {
6413 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6414 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6415 }
6416 }
6417
6418 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6419 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6420 {
6421 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6422 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6423 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6424 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6425 AssertRCReturn(rc, rc);
6426
6427 if (VMMRZCallRing3IsEnabled(pVCpu))
6428 {
6429 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6430 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6431 }
6432 else
6433 {
6434 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6435 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6436 }
6437 }
6438 }
6439
6440 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6441 }
6442
6443 /*
6444 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6445 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6446 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6447 *
6448 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6449 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6450 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6451 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6452 *
6453 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6454 */
6455 if (VMMRZCallRing3IsEnabled(pVCpu))
6456 {
6457 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6458 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6459
6460 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6461 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6462
6463 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6464 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6465 }
6466
6467 return rc;
6468}
6469
6470
6471/**
6472 * Reads a guest segment register from the current VMCS into the guest-CPU
6473 * context.
6474 *
6475 * @returns VBox status code.
6476 * @param pVCpu The cross context virtual CPU structure.
6477 * @param idxSel Index of the selector in the VMCS.
6478 * @param idxLimit Index of the segment limit in the VMCS.
6479 * @param idxBase Index of the segment base in the VMCS.
6480 * @param idxAccess Index of the access rights of the segment in the VMCS.
6481 * @param pSelReg Pointer to the segment selector.
6482 *
6483 * @remarks No-long-jump zone!!!
6484 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6485 * macro as that takes care of whether to read from the VMCS cache or
6486 * not.
6487 */
6488DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6489 PCPUMSELREG pSelReg)
6490{
6491 NOREF(pVCpu);
6492
6493 uint32_t u32Val = 0;
6494 int rc = VMXReadVmcs32(idxSel, &u32Val);
6495 AssertRCReturn(rc, rc);
6496 pSelReg->Sel = (uint16_t)u32Val;
6497 pSelReg->ValidSel = (uint16_t)u32Val;
6498 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6499
6500 rc = VMXReadVmcs32(idxLimit, &u32Val);
6501 AssertRCReturn(rc, rc);
6502 pSelReg->u32Limit = u32Val;
6503
6504 uint64_t u64Val = 0;
6505 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6506 AssertRCReturn(rc, rc);
6507 pSelReg->u64Base = u64Val;
6508
6509 rc = VMXReadVmcs32(idxAccess, &u32Val);
6510 AssertRCReturn(rc, rc);
6511 pSelReg->Attr.u = u32Val;
6512
6513 /*
6514 * If VT-x marks the segment as unusable, most other bits remain undefined:
6515 * - For CS the L, D and G bits have meaning.
6516 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6517 * - For the remaining data segments no bits are defined.
6518 *
6519 * The present bit and the unusable bit has been observed to be set at the
6520 * same time (the selector was supposed to be invalid as we started executing
6521 * a V8086 interrupt in ring-0).
6522 *
6523 * What should be important for the rest of the VBox code, is that the P bit is
6524 * cleared. Some of the other VBox code recognizes the unusable bit, but
6525 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6526 * safe side here, we'll strip off P and other bits we don't care about. If
6527 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6528 *
6529 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6530 */
6531 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6532 {
6533 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6534
6535 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6536 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6537 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6538
6539 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6540#ifdef DEBUG_bird
6541 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6542 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6543 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6544#endif
6545 }
6546 return VINF_SUCCESS;
6547}
6548
6549
6550#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6551# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6552 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6553 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6554#else
6555# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6556 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6557 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6558#endif
6559
6560
6561/**
6562 * Saves the guest segment registers from the current VMCS into the guest-CPU
6563 * context.
6564 *
6565 * @returns VBox status code.
6566 * @param pVCpu The cross context virtual CPU structure.
6567 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6568 * out-of-sync. Make sure to update the required fields
6569 * before using them.
6570 *
6571 * @remarks No-long-jump zone!!!
6572 */
6573static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6574{
6575 /* Guest segment registers. */
6576 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6577 {
6578 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6579 AssertRCReturn(rc, rc);
6580
6581 rc = VMXLOCAL_READ_SEG(CS, cs);
6582 rc |= VMXLOCAL_READ_SEG(SS, ss);
6583 rc |= VMXLOCAL_READ_SEG(DS, ds);
6584 rc |= VMXLOCAL_READ_SEG(ES, es);
6585 rc |= VMXLOCAL_READ_SEG(FS, fs);
6586 rc |= VMXLOCAL_READ_SEG(GS, gs);
6587 AssertRCReturn(rc, rc);
6588
6589 /* Restore segment attributes for real-on-v86 mode hack. */
6590 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6591 {
6592 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6593 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6594 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6595 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6596 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6597 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6598 }
6599 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6600 }
6601
6602 return VINF_SUCCESS;
6603}
6604
6605
6606/**
6607 * Saves the guest descriptor table registers and task register from the current
6608 * VMCS into the guest-CPU context.
6609 *
6610 * @returns VBox status code.
6611 * @param pVCpu The cross context virtual CPU structure.
6612 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6613 * out-of-sync. Make sure to update the required fields
6614 * before using them.
6615 *
6616 * @remarks No-long-jump zone!!!
6617 */
6618static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6619{
6620 int rc = VINF_SUCCESS;
6621
6622 /* Guest LDTR. */
6623 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6624 {
6625 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6626 AssertRCReturn(rc, rc);
6627 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6628 }
6629
6630 /* Guest GDTR. */
6631 uint64_t u64Val = 0;
6632 uint32_t u32Val = 0;
6633 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6634 {
6635 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6636 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6637 pMixedCtx->gdtr.pGdt = u64Val;
6638 pMixedCtx->gdtr.cbGdt = u32Val;
6639 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6640 }
6641
6642 /* Guest IDTR. */
6643 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6644 {
6645 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6646 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6647 pMixedCtx->idtr.pIdt = u64Val;
6648 pMixedCtx->idtr.cbIdt = u32Val;
6649 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6650 }
6651
6652 /* Guest TR. */
6653 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6654 {
6655 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6656 AssertRCReturn(rc, rc);
6657
6658 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6659 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6660 {
6661 rc = VMXLOCAL_READ_SEG(TR, tr);
6662 AssertRCReturn(rc, rc);
6663 }
6664 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6665 }
6666 return rc;
6667}
6668
6669#undef VMXLOCAL_READ_SEG
6670
6671
6672/**
6673 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6674 * context.
6675 *
6676 * @returns VBox status code.
6677 * @param pVCpu The cross context virtual CPU structure.
6678 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6679 * out-of-sync. Make sure to update the required fields
6680 * before using them.
6681 *
6682 * @remarks No-long-jump zone!!!
6683 */
6684static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6685{
6686 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6687 {
6688 if (!pVCpu->hm.s.fUsingHyperDR7)
6689 {
6690 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6691 uint32_t u32Val;
6692 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6693 pMixedCtx->dr[7] = u32Val;
6694 }
6695
6696 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6697 }
6698 return VINF_SUCCESS;
6699}
6700
6701
6702/**
6703 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6704 *
6705 * @returns VBox status code.
6706 * @param pVCpu The cross context virtual CPU structure.
6707 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6708 * out-of-sync. Make sure to update the required fields
6709 * before using them.
6710 *
6711 * @remarks No-long-jump zone!!!
6712 */
6713static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6714{
6715 NOREF(pMixedCtx);
6716
6717 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6718 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6719 return VINF_SUCCESS;
6720}
6721
6722
6723/**
6724 * Saves the entire guest state from the currently active VMCS into the
6725 * guest-CPU context.
6726 *
6727 * This essentially VMREADs all guest-data.
6728 *
6729 * @returns VBox status code.
6730 * @param pVCpu The cross context virtual CPU structure.
6731 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6732 * out-of-sync. Make sure to update the required fields
6733 * before using them.
6734 */
6735static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6736{
6737 Assert(pVCpu);
6738 Assert(pMixedCtx);
6739
6740 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6741 return VINF_SUCCESS;
6742
6743 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6744 again on the ring-3 callback path, there is no real need to. */
6745 if (VMMRZCallRing3IsEnabled(pVCpu))
6746 VMMR0LogFlushDisable(pVCpu);
6747 else
6748 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6749 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6750
6751 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6752 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6753
6754 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6755 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6756
6757 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6758 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6759
6760 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6761 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6762
6763 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6764 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6765
6766 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6767 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6768
6769 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6770 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6771
6772 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6773 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6774
6775 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6776 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6777
6778 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6779 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6780
6781 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6782 ("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
6783 HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
6784
6785 if (VMMRZCallRing3IsEnabled(pVCpu))
6786 VMMR0LogFlushEnable(pVCpu);
6787
6788 return VINF_SUCCESS;
6789}
6790
6791
6792/**
6793 * Saves basic guest registers needed for IEM instruction execution.
6794 *
6795 * @returns VBox status code (OR-able).
6796 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6797 * @param pMixedCtx Pointer to the CPU context of the guest.
6798 * @param fMemory Whether the instruction being executed operates on
6799 * memory or not. Only CR0 is synced up if clear.
6800 * @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
6801 */
6802static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
6803{
6804 /*
6805 * We assume all general purpose registers other than RSP are available.
6806 *
6807 * - RIP is a must, as it will be incremented or otherwise changed.
6808 * - RFLAGS are always required to figure the CPL.
6809 * - RSP isn't always required, however it's a GPR, so frequently required.
6810 * - SS and CS are the only segment register needed if IEM doesn't do memory
6811 * access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
6812 * - CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
6813 * be required for memory accesses.
6814 *
6815 * Note! Before IEM dispatches an exception, it will call us to sync in everything.
6816 */
6817 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6818 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6819 if (fNeedRsp)
6820 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6821 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6822 if (!fMemory)
6823 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6824 else
6825 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6826 AssertRCReturn(rc, rc);
6827 return rc;
6828}
6829
6830
6831/**
6832 * Ensures that we've got a complete basic guest-context.
6833 *
6834 * This excludes the FPU, SSE, AVX, and similar extended state. The interface
6835 * is for the interpreter.
6836 *
6837 * @returns VBox status code.
6838 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6839 * @param pMixedCtx Pointer to the guest-CPU context which may have data
6840 * needing to be synced in.
6841 * @thread EMT(pVCpu)
6842 */
6843VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6844{
6845 /* Note! Since this is only applicable to VT-x, the implementation is placed
6846 in the VT-x part of the sources instead of the generic stuff. */
6847 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
6848 {
6849 /* For now, imply that the caller might change everything too. */
6850 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6851 return hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6852 }
6853 return VINF_SUCCESS;
6854}
6855
6856
6857/**
6858 * Check per-VM and per-VCPU force flag actions that require us to go back to
6859 * ring-3 for one reason or another.
6860 *
6861 * @returns Strict VBox status code (i.e. informational status codes too)
6862 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6863 * ring-3.
6864 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6865 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6866 * interrupts)
6867 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6868 * all EMTs to be in ring-3.
6869 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6870 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6871 * to the EM loop.
6872 *
6873 * @param pVM The cross context VM structure.
6874 * @param pVCpu The cross context virtual CPU structure.
6875 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6876 * out-of-sync. Make sure to update the required fields
6877 * before using them.
6878 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6879 */
6880static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6881{
6882 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6883
6884 /*
6885 * Anything pending? Should be more likely than not if we're doing a good job.
6886 */
6887 if ( !fStepping
6888 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6889 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6890 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6891 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6892 return VINF_SUCCESS;
6893
6894 /* We need the control registers now, make sure the guest-CPU context is updated. */
6895 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6896 AssertRCReturn(rc3, rc3);
6897
6898 /* Pending HM CR3 sync. */
6899 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6900 {
6901 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6902 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6903 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6904 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6905 }
6906
6907 /* Pending HM PAE PDPEs. */
6908 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6909 {
6910 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6911 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6912 }
6913
6914 /* Pending PGM C3 sync. */
6915 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6916 {
6917 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6918 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6919 if (rcStrict2 != VINF_SUCCESS)
6920 {
6921 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6922 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6923 return rcStrict2;
6924 }
6925 }
6926
6927 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6928 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6929 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6930 {
6931 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6932 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6933 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6934 return rc2;
6935 }
6936
6937 /* Pending VM request packets, such as hardware interrupts. */
6938 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6939 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6940 {
6941 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6942 return VINF_EM_PENDING_REQUEST;
6943 }
6944
6945 /* Pending PGM pool flushes. */
6946 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6947 {
6948 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6949 return VINF_PGM_POOL_FLUSH_PENDING;
6950 }
6951
6952 /* Pending DMA requests. */
6953 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6954 {
6955 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6956 return VINF_EM_RAW_TO_R3;
6957 }
6958
6959 return VINF_SUCCESS;
6960}
6961
6962
6963/**
6964 * Converts any TRPM trap into a pending HM event. This is typically used when
6965 * entering from ring-3 (not longjmp returns).
6966 *
6967 * @param pVCpu The cross context virtual CPU structure.
6968 */
6969static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6970{
6971 Assert(TRPMHasTrap(pVCpu));
6972 Assert(!pVCpu->hm.s.Event.fPending);
6973
6974 uint8_t uVector;
6975 TRPMEVENT enmTrpmEvent;
6976 RTGCUINT uErrCode;
6977 RTGCUINTPTR GCPtrFaultAddress;
6978 uint8_t cbInstr;
6979
6980 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6981 AssertRC(rc);
6982
6983 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6984 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6985 if (enmTrpmEvent == TRPM_TRAP)
6986 {
6987 switch (uVector)
6988 {
6989 case X86_XCPT_NMI:
6990 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6991 break;
6992
6993 case X86_XCPT_BP:
6994 case X86_XCPT_OF:
6995 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6996 break;
6997
6998 case X86_XCPT_PF:
6999 case X86_XCPT_DF:
7000 case X86_XCPT_TS:
7001 case X86_XCPT_NP:
7002 case X86_XCPT_SS:
7003 case X86_XCPT_GP:
7004 case X86_XCPT_AC:
7005 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7006 /* no break! */
7007 default:
7008 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7009 break;
7010 }
7011 }
7012 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7013 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7014 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7015 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7016 else
7017 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7018
7019 rc = TRPMResetTrap(pVCpu);
7020 AssertRC(rc);
7021 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7022 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7023
7024 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7025}
7026
7027
7028/**
7029 * Converts the pending HM event into a TRPM trap.
7030 *
7031 * @param pVCpu The cross context virtual CPU structure.
7032 */
7033static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7034{
7035 Assert(pVCpu->hm.s.Event.fPending);
7036
7037 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7038 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7039 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
7040 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7041
7042 /* If a trap was already pending, we did something wrong! */
7043 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7044
7045 TRPMEVENT enmTrapType;
7046 switch (uVectorType)
7047 {
7048 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7049 enmTrapType = TRPM_HARDWARE_INT;
7050 break;
7051
7052 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7053 enmTrapType = TRPM_SOFTWARE_INT;
7054 break;
7055
7056 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7057 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7058 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7059 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7060 enmTrapType = TRPM_TRAP;
7061 break;
7062
7063 default:
7064 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7065 enmTrapType = TRPM_32BIT_HACK;
7066 break;
7067 }
7068
7069 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7070
7071 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7072 AssertRC(rc);
7073
7074 if (fErrorCodeValid)
7075 TRPMSetErrorCode(pVCpu, uErrorCode);
7076
7077 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7078 && uVector == X86_XCPT_PF)
7079 {
7080 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7081 }
7082 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7083 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7084 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7085 {
7086 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7087 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
7088 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7089 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7090 }
7091
7092 /* Clear any pending events from the VMCS. */
7093 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
7094 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
7095
7096 /* We're now done converting the pending event. */
7097 pVCpu->hm.s.Event.fPending = false;
7098}
7099
7100
7101/**
7102 * Does the necessary state syncing before returning to ring-3 for any reason
7103 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7104 *
7105 * @returns VBox status code.
7106 * @param pVCpu The cross context virtual CPU structure.
7107 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7108 * be out-of-sync. Make sure to update the required
7109 * fields before using them.
7110 * @param fSaveGuestState Whether to save the guest state or not.
7111 *
7112 * @remarks No-long-jmp zone!!!
7113 */
7114static int hmR0VmxLeave(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7115{
7116 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7117 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7118
7119 RTCPUID idCpu = RTMpCpuId();
7120 Log4Func(("HostCpuId=%u\n", idCpu));
7121
7122 /*
7123 * !!! IMPORTANT !!!
7124 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7125 */
7126
7127 /* Save the guest state if necessary. */
7128 if ( fSaveGuestState
7129 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7130 {
7131 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7132 AssertRCReturn(rc, rc);
7133 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7134 }
7135
7136 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
7137 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7138 {
7139 if (fSaveGuestState)
7140 {
7141 /* We shouldn't reload CR0 without saving it first. */
7142 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7143 AssertRCReturn(rc, rc);
7144 }
7145 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7146 }
7147
7148 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7149#ifdef VBOX_STRICT
7150 if (CPUMIsHyperDebugStateActive(pVCpu))
7151 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7152#endif
7153 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7154 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7155 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7156 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7157
7158#if HC_ARCH_BITS == 64
7159 /* Restore host-state bits that VT-x only restores partially. */
7160 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7161 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7162 {
7163 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7164 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7165 }
7166 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7167#endif
7168
7169 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7170 if (pVCpu->hm.s.vmx.fLazyMsrs)
7171 {
7172 /* We shouldn't reload the guest MSRs without saving it first. */
7173 if (!fSaveGuestState)
7174 {
7175 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7176 AssertRCReturn(rc, rc);
7177 }
7178 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7179 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7180 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7181 }
7182
7183 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7184 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7185
7186 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7187 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7188 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7189 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7190 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7191 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7192 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7193 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7194
7195 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7196
7197 /** @todo This partially defeats the purpose of having preemption hooks.
7198 * The problem is, deregistering the hooks should be moved to a place that
7199 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7200 * context.
7201 */
7202 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7203 {
7204 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7205 AssertRCReturn(rc, rc);
7206
7207 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7208 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7209 }
7210 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7211 NOREF(idCpu);
7212
7213 return VINF_SUCCESS;
7214}
7215
7216
7217/**
7218 * Leaves the VT-x session.
7219 *
7220 * @returns VBox status code.
7221 * @param pVCpu The cross context virtual CPU structure.
7222 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7223 * out-of-sync. Make sure to update the required fields
7224 * before using them.
7225 *
7226 * @remarks No-long-jmp zone!!!
7227 */
7228DECLINLINE(int) hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7229{
7230 HM_DISABLE_PREEMPT();
7231 HMVMX_ASSERT_CPU_SAFE();
7232 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7233 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7234
7235 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7236 and done this from the VMXR0ThreadCtxCallback(). */
7237 if (!pVCpu->hm.s.fLeaveDone)
7238 {
7239 int rc2 = hmR0VmxLeave(pVCpu, pMixedCtx, true /* fSaveGuestState */);
7240 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7241 pVCpu->hm.s.fLeaveDone = true;
7242 }
7243 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7244
7245 /*
7246 * !!! IMPORTANT !!!
7247 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7248 */
7249
7250 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7251 /** @todo Deregistering here means we need to VMCLEAR always
7252 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7253 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7254 VMMR0ThreadCtxHookDisable(pVCpu);
7255
7256 /* Leave HM context. This takes care of local init (term). */
7257 int rc = HMR0LeaveCpu(pVCpu);
7258
7259 HM_RESTORE_PREEMPT();
7260 return rc;
7261}
7262
7263
7264/**
7265 * Does the necessary state syncing before doing a longjmp to ring-3.
7266 *
7267 * @returns VBox status code.
7268 * @param pVCpu The cross context virtual CPU structure.
7269 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7270 * out-of-sync. Make sure to update the required fields
7271 * before using them.
7272 *
7273 * @remarks No-long-jmp zone!!!
7274 */
7275DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7276{
7277 return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7278}
7279
7280
7281/**
7282 * Take necessary actions before going back to ring-3.
7283 *
7284 * An action requires us to go back to ring-3. This function does the necessary
7285 * steps before we can safely return to ring-3. This is not the same as longjmps
7286 * to ring-3, this is voluntary and prepares the guest so it may continue
7287 * executing outside HM (recompiler/IEM).
7288 *
7289 * @returns VBox status code.
7290 * @param pVM The cross context VM structure.
7291 * @param pVCpu The cross context virtual CPU structure.
7292 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7293 * out-of-sync. Make sure to update the required fields
7294 * before using them.
7295 * @param rcExit The reason for exiting to ring-3. Can be
7296 * VINF_VMM_UNKNOWN_RING3_CALL.
7297 */
7298static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7299{
7300 Assert(pVM);
7301 Assert(pVCpu);
7302 Assert(pMixedCtx);
7303 HMVMX_ASSERT_PREEMPT_SAFE();
7304
7305 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7306 {
7307 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7308 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7309 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7310 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7311 }
7312
7313 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7314 VMMRZCallRing3Disable(pVCpu);
7315 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7316
7317 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7318 if (pVCpu->hm.s.Event.fPending)
7319 {
7320 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7321 Assert(!pVCpu->hm.s.Event.fPending);
7322 }
7323
7324 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7325 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7326
7327 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7328 and if we're injecting an event we should have a TRPM trap pending. */
7329 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7330#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7331 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7332#endif
7333
7334 /* Save guest state and restore host state bits. */
7335 int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7336 AssertRCReturn(rc, rc);
7337 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7338 /* Thread-context hooks are unregistered at this point!!! */
7339
7340 /* Sync recompiler state. */
7341 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7342 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7343 | CPUM_CHANGED_LDTR
7344 | CPUM_CHANGED_GDTR
7345 | CPUM_CHANGED_IDTR
7346 | CPUM_CHANGED_TR
7347 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7348 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7349 if ( pVM->hm.s.fNestedPaging
7350 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7351 {
7352 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7353 }
7354
7355 Assert(!pVCpu->hm.s.fClearTrapFlag);
7356
7357 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7358 if (rcExit != VINF_EM_RAW_INTERRUPT)
7359 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7360
7361 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7362
7363 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7364 VMMRZCallRing3RemoveNotification(pVCpu);
7365 VMMRZCallRing3Enable(pVCpu);
7366
7367 return rc;
7368}
7369
7370
7371/**
7372 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7373 * longjump to ring-3 and possibly get preempted.
7374 *
7375 * @returns VBox status code.
7376 * @param pVCpu The cross context virtual CPU structure.
7377 * @param enmOperation The operation causing the ring-3 longjump.
7378 * @param pvUser Opaque pointer to the guest-CPU context. The data
7379 * may be out-of-sync. Make sure to update the required
7380 * fields before using them.
7381 */
7382static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7383{
7384 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7385 {
7386 /*
7387 * !!! IMPORTANT !!!
7388 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7389 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7390 */
7391 VMMRZCallRing3RemoveNotification(pVCpu);
7392 VMMRZCallRing3Disable(pVCpu);
7393 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7394 RTThreadPreemptDisable(&PreemptState);
7395
7396 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7397 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7398
7399#if HC_ARCH_BITS == 64
7400 /* Restore host-state bits that VT-x only restores partially. */
7401 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7402 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7403 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7404 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7405#endif
7406 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7407 if (pVCpu->hm.s.vmx.fLazyMsrs)
7408 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7409
7410 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7411 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7412 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7413 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7414 {
7415 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7416 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7417 }
7418
7419 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7420 VMMR0ThreadCtxHookDisable(pVCpu);
7421 HMR0LeaveCpu(pVCpu);
7422 RTThreadPreemptRestore(&PreemptState);
7423 return VINF_SUCCESS;
7424 }
7425
7426 Assert(pVCpu);
7427 Assert(pvUser);
7428 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7429 HMVMX_ASSERT_PREEMPT_SAFE();
7430
7431 VMMRZCallRing3Disable(pVCpu);
7432 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7433
7434 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7435 enmOperation));
7436
7437 int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7438 AssertRCReturn(rc, rc);
7439
7440 VMMRZCallRing3Enable(pVCpu);
7441 return VINF_SUCCESS;
7442}
7443
7444
7445/**
7446 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7447 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7448 *
7449 * @param pVCpu The cross context virtual CPU structure.
7450 */
7451DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7452{
7453 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7454 {
7455 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7456 {
7457 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7458 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7459 AssertRC(rc);
7460 Log4(("Setup interrupt-window exiting\n"));
7461 }
7462 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7463}
7464
7465
7466/**
7467 * Clears the interrupt-window exiting control in the VMCS.
7468 *
7469 * @param pVCpu The cross context virtual CPU structure.
7470 */
7471DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7472{
7473 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7474 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7475 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7476 AssertRC(rc);
7477 Log4(("Cleared interrupt-window exiting\n"));
7478}
7479
7480
7481/**
7482 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7483 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7484 *
7485 * @param pVCpu The cross context virtual CPU structure.
7486 */
7487DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7488{
7489 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7490 {
7491 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7492 {
7493 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7494 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7495 AssertRC(rc);
7496 Log4(("Setup NMI-window exiting\n"));
7497 }
7498 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7499}
7500
7501
7502/**
7503 * Clears the NMI-window exiting control in the VMCS.
7504 *
7505 * @param pVCpu The cross context virtual CPU structure.
7506 */
7507DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7508{
7509 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7510 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7511 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7512 AssertRC(rc);
7513 Log4(("Cleared NMI-window exiting\n"));
7514}
7515
7516
7517/**
7518 * Evaluates the event to be delivered to the guest and sets it as the pending
7519 * event.
7520 *
7521 * @returns The VT-x guest-interruptibility state.
7522 * @param pVCpu The cross context virtual CPU structure.
7523 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7524 * out-of-sync. Make sure to update the required fields
7525 * before using them.
7526 */
7527static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7528{
7529 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7530 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7531 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7532 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7533 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7534
7535 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7536 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7537 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7538 Assert(!TRPMHasTrap(pVCpu));
7539
7540 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7541 APICUpdatePendingInterrupts(pVCpu);
7542
7543 /*
7544 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7545 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7546 */
7547 /** @todo SMI. SMIs take priority over NMIs. */
7548 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7549 {
7550 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7551 if ( !pVCpu->hm.s.Event.fPending
7552 && !fBlockNmi
7553 && !fBlockSti
7554 && !fBlockMovSS)
7555 {
7556 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7557 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7558 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7559
7560 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7561 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7562 }
7563 else
7564 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7565 }
7566 /*
7567 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7568 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7569 */
7570 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7571 && !pVCpu->hm.s.fSingleInstruction)
7572 {
7573 Assert(!DBGFIsStepping(pVCpu));
7574 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7575 AssertRC(rc);
7576 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7577 if ( !pVCpu->hm.s.Event.fPending
7578 && !fBlockInt
7579 && !fBlockSti
7580 && !fBlockMovSS)
7581 {
7582 uint8_t u8Interrupt;
7583 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7584 if (RT_SUCCESS(rc))
7585 {
7586 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7587 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7588 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7589
7590 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7591 }
7592 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7593 {
7594 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7595 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7596 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7597
7598 /*
7599 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7600 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7601 * need to re-set this force-flag here.
7602 */
7603 }
7604 else
7605 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7606 }
7607 else
7608 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7609 }
7610
7611 return uIntrState;
7612}
7613
7614
7615/**
7616 * Sets a pending-debug exception to be delivered to the guest if the guest is
7617 * single-stepping in the VMCS.
7618 *
7619 * @param pVCpu The cross context virtual CPU structure.
7620 */
7621DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7622{
7623 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7624 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7625 AssertRC(rc);
7626}
7627
7628
7629/**
7630 * Injects any pending events into the guest if the guest is in a state to
7631 * receive them.
7632 *
7633 * @returns Strict VBox status code (i.e. informational status codes too).
7634 * @param pVCpu The cross context virtual CPU structure.
7635 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7636 * out-of-sync. Make sure to update the required fields
7637 * before using them.
7638 * @param uIntrState The VT-x guest-interruptibility state.
7639 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7640 * return VINF_EM_DBG_STEPPED if the event was
7641 * dispatched directly.
7642 */
7643static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7644{
7645 HMVMX_ASSERT_PREEMPT_SAFE();
7646 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7647
7648 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7649 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7650
7651 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7652 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7653 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7654 Assert(!TRPMHasTrap(pVCpu));
7655
7656 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7657 if (pVCpu->hm.s.Event.fPending)
7658 {
7659 /*
7660 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7661 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7662 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7663 *
7664 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7665 */
7666 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7667#ifdef VBOX_STRICT
7668 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7669 {
7670 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7671 Assert(!fBlockInt);
7672 Assert(!fBlockSti);
7673 Assert(!fBlockMovSS);
7674 }
7675 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7676 {
7677 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7678 Assert(!fBlockSti);
7679 Assert(!fBlockMovSS);
7680 Assert(!fBlockNmi);
7681 }
7682#endif
7683 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7684 (uint8_t)uIntType));
7685 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7686 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7687 fStepping, &uIntrState);
7688 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7689
7690 /* Update the interruptibility-state as it could have been changed by
7691 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7692 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7693 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7694
7695 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7696 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7697 else
7698 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7699 }
7700
7701 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7702 if ( fBlockSti
7703 || fBlockMovSS)
7704 {
7705 if (!pVCpu->hm.s.fSingleInstruction)
7706 {
7707 /*
7708 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7709 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7710 * See Intel spec. 27.3.4 "Saving Non-Register State".
7711 */
7712 Assert(!DBGFIsStepping(pVCpu));
7713 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7714 AssertRCReturn(rc2, rc2);
7715 if (pMixedCtx->eflags.Bits.u1TF)
7716 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7717 }
7718 else if (pMixedCtx->eflags.Bits.u1TF)
7719 {
7720 /*
7721 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7722 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7723 */
7724 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7725 uIntrState = 0;
7726 }
7727 }
7728
7729 /*
7730 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7731 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7732 */
7733 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7734 AssertRC(rc2);
7735
7736 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7737 NOREF(fBlockMovSS); NOREF(fBlockSti);
7738 return rcStrict;
7739}
7740
7741
7742/**
7743 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7744 *
7745 * @param pVCpu The cross context virtual CPU structure.
7746 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7747 * out-of-sync. Make sure to update the required fields
7748 * before using them.
7749 */
7750DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7751{
7752 NOREF(pMixedCtx);
7753 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7754 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7755}
7756
7757
7758/**
7759 * Injects a double-fault (\#DF) exception into the VM.
7760 *
7761 * @returns Strict VBox status code (i.e. informational status codes too).
7762 * @param pVCpu The cross context virtual CPU structure.
7763 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7764 * out-of-sync. Make sure to update the required fields
7765 * before using them.
7766 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7767 * and should return VINF_EM_DBG_STEPPED if the event
7768 * is injected directly (register modified by us, not
7769 * by hardware on VM-entry).
7770 * @param puIntrState Pointer to the current guest interruptibility-state.
7771 * This interruptibility-state will be updated if
7772 * necessary. This cannot not be NULL.
7773 */
7774DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7775{
7776 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7777 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7778 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7779 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7780 fStepping, puIntrState);
7781}
7782
7783
7784/**
7785 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7786 *
7787 * @param pVCpu The cross context virtual CPU structure.
7788 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7789 * out-of-sync. Make sure to update the required fields
7790 * before using them.
7791 */
7792DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7793{
7794 NOREF(pMixedCtx);
7795 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7796 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7797 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7798}
7799
7800
7801/**
7802 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7803 *
7804 * @param pVCpu The cross context virtual CPU structure.
7805 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7806 * out-of-sync. Make sure to update the required fields
7807 * before using them.
7808 * @param cbInstr The value of RIP that is to be pushed on the guest
7809 * stack.
7810 */
7811DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7812{
7813 NOREF(pMixedCtx);
7814 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7815 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7816 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7817}
7818
7819
7820/**
7821 * Injects a general-protection (\#GP) fault into the VM.
7822 *
7823 * @returns Strict VBox status code (i.e. informational status codes too).
7824 * @param pVCpu The cross context virtual CPU structure.
7825 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7826 * out-of-sync. Make sure to update the required fields
7827 * before using them.
7828 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7829 * mode, i.e. in real-mode it's not valid).
7830 * @param u32ErrorCode The error code associated with the \#GP.
7831 * @param fStepping Whether we're running in
7832 * hmR0VmxRunGuestCodeStep() and should return
7833 * VINF_EM_DBG_STEPPED if the event is injected
7834 * directly (register modified by us, not by
7835 * hardware on VM-entry).
7836 * @param puIntrState Pointer to the current guest interruptibility-state.
7837 * This interruptibility-state will be updated if
7838 * necessary. This cannot not be NULL.
7839 */
7840DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7841 bool fStepping, uint32_t *puIntrState)
7842{
7843 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7844 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7845 if (fErrorCodeValid)
7846 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7847 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7848 fStepping, puIntrState);
7849}
7850
7851
7852#if 0 /* unused */
7853/**
7854 * Sets a general-protection (\#GP) exception as pending-for-injection into the
7855 * VM.
7856 *
7857 * @param pVCpu The cross context virtual CPU structure.
7858 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7859 * out-of-sync. Make sure to update the required fields
7860 * before using them.
7861 * @param u32ErrorCode The error code associated with the \#GP.
7862 */
7863DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7864{
7865 NOREF(pMixedCtx);
7866 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7867 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7868 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7869 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7870}
7871#endif /* unused */
7872
7873
7874/**
7875 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7876 *
7877 * @param pVCpu The cross context virtual CPU structure.
7878 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7879 * out-of-sync. Make sure to update the required fields
7880 * before using them.
7881 * @param uVector The software interrupt vector number.
7882 * @param cbInstr The value of RIP that is to be pushed on the guest
7883 * stack.
7884 */
7885DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7886{
7887 NOREF(pMixedCtx);
7888 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7889 if ( uVector == X86_XCPT_BP
7890 || uVector == X86_XCPT_OF)
7891 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7892 else
7893 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7894 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7895}
7896
7897
7898/**
7899 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7900 * stack.
7901 *
7902 * @returns Strict VBox status code (i.e. informational status codes too).
7903 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7904 * @param pVM The cross context VM structure.
7905 * @param pMixedCtx Pointer to the guest-CPU context.
7906 * @param uValue The value to push to the guest stack.
7907 */
7908DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7909{
7910 /*
7911 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7912 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7913 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7914 */
7915 if (pMixedCtx->sp == 1)
7916 return VINF_EM_RESET;
7917 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7918 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7919 AssertRC(rc);
7920 return rc;
7921}
7922
7923
7924/**
7925 * Injects an event into the guest upon VM-entry by updating the relevant fields
7926 * in the VM-entry area in the VMCS.
7927 *
7928 * @returns Strict VBox status code (i.e. informational status codes too).
7929 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7930 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7931 *
7932 * @param pVCpu The cross context virtual CPU structure.
7933 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7934 * be out-of-sync. Make sure to update the required
7935 * fields before using them.
7936 * @param u64IntInfo The VM-entry interruption-information field.
7937 * @param cbInstr The VM-entry instruction length in bytes (for
7938 * software interrupts, exceptions and privileged
7939 * software exceptions).
7940 * @param u32ErrCode The VM-entry exception error code.
7941 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7942 * @param puIntrState Pointer to the current guest interruptibility-state.
7943 * This interruptibility-state will be updated if
7944 * necessary. This cannot not be NULL.
7945 * @param fStepping Whether we're running in
7946 * hmR0VmxRunGuestCodeStep() and should return
7947 * VINF_EM_DBG_STEPPED if the event is injected
7948 * directly (register modified by us, not by
7949 * hardware on VM-entry).
7950 *
7951 * @remarks Requires CR0!
7952 */
7953static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7954 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
7955 uint32_t *puIntrState)
7956{
7957 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7958 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7959 Assert(puIntrState);
7960 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7961
7962 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7963 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7964
7965#ifdef VBOX_STRICT
7966 /* Validate the error-code-valid bit for hardware exceptions. */
7967 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7968 {
7969 switch (uVector)
7970 {
7971 case X86_XCPT_PF:
7972 case X86_XCPT_DF:
7973 case X86_XCPT_TS:
7974 case X86_XCPT_NP:
7975 case X86_XCPT_SS:
7976 case X86_XCPT_GP:
7977 case X86_XCPT_AC:
7978 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7979 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7980 /* fallthru */
7981 default:
7982 break;
7983 }
7984 }
7985#endif
7986
7987 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7988 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7989 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7990
7991 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7992
7993 /* We require CR0 to check if the guest is in real-mode. */
7994 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7995 AssertRCReturn(rc, rc);
7996
7997 /*
7998 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7999 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
8000 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
8001 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8002 */
8003 if (CPUMIsGuestInRealModeEx(pMixedCtx))
8004 {
8005 PVM pVM = pVCpu->CTX_SUFF(pVM);
8006 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
8007 {
8008 Assert(PDMVmmDevHeapIsEnabled(pVM));
8009 Assert(pVM->hm.s.vmx.pRealModeTSS);
8010
8011 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
8012 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8013 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
8014 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8015 AssertRCReturn(rc, rc);
8016 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
8017
8018 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8019 size_t const cbIdtEntry = sizeof(X86IDTR16);
8020 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
8021 {
8022 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8023 if (uVector == X86_XCPT_DF)
8024 return VINF_EM_RESET;
8025
8026 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
8027 if (uVector == X86_XCPT_GP)
8028 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
8029
8030 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
8031 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
8032 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
8033 fStepping, puIntrState);
8034 }
8035
8036 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8037 uint16_t uGuestIp = pMixedCtx->ip;
8038 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
8039 {
8040 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
8041 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
8042 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8043 }
8044 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
8045 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8046
8047 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8048 X86IDTR16 IdtEntry;
8049 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
8050 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8051 AssertRCReturn(rc, rc);
8052
8053 /* Construct the stack frame for the interrupt/exception handler. */
8054 VBOXSTRICTRC rcStrict;
8055 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
8056 if (rcStrict == VINF_SUCCESS)
8057 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
8058 if (rcStrict == VINF_SUCCESS)
8059 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
8060
8061 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
8062 if (rcStrict == VINF_SUCCESS)
8063 {
8064 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
8065 pMixedCtx->rip = IdtEntry.offSel;
8066 pMixedCtx->cs.Sel = IdtEntry.uSel;
8067 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
8068 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8069 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8070 && uVector == X86_XCPT_PF)
8071 pMixedCtx->cr2 = GCPtrFaultAddress;
8072
8073 /* If any other guest-state bits are changed here, make sure to update
8074 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
8075 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
8076 | HM_CHANGED_GUEST_RIP
8077 | HM_CHANGED_GUEST_RFLAGS
8078 | HM_CHANGED_GUEST_RSP);
8079
8080 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8081 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8082 {
8083 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8084 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8085 Log4(("Clearing inhibition due to STI.\n"));
8086 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
8087 }
8088 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8089 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
8090
8091 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8092 it, if we are returning to ring-3 before executing guest code. */
8093 pVCpu->hm.s.Event.fPending = false;
8094
8095 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8096 if (fStepping)
8097 rcStrict = VINF_EM_DBG_STEPPED;
8098 }
8099 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8100 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8101 return rcStrict;
8102 }
8103
8104 /*
8105 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8106 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8107 */
8108 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8109 }
8110
8111 /* Validate. */
8112 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8113 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8114 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8115
8116 /* Inject. */
8117 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8118 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8119 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8120 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8121
8122 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8123 && uVector == X86_XCPT_PF)
8124 pMixedCtx->cr2 = GCPtrFaultAddress;
8125
8126 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8127 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8128
8129 AssertRCReturn(rc, rc);
8130 return VINF_SUCCESS;
8131}
8132
8133
8134/**
8135 * Clears the interrupt-window exiting control in the VMCS and if necessary
8136 * clears the current event in the VMCS as well.
8137 *
8138 * @returns VBox status code.
8139 * @param pVCpu The cross context virtual CPU structure.
8140 *
8141 * @remarks Use this function only to clear events that have not yet been
8142 * delivered to the guest but are injected in the VMCS!
8143 * @remarks No-long-jump zone!!!
8144 */
8145static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8146{
8147 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8148
8149 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8150 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8151
8152 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8153 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8154}
8155
8156
8157/**
8158 * Enters the VT-x session.
8159 *
8160 * @returns VBox status code.
8161 * @param pVM The cross context VM structure.
8162 * @param pVCpu The cross context virtual CPU structure.
8163 * @param pCpu Pointer to the CPU info struct.
8164 */
8165VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8166{
8167 AssertPtr(pVM);
8168 AssertPtr(pVCpu);
8169 Assert(pVM->hm.s.vmx.fSupported);
8170 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8171 NOREF(pCpu); NOREF(pVM);
8172
8173 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8174 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8175
8176#ifdef VBOX_STRICT
8177 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8178 RTCCUINTREG uHostCR4 = ASMGetCR4();
8179 if (!(uHostCR4 & X86_CR4_VMXE))
8180 {
8181 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8182 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8183 }
8184#endif
8185
8186 /*
8187 * Load the VCPU's VMCS as the current (and active) one.
8188 */
8189 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8190 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8191 if (RT_FAILURE(rc))
8192 return rc;
8193
8194 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8195 pVCpu->hm.s.fLeaveDone = false;
8196 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8197
8198 return VINF_SUCCESS;
8199}
8200
8201
8202/**
8203 * The thread-context callback (only on platforms which support it).
8204 *
8205 * @param enmEvent The thread-context event.
8206 * @param pVCpu The cross context virtual CPU structure.
8207 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8208 * @thread EMT(pVCpu)
8209 */
8210VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8211{
8212 NOREF(fGlobalInit);
8213
8214 switch (enmEvent)
8215 {
8216 case RTTHREADCTXEVENT_OUT:
8217 {
8218 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8219 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8220 VMCPU_ASSERT_EMT(pVCpu);
8221
8222 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8223
8224 /* No longjmps (logger flushes, locks) in this fragile context. */
8225 VMMRZCallRing3Disable(pVCpu);
8226 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8227
8228 /*
8229 * Restore host-state (FPU, debug etc.)
8230 */
8231 if (!pVCpu->hm.s.fLeaveDone)
8232 {
8233 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8234 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8235 hmR0VmxLeave(pVCpu, pMixedCtx, false /* fSaveGuestState */);
8236 pVCpu->hm.s.fLeaveDone = true;
8237 }
8238
8239 /* Leave HM context, takes care of local init (term). */
8240 int rc = HMR0LeaveCpu(pVCpu);
8241 AssertRC(rc); NOREF(rc);
8242
8243 /* Restore longjmp state. */
8244 VMMRZCallRing3Enable(pVCpu);
8245 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8246 break;
8247 }
8248
8249 case RTTHREADCTXEVENT_IN:
8250 {
8251 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8252 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8253 VMCPU_ASSERT_EMT(pVCpu);
8254
8255 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8256 VMMRZCallRing3Disable(pVCpu);
8257 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8258
8259 /* Initialize the bare minimum state required for HM. This takes care of
8260 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8261 int rc = HMR0EnterCpu(pVCpu);
8262 AssertRC(rc);
8263 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8264
8265 /* Load the active VMCS as the current one. */
8266 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8267 {
8268 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8269 AssertRC(rc); NOREF(rc);
8270 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8271 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8272 }
8273 pVCpu->hm.s.fLeaveDone = false;
8274
8275 /* Restore longjmp state. */
8276 VMMRZCallRing3Enable(pVCpu);
8277 break;
8278 }
8279
8280 default:
8281 break;
8282 }
8283}
8284
8285
8286/**
8287 * Saves the host state in the VMCS host-state.
8288 * Sets up the VM-exit MSR-load area.
8289 *
8290 * The CPU state will be loaded from these fields on every successful VM-exit.
8291 *
8292 * @returns VBox status code.
8293 * @param pVM The cross context VM structure.
8294 * @param pVCpu The cross context virtual CPU structure.
8295 *
8296 * @remarks No-long-jump zone!!!
8297 */
8298static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8299{
8300 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8301
8302 int rc = VINF_SUCCESS;
8303 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8304 {
8305 rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8306 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8307
8308 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8309 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8310
8311 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8312 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8313
8314 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8315 }
8316 return rc;
8317}
8318
8319
8320/**
8321 * Saves the host state in the VMCS host-state.
8322 *
8323 * @returns VBox status code.
8324 * @param pVM The cross context VM structure.
8325 * @param pVCpu The cross context virtual CPU structure.
8326 *
8327 * @remarks No-long-jump zone!!!
8328 */
8329VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8330{
8331 AssertPtr(pVM);
8332 AssertPtr(pVCpu);
8333
8334 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8335
8336 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8337 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8338 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8339 return hmR0VmxSaveHostState(pVM, pVCpu);
8340}
8341
8342
8343/**
8344 * Loads the guest state into the VMCS guest-state area.
8345 *
8346 * The will typically be done before VM-entry when the guest-CPU state and the
8347 * VMCS state may potentially be out of sync.
8348 *
8349 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8350 * VM-entry controls.
8351 * Sets up the appropriate VMX non-root function to execute guest code based on
8352 * the guest CPU mode.
8353 *
8354 * @returns VBox strict status code.
8355 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8356 * without unrestricted guest access and the VMMDev is not presently
8357 * mapped (e.g. EFI32).
8358 *
8359 * @param pVM The cross context VM structure.
8360 * @param pVCpu The cross context virtual CPU structure.
8361 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8362 * out-of-sync. Make sure to update the required fields
8363 * before using them.
8364 *
8365 * @remarks No-long-jump zone!!!
8366 */
8367static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8368{
8369 AssertPtr(pVM);
8370 AssertPtr(pVCpu);
8371 AssertPtr(pMixedCtx);
8372 HMVMX_ASSERT_PREEMPT_SAFE();
8373
8374 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8375
8376 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8377
8378 /* Determine real-on-v86 mode. */
8379 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8380 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8381 && CPUMIsGuestInRealModeEx(pMixedCtx))
8382 {
8383 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8384 }
8385
8386 /*
8387 * Load the guest-state into the VMCS.
8388 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8389 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8390 */
8391 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8392 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8393
8394 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8395 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8396 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8397
8398 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8399 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8400 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8401
8402 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8403 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8404
8405 VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8406 if (rcStrict == VINF_SUCCESS)
8407 { /* likely */ }
8408 else
8409 {
8410 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8411 return rcStrict;
8412 }
8413
8414 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8415 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8416 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8417
8418 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8419 determine we don't have to swap EFER after all. */
8420 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8421 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8422
8423 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8424 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8425
8426 rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8427 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8428
8429 /*
8430 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8431 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8432 */
8433 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8434 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8435
8436 /* Clear any unused and reserved bits. */
8437 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8438
8439 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8440 return rc;
8441}
8442
8443
8444/**
8445 * Loads the state shared between the host and guest into the VMCS.
8446 *
8447 * @param pVM The cross context VM structure.
8448 * @param pVCpu The cross context virtual CPU structure.
8449 * @param pCtx Pointer to the guest-CPU context.
8450 *
8451 * @remarks No-long-jump zone!!!
8452 */
8453static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8454{
8455 NOREF(pVM);
8456
8457 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8458 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8459
8460 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8461 {
8462 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8463 AssertRC(rc);
8464 }
8465
8466 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8467 {
8468 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8469 AssertRC(rc);
8470
8471 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8472 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8473 {
8474 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8475 AssertRC(rc);
8476 }
8477 }
8478
8479 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8480 {
8481 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8482 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8483 }
8484
8485 /* Loading CR0, debug state might have changed intercepts, update VMCS. */
8486 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8487 {
8488 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8489 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8490 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8491 AssertRC(rc);
8492 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8493 }
8494
8495 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8496 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8497}
8498
8499
8500/**
8501 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8502 *
8503 * @returns Strict VBox status code (i.e. informational status codes too).
8504 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8505 * without unrestricted guest access and the VMMDev is not presently
8506 * mapped (e.g. EFI32).
8507 *
8508 * @param pVM The cross context VM structure.
8509 * @param pVCpu The cross context virtual CPU structure.
8510 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8511 * out-of-sync. Make sure to update the required fields
8512 * before using them.
8513 *
8514 * @remarks No-long-jump zone!!!
8515 */
8516static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8517{
8518 HMVMX_ASSERT_PREEMPT_SAFE();
8519 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8520 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8521
8522 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8523#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8524 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8525#endif
8526
8527 /*
8528 * RIP is what changes the most often and hence if it's the only bit needing to be
8529 * updated, we shall handle it early for performance reasons.
8530 */
8531 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8532 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8533 {
8534 rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8535 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8536 { /* likely */}
8537 else
8538 {
8539 AssertMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8540 VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8541 }
8542 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8543 }
8544 else if (HMCPU_CF_VALUE(pVCpu))
8545 {
8546 rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8547 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8548 { /* likely */}
8549 else
8550 {
8551 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8552 ("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8553 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8554 return rcStrict;
8555 }
8556 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8557 }
8558
8559 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8560 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8561 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8562 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8563 return rcStrict;
8564}
8565
8566
8567/**
8568 * Does the preparations before executing guest code in VT-x.
8569 *
8570 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8571 * recompiler/IEM. We must be cautious what we do here regarding committing
8572 * guest-state information into the VMCS assuming we assuredly execute the
8573 * guest in VT-x mode.
8574 *
8575 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8576 * the common-state (TRPM/forceflags), we must undo those changes so that the
8577 * recompiler/IEM can (and should) use them when it resumes guest execution.
8578 * Otherwise such operations must be done when we can no longer exit to ring-3.
8579 *
8580 * @returns Strict VBox status code (i.e. informational status codes too).
8581 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8582 * have been disabled.
8583 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8584 * double-fault into the guest.
8585 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8586 * dispatched directly.
8587 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8588 *
8589 * @param pVM The cross context VM structure.
8590 * @param pVCpu The cross context virtual CPU structure.
8591 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8592 * out-of-sync. Make sure to update the required fields
8593 * before using them.
8594 * @param pVmxTransient Pointer to the VMX transient structure.
8595 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8596 * us ignore some of the reasons for returning to
8597 * ring-3, and return VINF_EM_DBG_STEPPED if event
8598 * dispatching took place.
8599 */
8600static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8601{
8602 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8603
8604#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8605 PGMRZDynMapFlushAutoSet(pVCpu);
8606#endif
8607
8608 /* Check force flag actions that might require us to go back to ring-3. */
8609 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8610 if (rcStrict == VINF_SUCCESS)
8611 { /* FFs doesn't get set all the time. */ }
8612 else
8613 return rcStrict;
8614
8615#ifndef IEM_VERIFICATION_MODE_FULL
8616 /*
8617 * Setup the virtualized-APIC accesses.
8618 *
8619 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8620 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8621 *
8622 * This is the reason we do it here and not in hmR0VmxLoadGuestState().
8623 */
8624 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8625 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8626 && PDMHasApic(pVM))
8627 {
8628 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8629 Assert(u64MsrApicBase);
8630 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8631
8632 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8633
8634 /* Unalias any existing mapping. */
8635 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8636 AssertRCReturn(rc, rc);
8637
8638 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8639 LogRel(("hmR0VmxPreRunGuest: VCPU%u: Mapped HC APIC-access page at %#RGp\n", pVCpu->idCpu, GCPhysApicBase));
8640 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8641 AssertRCReturn(rc, rc);
8642
8643 /* Update the per-VCPU cache of the APIC base MSR. */
8644 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8645 }
8646#endif /* !IEM_VERIFICATION_MODE_FULL */
8647
8648 if (TRPMHasTrap(pVCpu))
8649 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8650 uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8651
8652 /*
8653 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8654 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8655 */
8656 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
8657 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8658 { /* likely */ }
8659 else
8660 {
8661 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8662 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8663 return rcStrict;
8664 }
8665
8666 /*
8667 * No longjmps to ring-3 from this point on!!!
8668 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8669 * This also disables flushing of the R0-logger instance (if any).
8670 */
8671 VMMRZCallRing3Disable(pVCpu);
8672
8673 /*
8674 * Load the guest state bits.
8675 *
8676 * We cannot perform longjmps while loading the guest state because we do not preserve the
8677 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8678 * CPU migration.
8679 *
8680 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8681 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8682 * Hence, loading of the guest state needs to be done -after- injection of events.
8683 */
8684 rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8685 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8686 { /* likely */ }
8687 else
8688 {
8689 VMMRZCallRing3Enable(pVCpu);
8690 return rcStrict;
8691 }
8692
8693 /*
8694 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8695 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8696 *
8697 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8698 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8699 *
8700 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8701 * executing guest code.
8702 */
8703 pVmxTransient->fEFlags = ASMIntDisableFlags();
8704
8705 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8706 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8707 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8708 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8709 {
8710 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8711 {
8712 pVCpu->hm.s.Event.fPending = false;
8713
8714 /*
8715 * We've injected any pending events. This is really the point of no return (to ring-3).
8716 *
8717 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
8718 * returns from this function, so don't enable them here.
8719 */
8720 return VINF_SUCCESS;
8721 }
8722
8723 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8724 rcStrict = VINF_EM_RAW_INTERRUPT;
8725 }
8726 else
8727 {
8728 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8729 rcStrict = VINF_EM_RAW_TO_R3;
8730 }
8731
8732 ASMSetFlags(pVmxTransient->fEFlags);
8733 VMMRZCallRing3Enable(pVCpu);
8734
8735 return rcStrict;
8736}
8737
8738
8739/**
8740 * Prepares to run guest code in VT-x and we've committed to doing so. This
8741 * means there is no backing out to ring-3 or anywhere else at this
8742 * point.
8743 *
8744 * @param pVM The cross context VM structure.
8745 * @param pVCpu The cross context virtual CPU structure.
8746 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8747 * out-of-sync. Make sure to update the required fields
8748 * before using them.
8749 * @param pVmxTransient Pointer to the VMX transient structure.
8750 *
8751 * @remarks Called with preemption disabled.
8752 * @remarks No-long-jump zone!!!
8753 */
8754static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8755{
8756 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8757 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8758 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8759
8760 /*
8761 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8762 */
8763 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8764 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8765
8766#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8767 if (!CPUMIsGuestFPUStateActive(pVCpu))
8768 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8769 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8770 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8771#endif
8772
8773 if ( pVCpu->hm.s.fPreloadGuestFpu
8774 && !CPUMIsGuestFPUStateActive(pVCpu))
8775 {
8776 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8777 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8778 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8779 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8780 }
8781
8782 /*
8783 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8784 */
8785 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8786 && pVCpu->hm.s.vmx.cMsrs > 0)
8787 {
8788 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8789 }
8790
8791 /*
8792 * Load the host state bits as we may've been preempted (only happens when
8793 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8794 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8795 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8796 * See @bugref{8432}.
8797 */
8798 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8799 {
8800 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8801 AssertRC(rc);
8802 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
8803 }
8804 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8805
8806 /*
8807 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8808 */
8809 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8810 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8811 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8812
8813 /* Store status of the shared guest-host state at the time of VM-entry. */
8814#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8815 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8816 {
8817 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8818 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8819 }
8820 else
8821#endif
8822 {
8823 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8824 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8825 }
8826 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8827
8828 /*
8829 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8830 */
8831 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8832 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8833
8834 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8835 RTCPUID idCurrentCpu = pCpu->idCpu;
8836 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8837 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8838 {
8839 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8840 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8841 }
8842
8843 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8844 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8845 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8846 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8847
8848 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8849
8850 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8851 to start executing. */
8852
8853 /*
8854 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8855 */
8856 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8857 {
8858 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8859 {
8860 bool fMsrUpdated;
8861 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8862 AssertRC(rc2);
8863 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8864
8865 rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8866 &fMsrUpdated);
8867 AssertRC(rc2);
8868 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8869
8870 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8871 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8872 }
8873 else
8874 {
8875 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8876 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8877 }
8878 }
8879
8880#ifdef VBOX_STRICT
8881 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8882 hmR0VmxCheckHostEferMsr(pVCpu);
8883 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8884#endif
8885#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8886 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8887 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8888 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8889#endif
8890}
8891
8892
8893/**
8894 * Performs some essential restoration of state after running guest code in
8895 * VT-x.
8896 *
8897 * @param pVM The cross context VM structure.
8898 * @param pVCpu The cross context virtual CPU structure.
8899 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8900 * out-of-sync. Make sure to update the required fields
8901 * before using them.
8902 * @param pVmxTransient Pointer to the VMX transient structure.
8903 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8904 *
8905 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8906 *
8907 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8908 * unconditionally when it is safe to do so.
8909 */
8910static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8911{
8912 NOREF(pVM);
8913
8914 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8915
8916 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8917 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8918 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8919 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8920 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8921 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8922
8923 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8924 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
8925
8926 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8927 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8928 Assert(!ASMIntAreEnabled());
8929 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8930
8931#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8932 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVM, pVCpu))
8933 {
8934 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8935 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8936 }
8937#endif
8938
8939#if HC_ARCH_BITS == 64
8940 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8941#endif
8942#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8943 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8944 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8945 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8946#else
8947 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8948#endif
8949#ifdef VBOX_STRICT
8950 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8951#endif
8952 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8953 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8954
8955 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8956 uint32_t uExitReason;
8957 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8958 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8959 AssertRC(rc);
8960 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8961 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8962
8963 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8964 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8965 {
8966 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8967 pVmxTransient->fVMEntryFailed));
8968 return;
8969 }
8970
8971 /*
8972 * Update the VM-exit history array here even if the VM-entry failed due to:
8973 * - Invalid guest state.
8974 * - MSR loading.
8975 * - Machine-check event.
8976 *
8977 * In any of the above cases we will still have a "valid" VM-exit reason
8978 * despite @a fVMEntryFailed being false.
8979 *
8980 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8981 */
8982 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8983
8984 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8985 {
8986 /** @todo We can optimize this by only syncing with our force-flags when
8987 * really needed and keeping the VMCS state as it is for most
8988 * VM-exits. */
8989 /* Update the guest interruptibility-state from the VMCS. */
8990 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8991
8992#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8993 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8994 AssertRC(rc);
8995#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8996 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8997 AssertRC(rc);
8998#endif
8999
9000 /*
9001 * Sync the TPR shadow with our APIC state.
9002 */
9003 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9004 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
9005 {
9006 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
9007 AssertRC(rc);
9008 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9009 }
9010 }
9011}
9012
9013
9014/**
9015 * Runs the guest code using VT-x the normal way.
9016 *
9017 * @returns VBox status code.
9018 * @param pVM The cross context VM structure.
9019 * @param pVCpu The cross context virtual CPU structure.
9020 * @param pCtx Pointer to the guest-CPU context.
9021 *
9022 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
9023 */
9024static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9025{
9026 VMXTRANSIENT VmxTransient;
9027 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9028 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9029 uint32_t cLoops = 0;
9030
9031 for (;; cLoops++)
9032 {
9033 Assert(!HMR0SuspendPending());
9034 HMVMX_ASSERT_CPU_SAFE();
9035
9036 /* Preparatory work for running guest code, this may force us to return
9037 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
9038 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9039 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
9040 if (rcStrict != VINF_SUCCESS)
9041 break;
9042
9043 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9044 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9045 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9046
9047 /* Restore any residual host-state and save any bits shared between host
9048 and guest into the guest-CPU state. Re-enables interrupts! */
9049 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
9050
9051 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9052 if (RT_SUCCESS(rcRun))
9053 { /* very likely */ }
9054 else
9055 {
9056 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9057 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9058 return rcRun;
9059 }
9060
9061 /* Profile the VM-exit. */
9062 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9063 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9064 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9065 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9066 HMVMX_START_EXIT_DISPATCH_PROF();
9067
9068 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9069
9070 /* Handle the VM-exit. */
9071#ifdef HMVMX_USE_FUNCTION_TABLE
9072 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
9073#else
9074 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
9075#endif
9076 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9077 if (rcStrict == VINF_SUCCESS)
9078 {
9079 if (cLoops <= pVM->hm.s.cMaxResumeLoops)
9080 continue; /* likely */
9081 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9082 rcStrict = VINF_EM_RAW_INTERRUPT;
9083 }
9084 break;
9085 }
9086
9087 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9088 return rcStrict;
9089}
9090
9091
9092
9093/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
9094 * probes.
9095 *
9096 * The following few functions and associated structure contains the bloat
9097 * necessary for providing detailed debug events and dtrace probes as well as
9098 * reliable host side single stepping. This works on the principle of
9099 * "subclassing" the normal execution loop and workers. We replace the loop
9100 * method completely and override selected helpers to add necessary adjustments
9101 * to their core operation.
9102 *
9103 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
9104 * any performance for debug and analysis features.
9105 *
9106 * @{
9107 */
9108
9109/**
9110 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
9111 * the debug run loop.
9112 */
9113typedef struct VMXRUNDBGSTATE
9114{
9115 /** The RIP we started executing at. This is for detecting that we stepped. */
9116 uint64_t uRipStart;
9117 /** The CS we started executing with. */
9118 uint16_t uCsStart;
9119
9120 /** Whether we've actually modified the 1st execution control field. */
9121 bool fModifiedProcCtls : 1;
9122 /** Whether we've actually modified the 2nd execution control field. */
9123 bool fModifiedProcCtls2 : 1;
9124 /** Whether we've actually modified the exception bitmap. */
9125 bool fModifiedXcptBitmap : 1;
9126
9127 /** We desire the modified the CR0 mask to be cleared. */
9128 bool fClearCr0Mask : 1;
9129 /** We desire the modified the CR4 mask to be cleared. */
9130 bool fClearCr4Mask : 1;
9131 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9132 uint32_t fCpe1Extra;
9133 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9134 uint32_t fCpe1Unwanted;
9135 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9136 uint32_t fCpe2Extra;
9137 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9138 uint32_t bmXcptExtra;
9139 /** The sequence number of the Dtrace provider settings the state was
9140 * configured against. */
9141 uint32_t uDtraceSettingsSeqNo;
9142 /** VM-exits to check (one bit per VM-exit). */
9143 uint32_t bmExitsToCheck[3];
9144
9145 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9146 uint32_t fProcCtlsInitial;
9147 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9148 uint32_t fProcCtls2Initial;
9149 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9150 uint32_t bmXcptInitial;
9151} VMXRUNDBGSTATE;
9152AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9153typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9154
9155
9156/**
9157 * Initializes the VMXRUNDBGSTATE structure.
9158 *
9159 * @param pVCpu The cross context virtual CPU structure of the
9160 * calling EMT.
9161 * @param pCtx The CPU register context to go with @a pVCpu.
9162 * @param pDbgState The structure to initialize.
9163 */
9164DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9165{
9166 pDbgState->uRipStart = pCtx->rip;
9167 pDbgState->uCsStart = pCtx->cs.Sel;
9168
9169 pDbgState->fModifiedProcCtls = false;
9170 pDbgState->fModifiedProcCtls2 = false;
9171 pDbgState->fModifiedXcptBitmap = false;
9172 pDbgState->fClearCr0Mask = false;
9173 pDbgState->fClearCr4Mask = false;
9174 pDbgState->fCpe1Extra = 0;
9175 pDbgState->fCpe1Unwanted = 0;
9176 pDbgState->fCpe2Extra = 0;
9177 pDbgState->bmXcptExtra = 0;
9178 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9179 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9180 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9181}
9182
9183
9184/**
9185 * Updates the VMSC fields with changes requested by @a pDbgState.
9186 *
9187 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9188 * immediately before executing guest code, i.e. when interrupts are disabled.
9189 * We don't check status codes here as we cannot easily assert or return in the
9190 * latter case.
9191 *
9192 * @param pVCpu The cross context virtual CPU structure.
9193 * @param pDbgState The debug state.
9194 */
9195DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9196{
9197 /*
9198 * Ensure desired flags in VMCS control fields are set.
9199 * (Ignoring write failure here, as we're committed and it's just debug extras.)
9200 *
9201 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9202 * there should be no stale data in pCtx at this point.
9203 */
9204 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9205 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9206 {
9207 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
9208 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9209 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9210 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9211 pDbgState->fModifiedProcCtls = true;
9212 }
9213
9214 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9215 {
9216 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
9217 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9218 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9219 pDbgState->fModifiedProcCtls2 = true;
9220 }
9221
9222 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9223 {
9224 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
9225 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9226 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9227 pDbgState->fModifiedXcptBitmap = true;
9228 }
9229
9230 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9231 {
9232 pVCpu->hm.s.vmx.u32CR0Mask = 0;
9233 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9234 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9235 }
9236
9237 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9238 {
9239 pVCpu->hm.s.vmx.u32CR4Mask = 0;
9240 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9241 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9242 }
9243}
9244
9245
9246DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9247{
9248 /*
9249 * Restore VM-exit control settings as we may not reenter this function the
9250 * next time around.
9251 */
9252 /* We reload the initial value, trigger what we can of recalculations the
9253 next time around. From the looks of things, that's all that's required atm. */
9254 if (pDbgState->fModifiedProcCtls)
9255 {
9256 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9257 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9258 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9259 AssertRCReturn(rc2, rc2);
9260 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9261 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0 | HM_CHANGED_GUEST_DEBUG);
9262 }
9263
9264 /* We're currently the only ones messing with this one, so just restore the
9265 cached value and reload the field. */
9266 if ( pDbgState->fModifiedProcCtls2
9267 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9268 {
9269 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9270 AssertRCReturn(rc2, rc2);
9271 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9272 }
9273
9274 /* If we've modified the exception bitmap, we restore it and trigger
9275 reloading and partial recalculation the next time around. */
9276 if (pDbgState->fModifiedXcptBitmap)
9277 {
9278 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9279 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS | HM_CHANGED_GUEST_CR0);
9280 }
9281
9282 /* We assume hmR0VmxLoadSharedCR0 will recalculate and load the CR0 mask. */
9283 if (pDbgState->fClearCr0Mask)
9284 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9285
9286 /* We assume hmR0VmxLoadGuestCR3AndCR4 will recalculate and load the CR4 mask. */
9287 if (pDbgState->fClearCr4Mask)
9288 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9289
9290 return rcStrict;
9291}
9292
9293
9294/**
9295 * Configures VM-exit controls for current DBGF and DTrace settings.
9296 *
9297 * This updates @a pDbgState and the VMCS execution control fields to reflect
9298 * the necessary VM-exits demanded by DBGF and DTrace.
9299 *
9300 * @param pVM The cross context VM structure.
9301 * @param pVCpu The cross context virtual CPU structure.
9302 * @param pCtx Pointer to the guest-CPU context.
9303 * @param pDbgState The debug state.
9304 * @param pVmxTransient Pointer to the VMX transient structure. May update
9305 * fUpdateTscOffsettingAndPreemptTimer.
9306 */
9307static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9308 PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9309{
9310 /*
9311 * Take down the dtrace serial number so we can spot changes.
9312 */
9313 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9314 ASMCompilerBarrier();
9315
9316 /*
9317 * We'll rebuild most of the middle block of data members (holding the
9318 * current settings) as we go along here, so start by clearing it all.
9319 */
9320 pDbgState->bmXcptExtra = 0;
9321 pDbgState->fCpe1Extra = 0;
9322 pDbgState->fCpe1Unwanted = 0;
9323 pDbgState->fCpe2Extra = 0;
9324 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9325 pDbgState->bmExitsToCheck[i] = 0;
9326
9327 /*
9328 * Software interrupts (INT XXh) - no idea how to trigger these...
9329 */
9330 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9331 || VBOXVMM_INT_SOFTWARE_ENABLED())
9332 {
9333 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9334 }
9335
9336 /*
9337 * INT3 breakpoints - triggered by #BP exceptions.
9338 */
9339 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9340 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9341
9342 /*
9343 * Exception bitmap and XCPT events+probes.
9344 */
9345 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9346 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9347 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9348
9349 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9350 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9351 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9352 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9353 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9354 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9355 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9356 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9357 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9358 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9359 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9360 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9361 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9362 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9363 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9364 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9365 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9366 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9367
9368 if (pDbgState->bmXcptExtra)
9369 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9370
9371 /*
9372 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9373 *
9374 * Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9375 * So, when adding/changing/removing please don't forget to update it.
9376 *
9377 * Some of the macros are picking up local variables to save horizontal space,
9378 * (being able to see it in a table is the lesser evil here).
9379 */
9380#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9381 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9382 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9383#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9384 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9385 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9386 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9387 } else do { } while (0)
9388#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9389 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9390 { \
9391 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9392 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9393 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9394 } else do { } while (0)
9395#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9396 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9397 { \
9398 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9399 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9400 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9401 } else do { } while (0)
9402#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9403 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9404 { \
9405 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9406 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9407 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9408 } else do { } while (0)
9409
9410 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9411 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9412 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9413 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9414 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9415
9416 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9417 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9418 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9419 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9420 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9421 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9422 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9423 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9424 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9425 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9426 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9427 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9428 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9429 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9430 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9431 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9432 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9433 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9434 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9435 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9436 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9437 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9438 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9439 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9440 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9441 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9442 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9443 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9444 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9445 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9446 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9447 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9448 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9449 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9450 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9451 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9452
9453 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9454 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9455 {
9456 int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9457 rc2 |= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9458 rc2 |= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9459 AssertRC(rc2);
9460
9461#if 0 /** @todo fix me */
9462 pDbgState->fClearCr0Mask = true;
9463 pDbgState->fClearCr4Mask = true;
9464#endif
9465 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9466 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9467 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9468 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9469 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9470 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9471 require clearing here and in the loop if we start using it. */
9472 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9473 }
9474 else
9475 {
9476 if (pDbgState->fClearCr0Mask)
9477 {
9478 pDbgState->fClearCr0Mask = false;
9479 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9480 }
9481 if (pDbgState->fClearCr4Mask)
9482 {
9483 pDbgState->fClearCr4Mask = false;
9484 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9485 }
9486 }
9487 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9488 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9489
9490 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9491 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9492 {
9493 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9494 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9495 }
9496 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9497 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9498
9499 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9500 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9501 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9502 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9503 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9504 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9505 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9506 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9507#if 0 /** @todo too slow, fix handler. */
9508 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9509#endif
9510 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9511
9512 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9513 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9514 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9515 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9516 {
9517 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9518 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9519 }
9520 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9521 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9522 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9523 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9524
9525 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9526 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9527 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9528 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9529 {
9530 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9531 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9532 }
9533 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9534 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9535 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9536 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9537
9538 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9539 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9540 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9541 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9542 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9543 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9544 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9545 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9546 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9547 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9548 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9549 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9550 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9551 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9552 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9553 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9554 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9555 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9556 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9557 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9558 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9559 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9560
9561#undef IS_EITHER_ENABLED
9562#undef SET_ONLY_XBM_IF_EITHER_EN
9563#undef SET_CPE1_XBM_IF_EITHER_EN
9564#undef SET_CPEU_XBM_IF_EITHER_EN
9565#undef SET_CPE2_XBM_IF_EITHER_EN
9566
9567 /*
9568 * Sanitize the control stuff.
9569 */
9570 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9571 if (pDbgState->fCpe2Extra)
9572 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9573 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9574 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9575 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9576 {
9577 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9578 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9579 }
9580
9581 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9582 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9583 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9584 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9585}
9586
9587
9588/**
9589 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9590 * appropriate.
9591 *
9592 * The caller has checked the VM-exit against the
9593 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9594 * already, so we don't have to do that either.
9595 *
9596 * @returns Strict VBox status code (i.e. informational status codes too).
9597 * @param pVM The cross context VM structure.
9598 * @param pVCpu The cross context virtual CPU structure.
9599 * @param pMixedCtx Pointer to the guest-CPU context.
9600 * @param pVmxTransient Pointer to the VMX-transient structure.
9601 * @param uExitReason The VM-exit reason.
9602 *
9603 * @remarks The name of this function is displayed by dtrace, so keep it short
9604 * and to the point. No longer than 33 chars long, please.
9605 */
9606static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9607 PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9608{
9609 /*
9610 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9611 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9612 *
9613 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9614 * does. Must add/change/remove both places. Same ordering, please.
9615 *
9616 * Added/removed events must also be reflected in the next section
9617 * where we dispatch dtrace events.
9618 */
9619 bool fDtrace1 = false;
9620 bool fDtrace2 = false;
9621 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9622 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9623 uint32_t uEventArg = 0;
9624#define SET_EXIT(a_EventSubName) \
9625 do { \
9626 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9627 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9628 } while (0)
9629#define SET_BOTH(a_EventSubName) \
9630 do { \
9631 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9632 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9633 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9634 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9635 } while (0)
9636 switch (uExitReason)
9637 {
9638 case VMX_EXIT_MTF:
9639 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9640
9641 case VMX_EXIT_XCPT_OR_NMI:
9642 {
9643 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9644 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9645 {
9646 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9647 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9648 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9649 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9650 {
9651 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9652 {
9653 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9654 uEventArg = pVmxTransient->uExitIntErrorCode;
9655 }
9656 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9657 switch (enmEvent1)
9658 {
9659 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9660 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9661 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9662 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9663 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9664 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9665 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9666 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9667 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9668 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9669 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9670 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9671 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9672 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9673 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9674 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9675 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9676 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9677 default: break;
9678 }
9679 }
9680 else
9681 AssertFailed();
9682 break;
9683
9684 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9685 uEventArg = idxVector;
9686 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9687 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9688 break;
9689 }
9690 break;
9691 }
9692
9693 case VMX_EXIT_TRIPLE_FAULT:
9694 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9695 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9696 break;
9697 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9698 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9699 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9700 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9701 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9702
9703 /* Instruction specific VM-exits: */
9704 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9705 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9706 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9707 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9708 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9709 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9710 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9711 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9712 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9713 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9714 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9715 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9716 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9717 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9718 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9719 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9720 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9721 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9722 case VMX_EXIT_MOV_CRX:
9723 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9724/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
9725* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
9726 if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
9727 == VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
9728 SET_BOTH(CRX_READ);
9729 else
9730 SET_BOTH(CRX_WRITE);
9731 uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
9732 break;
9733 case VMX_EXIT_MOV_DRX:
9734 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9735 if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
9736 == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
9737 SET_BOTH(DRX_READ);
9738 else
9739 SET_BOTH(DRX_WRITE);
9740 uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
9741 break;
9742 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9743 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9744 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9745 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9746 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9747 case VMX_EXIT_XDTR_ACCESS:
9748 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9749 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9750 {
9751 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9752 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9753 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9754 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9755 }
9756 break;
9757
9758 case VMX_EXIT_TR_ACCESS:
9759 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9760 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9761 {
9762 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9763 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9764 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9765 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9766 }
9767 break;
9768
9769 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9770 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9771 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9772 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9773 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9774 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9775 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9776 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9777 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9778 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9779 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9780
9781 /* Events that aren't relevant at this point. */
9782 case VMX_EXIT_EXT_INT:
9783 case VMX_EXIT_INT_WINDOW:
9784 case VMX_EXIT_NMI_WINDOW:
9785 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9786 case VMX_EXIT_PREEMPT_TIMER:
9787 case VMX_EXIT_IO_INSTR:
9788 break;
9789
9790 /* Errors and unexpected events. */
9791 case VMX_EXIT_INIT_SIGNAL:
9792 case VMX_EXIT_SIPI:
9793 case VMX_EXIT_IO_SMI:
9794 case VMX_EXIT_SMI:
9795 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9796 case VMX_EXIT_ERR_MSR_LOAD:
9797 case VMX_EXIT_ERR_MACHINE_CHECK:
9798 break;
9799
9800 default:
9801 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9802 break;
9803 }
9804#undef SET_BOTH
9805#undef SET_EXIT
9806
9807 /*
9808 * Dtrace tracepoints go first. We do them here at once so we don't
9809 * have to copy the guest state saving and stuff a few dozen times.
9810 * Down side is that we've got to repeat the switch, though this time
9811 * we use enmEvent since the probes are a subset of what DBGF does.
9812 */
9813 if (fDtrace1 || fDtrace2)
9814 {
9815 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9816 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9817 switch (enmEvent1)
9818 {
9819 /** @todo consider which extra parameters would be helpful for each probe. */
9820 case DBGFEVENT_END: break;
9821 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9822 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9823 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9824 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9825 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9826 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9827 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9828 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9829 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9830 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9831 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9832 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9833 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9834 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9835 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9836 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9837 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9838 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9839 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9840 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9841 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9842 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9843 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9844 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9845 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9846 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9847 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9848 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9849 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9850 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9851 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9852 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9853 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9854 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9855 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9856 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9857 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9858 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9859 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9860 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9861 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9862 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9863 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9864 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9865 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9866 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9867 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9868 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9869 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9870 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9871 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9872 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9873 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9874 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9875 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9876 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9877 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9878 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9879 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9880 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9881 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9882 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9883 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9884 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9885 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9886 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9887 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9888 }
9889 switch (enmEvent2)
9890 {
9891 /** @todo consider which extra parameters would be helpful for each probe. */
9892 case DBGFEVENT_END: break;
9893 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9894 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9895 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9896 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9897 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9898 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9899 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9900 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9901 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9902 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9903 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9904 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9905 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9906 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9907 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9908 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9909 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9910 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9911 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9912 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9913 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9914 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9915 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9916 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9917 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9918 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9919 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9920 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9921 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9922 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9923 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9924 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9925 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9926 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9927 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9928 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9929 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9930 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9931 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9932 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9933 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9934 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9935 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9936 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9937 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9938 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9939 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9940 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9941 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9942 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9943 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9944 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9945 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9946 }
9947 }
9948
9949 /*
9950 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9951 * the DBGF call will do a full check).
9952 *
9953 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9954 * Note! If we have to events, we prioritize the first, i.e. the instruction
9955 * one, in order to avoid event nesting.
9956 */
9957 if ( enmEvent1 != DBGFEVENT_END
9958 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9959 {
9960 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9961 if (rcStrict != VINF_SUCCESS)
9962 return rcStrict;
9963 }
9964 else if ( enmEvent2 != DBGFEVENT_END
9965 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9966 {
9967 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9968 if (rcStrict != VINF_SUCCESS)
9969 return rcStrict;
9970 }
9971
9972 return VINF_SUCCESS;
9973}
9974
9975
9976/**
9977 * Single-stepping VM-exit filtering.
9978 *
9979 * This is preprocessing the VM-exits and deciding whether we've gotten far
9980 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9981 * handling is performed.
9982 *
9983 * @returns Strict VBox status code (i.e. informational status codes too).
9984 * @param pVM The cross context VM structure.
9985 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9986 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9987 * out-of-sync. Make sure to update the required
9988 * fields before using them.
9989 * @param pVmxTransient Pointer to the VMX-transient structure.
9990 * @param uExitReason The VM-exit reason.
9991 * @param pDbgState The debug state.
9992 */
9993DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9994 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9995{
9996 /*
9997 * Expensive (saves context) generic dtrace VM-exit probe.
9998 */
9999 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
10000 { /* more likely */ }
10001 else
10002 {
10003 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10004 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10005 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
10006 }
10007
10008 /*
10009 * Check for host NMI, just to get that out of the way.
10010 */
10011 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
10012 { /* normally likely */ }
10013 else
10014 {
10015 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10016 AssertRCReturn(rc2, rc2);
10017 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10018 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10019 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
10020 }
10021
10022 /*
10023 * Check for single stepping event if we're stepping.
10024 */
10025 if (pVCpu->hm.s.fSingleInstruction)
10026 {
10027 switch (uExitReason)
10028 {
10029 case VMX_EXIT_MTF:
10030 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
10031
10032 /* Various events: */
10033 case VMX_EXIT_XCPT_OR_NMI:
10034 case VMX_EXIT_EXT_INT:
10035 case VMX_EXIT_TRIPLE_FAULT:
10036 case VMX_EXIT_INT_WINDOW:
10037 case VMX_EXIT_NMI_WINDOW:
10038 case VMX_EXIT_TASK_SWITCH:
10039 case VMX_EXIT_TPR_BELOW_THRESHOLD:
10040 case VMX_EXIT_APIC_ACCESS:
10041 case VMX_EXIT_EPT_VIOLATION:
10042 case VMX_EXIT_EPT_MISCONFIG:
10043 case VMX_EXIT_PREEMPT_TIMER:
10044
10045 /* Instruction specific VM-exits: */
10046 case VMX_EXIT_CPUID:
10047 case VMX_EXIT_GETSEC:
10048 case VMX_EXIT_HLT:
10049 case VMX_EXIT_INVD:
10050 case VMX_EXIT_INVLPG:
10051 case VMX_EXIT_RDPMC:
10052 case VMX_EXIT_RDTSC:
10053 case VMX_EXIT_RSM:
10054 case VMX_EXIT_VMCALL:
10055 case VMX_EXIT_VMCLEAR:
10056 case VMX_EXIT_VMLAUNCH:
10057 case VMX_EXIT_VMPTRLD:
10058 case VMX_EXIT_VMPTRST:
10059 case VMX_EXIT_VMREAD:
10060 case VMX_EXIT_VMRESUME:
10061 case VMX_EXIT_VMWRITE:
10062 case VMX_EXIT_VMXOFF:
10063 case VMX_EXIT_VMXON:
10064 case VMX_EXIT_MOV_CRX:
10065 case VMX_EXIT_MOV_DRX:
10066 case VMX_EXIT_IO_INSTR:
10067 case VMX_EXIT_RDMSR:
10068 case VMX_EXIT_WRMSR:
10069 case VMX_EXIT_MWAIT:
10070 case VMX_EXIT_MONITOR:
10071 case VMX_EXIT_PAUSE:
10072 case VMX_EXIT_XDTR_ACCESS:
10073 case VMX_EXIT_TR_ACCESS:
10074 case VMX_EXIT_INVEPT:
10075 case VMX_EXIT_RDTSCP:
10076 case VMX_EXIT_INVVPID:
10077 case VMX_EXIT_WBINVD:
10078 case VMX_EXIT_XSETBV:
10079 case VMX_EXIT_RDRAND:
10080 case VMX_EXIT_INVPCID:
10081 case VMX_EXIT_VMFUNC:
10082 case VMX_EXIT_RDSEED:
10083 case VMX_EXIT_XSAVES:
10084 case VMX_EXIT_XRSTORS:
10085 {
10086 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10087 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10088 AssertRCReturn(rc2, rc2);
10089 if ( pMixedCtx->rip != pDbgState->uRipStart
10090 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
10091 return VINF_EM_DBG_STEPPED;
10092 break;
10093 }
10094
10095 /* Errors and unexpected events: */
10096 case VMX_EXIT_INIT_SIGNAL:
10097 case VMX_EXIT_SIPI:
10098 case VMX_EXIT_IO_SMI:
10099 case VMX_EXIT_SMI:
10100 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10101 case VMX_EXIT_ERR_MSR_LOAD:
10102 case VMX_EXIT_ERR_MACHINE_CHECK:
10103 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
10104 break;
10105
10106 default:
10107 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10108 break;
10109 }
10110 }
10111
10112 /*
10113 * Check for debugger event breakpoints and dtrace probes.
10114 */
10115 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
10116 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
10117 {
10118 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10119 if (rcStrict != VINF_SUCCESS)
10120 return rcStrict;
10121 }
10122
10123 /*
10124 * Normal processing.
10125 */
10126#ifdef HMVMX_USE_FUNCTION_TABLE
10127 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
10128#else
10129 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10130#endif
10131}
10132
10133
10134/**
10135 * Single steps guest code using VT-x.
10136 *
10137 * @returns Strict VBox status code (i.e. informational status codes too).
10138 * @param pVM The cross context VM structure.
10139 * @param pVCpu The cross context virtual CPU structure.
10140 * @param pCtx Pointer to the guest-CPU context.
10141 *
10142 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10143 */
10144static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10145{
10146 VMXTRANSIENT VmxTransient;
10147 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10148
10149 /* Set HMCPU indicators. */
10150 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10151 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
10152 pVCpu->hm.s.fDebugWantRdTscExit = false;
10153 pVCpu->hm.s.fUsingDebugLoop = true;
10154
10155 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10156 VMXRUNDBGSTATE DbgState;
10157 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10158 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10159
10160 /*
10161 * The loop.
10162 */
10163 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10164 for (uint32_t cLoops = 0; ; cLoops++)
10165 {
10166 Assert(!HMR0SuspendPending());
10167 HMVMX_ASSERT_CPU_SAFE();
10168 bool fStepping = pVCpu->hm.s.fSingleInstruction;
10169
10170 /*
10171 * Preparatory work for running guest code, this may force us to return
10172 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10173 */
10174 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10175 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10176 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10177 if (rcStrict != VINF_SUCCESS)
10178 break;
10179
10180 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10181 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10182
10183 /*
10184 * Now we can run the guest code.
10185 */
10186 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10187
10188 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10189
10190 /*
10191 * Restore any residual host-state and save any bits shared between host
10192 * and guest into the guest-CPU state. Re-enables interrupts!
10193 */
10194 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
10195
10196 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10197 if (RT_SUCCESS(rcRun))
10198 { /* very likely */ }
10199 else
10200 {
10201 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10202 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10203 return rcRun;
10204 }
10205
10206 /* Profile the VM-exit. */
10207 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10208 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10209 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10210 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10211 HMVMX_START_EXIT_DISPATCH_PROF();
10212
10213 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10214
10215 /*
10216 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10217 */
10218 rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10219 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10220 if (rcStrict != VINF_SUCCESS)
10221 break;
10222 if (cLoops > pVM->hm.s.cMaxResumeLoops)
10223 {
10224 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10225 rcStrict = VINF_EM_RAW_INTERRUPT;
10226 break;
10227 }
10228
10229 /*
10230 * Stepping: Did the RIP change, if so, consider it a single step.
10231 * Otherwise, make sure one of the TFs gets set.
10232 */
10233 if (fStepping)
10234 {
10235 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10236 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10237 AssertRCReturn(rc2, rc2);
10238 if ( pCtx->rip != DbgState.uRipStart
10239 || pCtx->cs.Sel != DbgState.uCsStart)
10240 {
10241 rcStrict = VINF_EM_DBG_STEPPED;
10242 break;
10243 }
10244 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10245 }
10246
10247 /*
10248 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
10249 */
10250 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10251 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10252 }
10253
10254 /*
10255 * Clear the X86_EFL_TF if necessary.
10256 */
10257 if (pVCpu->hm.s.fClearTrapFlag)
10258 {
10259 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10260 AssertRCReturn(rc2, rc2);
10261 pVCpu->hm.s.fClearTrapFlag = false;
10262 pCtx->eflags.Bits.u1TF = 0;
10263 }
10264 /** @todo there seems to be issues with the resume flag when the monitor trap
10265 * flag is pending without being used. Seen early in bios init when
10266 * accessing APIC page in protected mode. */
10267
10268 /*
10269 * Restore VM-exit control settings as we may not reenter this function the
10270 * next time around.
10271 */
10272 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10273
10274 /* Restore HMCPU indicators. */
10275 pVCpu->hm.s.fUsingDebugLoop = false;
10276 pVCpu->hm.s.fDebugWantRdTscExit = false;
10277 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10278
10279 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10280 return rcStrict;
10281}
10282
10283
10284/** @} */
10285
10286
10287/**
10288 * Checks if any expensive dtrace probes are enabled and we should go to the
10289 * debug loop.
10290 *
10291 * @returns true if we should use debug loop, false if not.
10292 */
10293static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10294{
10295 /* It's probably faster to OR the raw 32-bit counter variables together.
10296 Since the variables are in an array and the probes are next to one
10297 another (more or less), we have good locality. So, better read
10298 eight-nine cache lines ever time and only have one conditional, than
10299 128+ conditionals, right? */
10300 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10301 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10302 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10303 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10304 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10305 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10306 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10307 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10308 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10309 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10310 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10311 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10312 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10313 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10314 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10315 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10316 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10317 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10318 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10319 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10320 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10321 ) != 0
10322 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10323 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10324 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10325 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10326 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10327 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10328 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10329 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10330 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10331 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10332 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10333 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10334 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10335 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10336 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10337 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10338 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10339 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10340 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10341 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10342 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10343 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10344 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10345 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10346 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10347 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10348 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10349 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10350 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10351 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10352 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10353 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10354 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10355 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10356 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10357 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10358 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10359 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10360 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10361 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10362 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10363 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10364 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10365 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10366 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10367 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10368 ) != 0
10369 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10370 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10371 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10372 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10373 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10374 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10375 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10376 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10377 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10378 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10379 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10380 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10381 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10382 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10383 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10384 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10385 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10386 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10387 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10388 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10389 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10390 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10391 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10392 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10393 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10394 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10395 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10396 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10397 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10398 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10399 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10400 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10401 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10402 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10403 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10404 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10405 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10406 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10407 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10408 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10409 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10410 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10411 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10412 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10413 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10414 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10415 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10416 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10417 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10418 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10419 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10420 ) != 0;
10421}
10422
10423
10424/**
10425 * Runs the guest code using VT-x.
10426 *
10427 * @returns Strict VBox status code (i.e. informational status codes too).
10428 * @param pVM The cross context VM structure.
10429 * @param pVCpu The cross context virtual CPU structure.
10430 * @param pCtx Pointer to the guest-CPU context.
10431 */
10432VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10433{
10434 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10435 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10436 HMVMX_ASSERT_PREEMPT_SAFE();
10437
10438 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10439
10440 VBOXSTRICTRC rcStrict;
10441 if ( !pVCpu->hm.s.fUseDebugLoop
10442 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10443 && !DBGFIsStepping(pVCpu)
10444 && !pVM->dbgf.ro.cEnabledInt3Breakpoints)
10445 rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10446 else
10447 rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10448
10449 if (rcStrict == VERR_EM_INTERPRETER)
10450 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10451 else if (rcStrict == VINF_EM_RESET)
10452 rcStrict = VINF_EM_TRIPLE_FAULT;
10453
10454 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10455 if (RT_FAILURE(rc2))
10456 {
10457 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10458 rcStrict = rc2;
10459 }
10460 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10461 return rcStrict;
10462}
10463
10464
10465#ifndef HMVMX_USE_FUNCTION_TABLE
10466DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10467{
10468# ifdef DEBUG_ramshankar
10469# define RETURN_EXIT_CALL(a_CallExpr) \
10470 do { \
10471 int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10472 VBOXSTRICTRC rcStrict = a_CallExpr; \
10473 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10474 return rcStrict; \
10475 } while (0)
10476# else
10477# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10478# endif
10479 switch (rcReason)
10480 {
10481 case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10482 case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10483 case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10484 case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10485 case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10486 case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10487 case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10488 case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10489 case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10490 case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10491 case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10492 case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10493 case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10494 case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10495 case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10496 case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10497 case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10498 case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10499 case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10500 case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10501 case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10502 case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10503 case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10504 case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10505 case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10506 case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10507 case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10508 case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10509 case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10510 case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10511 case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10512 case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10513 case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10514 case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10515
10516 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10517 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10518 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10519 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10520 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10521 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10522 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10523 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10524 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10525
10526 case VMX_EXIT_VMCLEAR:
10527 case VMX_EXIT_VMLAUNCH:
10528 case VMX_EXIT_VMPTRLD:
10529 case VMX_EXIT_VMPTRST:
10530 case VMX_EXIT_VMREAD:
10531 case VMX_EXIT_VMRESUME:
10532 case VMX_EXIT_VMWRITE:
10533 case VMX_EXIT_VMXOFF:
10534 case VMX_EXIT_VMXON:
10535 case VMX_EXIT_INVEPT:
10536 case VMX_EXIT_INVVPID:
10537 case VMX_EXIT_VMFUNC:
10538 case VMX_EXIT_XSAVES:
10539 case VMX_EXIT_XRSTORS:
10540 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10541 case VMX_EXIT_ENCLS:
10542 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10543 case VMX_EXIT_PML_FULL:
10544 default:
10545 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10546 }
10547#undef RETURN_EXIT_CALL
10548}
10549#endif /* !HMVMX_USE_FUNCTION_TABLE */
10550
10551
10552#ifdef VBOX_STRICT
10553/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10554# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10555 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10556
10557# define HMVMX_ASSERT_PREEMPT_CPUID() \
10558 do { \
10559 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10560 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10561 } while (0)
10562
10563# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10564 do { \
10565 AssertPtr(pVCpu); \
10566 AssertPtr(pMixedCtx); \
10567 AssertPtr(pVmxTransient); \
10568 Assert(pVmxTransient->fVMEntryFailed == false); \
10569 Assert(ASMIntAreEnabled()); \
10570 HMVMX_ASSERT_PREEMPT_SAFE(); \
10571 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10572 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10573 HMVMX_ASSERT_PREEMPT_SAFE(); \
10574 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10575 HMVMX_ASSERT_PREEMPT_CPUID(); \
10576 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10577 } while (0)
10578
10579# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10580 do { \
10581 Log4Func(("\n")); \
10582 } while (0)
10583#else /* nonstrict builds: */
10584# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10585 do { \
10586 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10587 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10588 } while (0)
10589# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10590#endif
10591
10592
10593/**
10594 * Advances the guest RIP by the specified number of bytes.
10595 *
10596 * @param pVCpu The cross context virtual CPU structure.
10597 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10598 * out-of-sync. Make sure to update the required fields
10599 * before using them.
10600 * @param cbInstr Number of bytes to advance the RIP by.
10601 *
10602 * @remarks No-long-jump zone!!!
10603 */
10604DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10605{
10606 /* Advance the RIP. */
10607 pMixedCtx->rip += cbInstr;
10608 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10609
10610 /* Update interrupt inhibition. */
10611 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10612 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10613 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10614}
10615
10616
10617/**
10618 * Advances the guest RIP after reading it from the VMCS.
10619 *
10620 * @returns VBox status code, no informational status codes.
10621 * @param pVCpu The cross context virtual CPU structure.
10622 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10623 * out-of-sync. Make sure to update the required fields
10624 * before using them.
10625 * @param pVmxTransient Pointer to the VMX transient structure.
10626 *
10627 * @remarks No-long-jump zone!!!
10628 */
10629static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10630{
10631 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10632 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10633 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10634 AssertRCReturn(rc, rc);
10635
10636 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10637
10638 /*
10639 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10640 * pending debug exception field as it takes care of priority of events.
10641 *
10642 * See Intel spec. 32.2.1 "Debug Exceptions".
10643 */
10644 if ( !pVCpu->hm.s.fSingleInstruction
10645 && pMixedCtx->eflags.Bits.u1TF)
10646 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10647
10648 return VINF_SUCCESS;
10649}
10650
10651
10652/**
10653 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10654 * and update error record fields accordingly.
10655 *
10656 * @return VMX_IGS_* return codes.
10657 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10658 * wrong with the guest state.
10659 *
10660 * @param pVM The cross context VM structure.
10661 * @param pVCpu The cross context virtual CPU structure.
10662 * @param pCtx Pointer to the guest-CPU state.
10663 *
10664 * @remarks This function assumes our cache of the VMCS controls
10665 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10666 */
10667static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10668{
10669#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10670#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10671 uError = (err); \
10672 break; \
10673 } else do { } while (0)
10674
10675 int rc;
10676 uint32_t uError = VMX_IGS_ERROR;
10677 uint32_t u32Val;
10678 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10679
10680 do
10681 {
10682 /*
10683 * CR0.
10684 */
10685 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10686 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10687 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10688 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10689 if (fUnrestrictedGuest)
10690 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
10691
10692 uint32_t u32GuestCR0;
10693 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
10694 AssertRCBreak(rc);
10695 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
10696 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
10697 if ( !fUnrestrictedGuest
10698 && (u32GuestCR0 & X86_CR0_PG)
10699 && !(u32GuestCR0 & X86_CR0_PE))
10700 {
10701 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10702 }
10703
10704 /*
10705 * CR4.
10706 */
10707 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10708 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10709
10710 uint32_t u32GuestCR4;
10711 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
10712 AssertRCBreak(rc);
10713 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
10714 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
10715
10716 /*
10717 * IA32_DEBUGCTL MSR.
10718 */
10719 uint64_t u64Val;
10720 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10721 AssertRCBreak(rc);
10722 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10723 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10724 {
10725 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10726 }
10727 uint64_t u64DebugCtlMsr = u64Val;
10728
10729#ifdef VBOX_STRICT
10730 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10731 AssertRCBreak(rc);
10732 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10733#endif
10734 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10735
10736 /*
10737 * RIP and RFLAGS.
10738 */
10739 uint32_t u32Eflags;
10740#if HC_ARCH_BITS == 64
10741 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10742 AssertRCBreak(rc);
10743 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10744 if ( !fLongModeGuest
10745 || !pCtx->cs.Attr.n.u1Long)
10746 {
10747 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10748 }
10749 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10750 * must be identical if the "IA-32e mode guest" VM-entry
10751 * control is 1 and CS.L is 1. No check applies if the
10752 * CPU supports 64 linear-address bits. */
10753
10754 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10755 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10756 AssertRCBreak(rc);
10757 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10758 VMX_IGS_RFLAGS_RESERVED);
10759 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10760 u32Eflags = u64Val;
10761#else
10762 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10763 AssertRCBreak(rc);
10764 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10765 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10766#endif
10767
10768 if ( fLongModeGuest
10769 || ( fUnrestrictedGuest
10770 && !(u32GuestCR0 & X86_CR0_PE)))
10771 {
10772 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10773 }
10774
10775 uint32_t u32EntryInfo;
10776 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10777 AssertRCBreak(rc);
10778 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10779 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10780 {
10781 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10782 }
10783
10784 /*
10785 * 64-bit checks.
10786 */
10787#if HC_ARCH_BITS == 64
10788 if (fLongModeGuest)
10789 {
10790 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10791 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10792 }
10793
10794 if ( !fLongModeGuest
10795 && (u32GuestCR4 & X86_CR4_PCIDE))
10796 {
10797 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10798 }
10799
10800 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10801 * 51:32 beyond the processor's physical-address width are 0. */
10802
10803 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10804 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10805 {
10806 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10807 }
10808
10809 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10810 AssertRCBreak(rc);
10811 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10812
10813 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10814 AssertRCBreak(rc);
10815 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10816#endif
10817
10818 /*
10819 * PERF_GLOBAL MSR.
10820 */
10821 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10822 {
10823 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10824 AssertRCBreak(rc);
10825 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10826 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10827 }
10828
10829 /*
10830 * PAT MSR.
10831 */
10832 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10833 {
10834 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10835 AssertRCBreak(rc);
10836 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10837 for (unsigned i = 0; i < 8; i++)
10838 {
10839 uint8_t u8Val = (u64Val & 0xff);
10840 if ( u8Val != 0 /* UC */
10841 && u8Val != 1 /* WC */
10842 && u8Val != 4 /* WT */
10843 && u8Val != 5 /* WP */
10844 && u8Val != 6 /* WB */
10845 && u8Val != 7 /* UC- */)
10846 {
10847 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10848 }
10849 u64Val >>= 8;
10850 }
10851 }
10852
10853 /*
10854 * EFER MSR.
10855 */
10856 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10857 {
10858 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10859 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10860 AssertRCBreak(rc);
10861 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10862 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10863 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10864 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10865 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10866 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10867 || !(u32GuestCR0 & X86_CR0_PG)
10868 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10869 VMX_IGS_EFER_LMA_LME_MISMATCH);
10870 }
10871
10872 /*
10873 * Segment registers.
10874 */
10875 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10876 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10877 if (!(u32Eflags & X86_EFL_VM))
10878 {
10879 /* CS */
10880 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10881 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10882 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10883 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10884 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10885 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10886 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10887 /* CS cannot be loaded with NULL in protected mode. */
10888 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10889 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10890 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10891 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10892 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10893 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10894 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10895 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10896 else
10897 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10898
10899 /* SS */
10900 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10901 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10902 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10903 if ( !(pCtx->cr0 & X86_CR0_PE)
10904 || pCtx->cs.Attr.n.u4Type == 3)
10905 {
10906 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10907 }
10908 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10909 {
10910 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10911 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10912 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10913 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10914 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10915 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10916 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10917 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10918 }
10919
10920 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
10921 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10922 {
10923 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10924 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10925 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10926 || pCtx->ds.Attr.n.u4Type > 11
10927 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10928 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10929 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10930 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10931 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10932 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10933 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10934 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10935 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10936 }
10937 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10938 {
10939 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10940 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10941 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10942 || pCtx->es.Attr.n.u4Type > 11
10943 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10944 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10945 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10946 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10947 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10948 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10949 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10950 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10951 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10952 }
10953 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10954 {
10955 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10956 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10957 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10958 || pCtx->fs.Attr.n.u4Type > 11
10959 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10960 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10961 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10962 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10963 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10964 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10965 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10966 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10967 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10968 }
10969 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10970 {
10971 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10972 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10973 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10974 || pCtx->gs.Attr.n.u4Type > 11
10975 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10976 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10977 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10978 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10979 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10980 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10981 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10982 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10983 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10984 }
10985 /* 64-bit capable CPUs. */
10986#if HC_ARCH_BITS == 64
10987 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10988 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10989 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10990 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10991 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10992 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10993 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10994 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10995 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10996 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10997 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10998#endif
10999 }
11000 else
11001 {
11002 /* V86 mode checks. */
11003 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
11004 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11005 {
11006 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
11007 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
11008 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
11009 }
11010 else
11011 {
11012 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
11013 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
11014 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
11015 }
11016
11017 /* CS */
11018 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
11019 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
11020 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
11021 /* SS */
11022 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
11023 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
11024 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
11025 /* DS */
11026 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
11027 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
11028 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
11029 /* ES */
11030 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
11031 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
11032 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
11033 /* FS */
11034 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
11035 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
11036 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
11037 /* GS */
11038 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
11039 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
11040 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
11041 /* 64-bit capable CPUs. */
11042#if HC_ARCH_BITS == 64
11043 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11044 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11045 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11046 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11047 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11048 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
11049 VMX_IGS_LONGMODE_SS_BASE_INVALID);
11050 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
11051 VMX_IGS_LONGMODE_DS_BASE_INVALID);
11052 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
11053 VMX_IGS_LONGMODE_ES_BASE_INVALID);
11054#endif
11055 }
11056
11057 /*
11058 * TR.
11059 */
11060 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
11061 /* 64-bit capable CPUs. */
11062#if HC_ARCH_BITS == 64
11063 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
11064#endif
11065 if (fLongModeGuest)
11066 {
11067 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
11068 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
11069 }
11070 else
11071 {
11072 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
11073 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
11074 VMX_IGS_TR_ATTR_TYPE_INVALID);
11075 }
11076 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
11077 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
11078 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
11079 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
11080 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11081 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
11082 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11083 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
11084
11085 /*
11086 * GDTR and IDTR.
11087 */
11088#if HC_ARCH_BITS == 64
11089 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
11090 AssertRCBreak(rc);
11091 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
11092
11093 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
11094 AssertRCBreak(rc);
11095 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
11096#endif
11097
11098 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
11099 AssertRCBreak(rc);
11100 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11101
11102 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
11103 AssertRCBreak(rc);
11104 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11105
11106 /*
11107 * Guest Non-Register State.
11108 */
11109 /* Activity State. */
11110 uint32_t u32ActivityState;
11111 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
11112 AssertRCBreak(rc);
11113 HMVMX_CHECK_BREAK( !u32ActivityState
11114 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
11115 VMX_IGS_ACTIVITY_STATE_INVALID);
11116 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
11117 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
11118 uint32_t u32IntrState;
11119 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
11120 AssertRCBreak(rc);
11121 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
11122 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11123 {
11124 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11125 }
11126
11127 /** @todo Activity state and injecting interrupts. Left as a todo since we
11128 * currently don't use activity states but ACTIVE. */
11129
11130 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11131 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11132
11133 /* Guest interruptibility-state. */
11134 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11135 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11136 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
11137 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11138 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11139 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11140 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11141 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11142 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11143 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
11144 {
11145 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11146 {
11147 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11148 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11149 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11150 }
11151 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11152 {
11153 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11154 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11155 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11156 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11157 }
11158 }
11159 /** @todo Assumes the processor is not in SMM. */
11160 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11161 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11162 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11163 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11164 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11165 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11166 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11167 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11168 {
11169 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11170 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11171 }
11172
11173 /* Pending debug exceptions. */
11174#if HC_ARCH_BITS == 64
11175 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11176 AssertRCBreak(rc);
11177 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11178 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11179 u32Val = u64Val; /* For pending debug exceptions checks below. */
11180#else
11181 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11182 AssertRCBreak(rc);
11183 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11184 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11185#endif
11186
11187 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11188 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11189 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11190 {
11191 if ( (u32Eflags & X86_EFL_TF)
11192 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11193 {
11194 /* Bit 14 is PendingDebug.BS. */
11195 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11196 }
11197 if ( !(u32Eflags & X86_EFL_TF)
11198 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11199 {
11200 /* Bit 14 is PendingDebug.BS. */
11201 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11202 }
11203 }
11204
11205 /* VMCS link pointer. */
11206 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11207 AssertRCBreak(rc);
11208 if (u64Val != UINT64_C(0xffffffffffffffff))
11209 {
11210 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11211 /** @todo Bits beyond the processor's physical-address width MBZ. */
11212 /** @todo 32-bit located in memory referenced by value of this field (as a
11213 * physical address) must contain the processor's VMCS revision ID. */
11214 /** @todo SMM checks. */
11215 }
11216
11217 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11218 * not using Nested Paging? */
11219 if ( pVM->hm.s.fNestedPaging
11220 && !fLongModeGuest
11221 && CPUMIsGuestInPAEModeEx(pCtx))
11222 {
11223 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11224 AssertRCBreak(rc);
11225 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11226
11227 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11228 AssertRCBreak(rc);
11229 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11230
11231 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11232 AssertRCBreak(rc);
11233 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11234
11235 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11236 AssertRCBreak(rc);
11237 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11238 }
11239
11240 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11241 if (uError == VMX_IGS_ERROR)
11242 uError = VMX_IGS_REASON_NOT_FOUND;
11243 } while (0);
11244
11245 pVCpu->hm.s.u32HMError = uError;
11246 return uError;
11247
11248#undef HMVMX_ERROR_BREAK
11249#undef HMVMX_CHECK_BREAK
11250}
11251
11252/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11253/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11254/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11255
11256/** @name VM-exit handlers.
11257 * @{
11258 */
11259
11260/**
11261 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11262 */
11263HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11264{
11265 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11266 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11267 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11268 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11269 return VINF_SUCCESS;
11270 return VINF_EM_RAW_INTERRUPT;
11271}
11272
11273
11274/**
11275 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11276 */
11277HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11278{
11279 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11280 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11281
11282 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11283 AssertRCReturn(rc, rc);
11284
11285 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11286 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11287 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11288 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11289
11290 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11291 {
11292 /*
11293 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11294 * anything we inject is not going to cause a VM-exit directly for the event being injected.
11295 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11296 *
11297 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11298 */
11299 VMXDispatchHostNmi();
11300 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11301 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11302 return VINF_SUCCESS;
11303 }
11304
11305 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11306 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11307 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11308 { /* likely */ }
11309 else
11310 {
11311 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11312 rcStrictRc1 = VINF_SUCCESS;
11313 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11314 return rcStrictRc1;
11315 }
11316
11317 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11318 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11319 switch (uIntType)
11320 {
11321 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11322 Assert(uVector == X86_XCPT_DB);
11323 /* no break */
11324 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11325 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11326 /* no break */
11327 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11328 {
11329 /*
11330 * If there's any exception caused as a result of event injection, go back to
11331 * the interpreter. The page-fault case is complicated and we manually handle
11332 * any currently pending event in hmR0VmxExitXcptPF. Nested #ACs are already
11333 * handled in hmR0VmxCheckExitDueToEventDelivery.
11334 */
11335 if (!pVCpu->hm.s.Event.fPending)
11336 { /* likely */ }
11337 else if ( uVector != X86_XCPT_PF
11338 && uVector != X86_XCPT_AC)
11339 {
11340 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
11341 rc = VERR_EM_INTERPRETER;
11342 break;
11343 }
11344
11345 switch (uVector)
11346 {
11347 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11348 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11349 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11350 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11351 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11352 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11353 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11354
11355 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11356 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11357 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11358 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11359 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11360 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11361 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11362 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11363 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11364 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11365 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11366 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11367 default:
11368 {
11369 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11370 AssertRCReturn(rc, rc);
11371
11372 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11373 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11374 {
11375 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11376 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11377 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11378
11379 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11380 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11381 AssertRCReturn(rc, rc);
11382 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11383 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11384 0 /* GCPtrFaultAddress */);
11385 AssertRCReturn(rc, rc);
11386 }
11387 else
11388 {
11389 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11390 pVCpu->hm.s.u32HMError = uVector;
11391 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11392 }
11393 break;
11394 }
11395 }
11396 break;
11397 }
11398
11399 default:
11400 {
11401 pVCpu->hm.s.u32HMError = uExitIntInfo;
11402 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11403 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11404 break;
11405 }
11406 }
11407 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11408 return rc;
11409}
11410
11411
11412/**
11413 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11414 */
11415HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11416{
11417 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11418
11419 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11420 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11421
11422 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11423 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11424 return VINF_SUCCESS;
11425}
11426
11427
11428/**
11429 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11430 */
11431HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11432{
11433 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11434 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11435 {
11436 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11437 HMVMX_RETURN_UNEXPECTED_EXIT();
11438 }
11439
11440 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11441
11442 /*
11443 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11444 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11445 */
11446 uint32_t uIntrState = 0;
11447 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11448 AssertRCReturn(rc, rc);
11449
11450 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11451 if ( fBlockSti
11452 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11453 {
11454 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11455 }
11456
11457 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11458 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11459
11460 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11461 return VINF_SUCCESS;
11462}
11463
11464
11465/**
11466 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11467 */
11468HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11469{
11470 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11471 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11472 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11473}
11474
11475
11476/**
11477 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11478 */
11479HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11480{
11481 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11482 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11483 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11484}
11485
11486
11487/**
11488 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11489 */
11490HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11491{
11492 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11493 PVM pVM = pVCpu->CTX_SUFF(pVM);
11494 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11495 if (RT_LIKELY(rc == VINF_SUCCESS))
11496 {
11497 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11498 Assert(pVmxTransient->cbInstr == 2);
11499 }
11500 else
11501 {
11502 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11503 rc = VERR_EM_INTERPRETER;
11504 }
11505 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11506 return rc;
11507}
11508
11509
11510/**
11511 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11512 */
11513HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11514{
11515 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11516 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11517 AssertRCReturn(rc, rc);
11518
11519 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11520 return VINF_EM_RAW_EMULATE_INSTR;
11521
11522 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11523 HMVMX_RETURN_UNEXPECTED_EXIT();
11524}
11525
11526
11527/**
11528 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11529 */
11530HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11531{
11532 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11533 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11534 AssertRCReturn(rc, rc);
11535
11536 PVM pVM = pVCpu->CTX_SUFF(pVM);
11537 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11538 if (RT_LIKELY(rc == VINF_SUCCESS))
11539 {
11540 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11541 Assert(pVmxTransient->cbInstr == 2);
11542 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11543 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11544 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11545 }
11546 else
11547 rc = VERR_EM_INTERPRETER;
11548 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11549 return rc;
11550}
11551
11552
11553/**
11554 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11555 */
11556HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11557{
11558 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11559 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11560 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11561 AssertRCReturn(rc, rc);
11562
11563 PVM pVM = pVCpu->CTX_SUFF(pVM);
11564 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11565 if (RT_SUCCESS(rc))
11566 {
11567 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11568 Assert(pVmxTransient->cbInstr == 3);
11569 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11570 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11571 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11572 }
11573 else
11574 {
11575 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11576 rc = VERR_EM_INTERPRETER;
11577 }
11578 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11579 return rc;
11580}
11581
11582
11583/**
11584 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11585 */
11586HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11587{
11588 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11589 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11590 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11591 AssertRCReturn(rc, rc);
11592
11593 PVM pVM = pVCpu->CTX_SUFF(pVM);
11594 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11595 if (RT_LIKELY(rc == VINF_SUCCESS))
11596 {
11597 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11598 Assert(pVmxTransient->cbInstr == 2);
11599 }
11600 else
11601 {
11602 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11603 rc = VERR_EM_INTERPRETER;
11604 }
11605 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11606 return rc;
11607}
11608
11609
11610/**
11611 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11612 */
11613HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11614{
11615 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11616 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11617
11618 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11619 if (pVCpu->hm.s.fHypercallsEnabled)
11620 {
11621#if 0
11622 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11623#else
11624 /* Aggressive state sync. for now. */
11625 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11626 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11627 AssertRCReturn(rc, rc);
11628#endif
11629
11630 /* Perform the hypercall. */
11631 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11632 if (rcStrict == VINF_SUCCESS)
11633 {
11634 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11635 AssertRCReturn(rc, rc);
11636 }
11637 else
11638 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11639 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11640 || RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
11641
11642 /* If the hypercall changes anything other than guest's general-purpose registers,
11643 we would need to reload the guest changed bits here before VM-entry. */
11644 }
11645 else
11646 Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11647
11648 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11649 if (RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)))
11650 {
11651 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11652 rcStrict = VINF_SUCCESS;
11653 }
11654
11655 return rcStrict;
11656}
11657
11658
11659/**
11660 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11661 */
11662HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11663{
11664 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11665 PVM pVM = pVCpu->CTX_SUFF(pVM);
11666 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11667
11668 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11669 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11670 AssertRCReturn(rc, rc);
11671
11672 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11673 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11674 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11675 else
11676 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11677 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11678 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
11679 return rcStrict;
11680}
11681
11682
11683/**
11684 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11685 */
11686HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11687{
11688 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11689 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11690 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11691 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11692 AssertRCReturn(rc, rc);
11693
11694 PVM pVM = pVCpu->CTX_SUFF(pVM);
11695 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11696 if (RT_LIKELY(rc == VINF_SUCCESS))
11697 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11698 else
11699 {
11700 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11701 rc = VERR_EM_INTERPRETER;
11702 }
11703 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11704 return rc;
11705}
11706
11707
11708/**
11709 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11710 */
11711HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11712{
11713 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11714 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11715 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11716 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11717 AssertRCReturn(rc, rc);
11718
11719 PVM pVM = pVCpu->CTX_SUFF(pVM);
11720 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11721 rc = VBOXSTRICTRC_VAL(rc2);
11722 if (RT_LIKELY( rc == VINF_SUCCESS
11723 || rc == VINF_EM_HALT))
11724 {
11725 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11726 AssertRCReturn(rc3, rc3);
11727
11728 if ( rc == VINF_EM_HALT
11729 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11730 {
11731 rc = VINF_SUCCESS;
11732 }
11733 }
11734 else
11735 {
11736 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11737 rc = VERR_EM_INTERPRETER;
11738 }
11739 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11740 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11741 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11742 return rc;
11743}
11744
11745
11746/**
11747 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11748 */
11749HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11750{
11751 /*
11752 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
11753 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
11754 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
11755 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
11756 */
11757 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11758 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11759 HMVMX_RETURN_UNEXPECTED_EXIT();
11760}
11761
11762
11763/**
11764 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11765 */
11766HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11767{
11768 /*
11769 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
11770 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
11771 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
11772 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
11773 */
11774 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11775 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11776 HMVMX_RETURN_UNEXPECTED_EXIT();
11777}
11778
11779
11780/**
11781 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11782 */
11783HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11784{
11785 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11786 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11787 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11788 HMVMX_RETURN_UNEXPECTED_EXIT();
11789}
11790
11791
11792/**
11793 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11794 */
11795HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11796{
11797 /*
11798 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
11799 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
11800 * See Intel spec. 25.3 "Other Causes of VM-exits".
11801 */
11802 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11803 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11804 HMVMX_RETURN_UNEXPECTED_EXIT();
11805}
11806
11807
11808/**
11809 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11810 * VM-exit.
11811 */
11812HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11813{
11814 /*
11815 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11816 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11817 *
11818 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11819 * See Intel spec. "23.8 Restrictions on VMX operation".
11820 */
11821 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11822 return VINF_SUCCESS;
11823}
11824
11825
11826/**
11827 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11828 * VM-exit.
11829 */
11830HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11831{
11832 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11833 return VINF_EM_RESET;
11834}
11835
11836
11837/**
11838 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11839 */
11840HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11841{
11842 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11843 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11844
11845 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11846 AssertRCReturn(rc, rc);
11847
11848 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11849 rc = VINF_SUCCESS;
11850 else
11851 rc = VINF_EM_HALT;
11852
11853 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11854 if (rc != VINF_SUCCESS)
11855 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11856 return rc;
11857}
11858
11859
11860/**
11861 * VM-exit handler for instructions that result in a \#UD exception delivered to
11862 * the guest.
11863 */
11864HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11865{
11866 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11867 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11868 return VINF_SUCCESS;
11869}
11870
11871
11872/**
11873 * VM-exit handler for expiry of the VMX preemption timer.
11874 */
11875HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11876{
11877 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11878
11879 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11880 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11881
11882 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11883 PVM pVM = pVCpu->CTX_SUFF(pVM);
11884 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11885 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11886 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11887}
11888
11889
11890/**
11891 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11892 */
11893HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11894{
11895 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11896
11897 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11898 rc |= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, false /*fNeedRsp*/);
11899 rc |= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11900 AssertRCReturn(rc, rc);
11901
11902 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11903 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
11904
11905 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11906
11907 return rcStrict;
11908}
11909
11910
11911/**
11912 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11913 */
11914HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11915{
11916 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11917
11918 /* The guest should not invalidate the host CPU's TLBs, fallback to interpreter. */
11919 /** @todo implement EMInterpretInvpcid() */
11920 return VERR_EM_INTERPRETER;
11921}
11922
11923
11924/**
11925 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11926 * Error VM-exit.
11927 */
11928HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11929{
11930 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11931 AssertRCReturn(rc, rc);
11932
11933 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11934 AssertRCReturn(rc, rc);
11935
11936 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11937 NOREF(uInvalidReason);
11938
11939#ifdef VBOX_STRICT
11940 uint32_t uIntrState;
11941 RTHCUINTREG uHCReg;
11942 uint64_t u64Val;
11943 uint32_t u32Val;
11944
11945 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11946 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11947 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11948 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11949 AssertRCReturn(rc, rc);
11950
11951 Log4(("uInvalidReason %u\n", uInvalidReason));
11952 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11953 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11954 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11955 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
11956
11957 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11958 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11959 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11960 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11961 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11962 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11963 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11964 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11965 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11966 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11967 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11968 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11969#else
11970 NOREF(pVmxTransient);
11971#endif
11972
11973 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11974 return VERR_VMX_INVALID_GUEST_STATE;
11975}
11976
11977
11978/**
11979 * VM-exit handler for VM-entry failure due to an MSR-load
11980 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11981 */
11982HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11983{
11984 NOREF(pVmxTransient);
11985 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11986 HMVMX_RETURN_UNEXPECTED_EXIT();
11987}
11988
11989
11990/**
11991 * VM-exit handler for VM-entry failure due to a machine-check event
11992 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11993 */
11994HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11995{
11996 NOREF(pVmxTransient);
11997 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11998 HMVMX_RETURN_UNEXPECTED_EXIT();
11999}
12000
12001
12002/**
12003 * VM-exit handler for all undefined reasons. Should never ever happen.. in
12004 * theory.
12005 */
12006HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12007{
12008 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
12009 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
12010 return VERR_VMX_UNDEFINED_EXIT_CODE;
12011}
12012
12013
12014/**
12015 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
12016 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
12017 * Conditional VM-exit.
12018 */
12019HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12020{
12021 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12022
12023 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
12024 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
12025 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
12026 return VERR_EM_INTERPRETER;
12027 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12028 HMVMX_RETURN_UNEXPECTED_EXIT();
12029}
12030
12031
12032/**
12033 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
12034 */
12035HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12036{
12037 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12038
12039 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
12040 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
12041 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
12042 return VERR_EM_INTERPRETER;
12043 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12044 HMVMX_RETURN_UNEXPECTED_EXIT();
12045}
12046
12047
12048/**
12049 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
12050 */
12051HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12052{
12053 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12054
12055 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
12056 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12057 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12058 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12059 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12060 {
12061 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12062 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12063 }
12064 AssertRCReturn(rc, rc);
12065 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
12066
12067#ifdef VBOX_STRICT
12068 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
12069 {
12070 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
12071 && pMixedCtx->ecx != MSR_K6_EFER)
12072 {
12073 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12074 pMixedCtx->ecx));
12075 HMVMX_RETURN_UNEXPECTED_EXIT();
12076 }
12077 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12078 {
12079 VMXMSREXITREAD enmRead;
12080 VMXMSREXITWRITE enmWrite;
12081 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12082 AssertRCReturn(rc2, rc2);
12083 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
12084 {
12085 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12086 HMVMX_RETURN_UNEXPECTED_EXIT();
12087 }
12088 }
12089 }
12090#endif
12091
12092 PVM pVM = pVCpu->CTX_SUFF(pVM);
12093 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12094 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
12095 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
12096 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
12097 if (RT_SUCCESS(rc))
12098 {
12099 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12100 Assert(pVmxTransient->cbInstr == 2);
12101 }
12102 return rc;
12103}
12104
12105
12106/**
12107 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12108 */
12109HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12110{
12111 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12112 PVM pVM = pVCpu->CTX_SUFF(pVM);
12113 int rc = VINF_SUCCESS;
12114
12115 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
12116 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12117 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12118 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12119 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12120 {
12121 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12122 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12123 }
12124 AssertRCReturn(rc, rc);
12125 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
12126
12127 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12128 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
12129 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12130
12131 if (RT_SUCCESS(rc))
12132 {
12133 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12134
12135 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
12136 if ( pMixedCtx->ecx == MSR_IA32_APICBASE
12137 || ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
12138 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
12139 {
12140 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12141 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
12142 EMInterpretWrmsr() changes it. */
12143 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12144 }
12145 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12146 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12147 else if (pMixedCtx->ecx == MSR_K6_EFER)
12148 {
12149 /*
12150 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12151 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12152 * the other bits as well, SCE and NXE. See @bugref{7368}.
12153 */
12154 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
12155 }
12156
12157 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12158 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12159 {
12160 switch (pMixedCtx->ecx)
12161 {
12162 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
12163 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
12164 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
12165 case MSR_K8_FS_BASE: /* no break */
12166 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
12167 case MSR_K6_EFER: /* already handled above */ break;
12168 default:
12169 {
12170 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12171 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12172 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12173 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
12174 break;
12175 }
12176 }
12177 }
12178#ifdef VBOX_STRICT
12179 else
12180 {
12181 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12182 switch (pMixedCtx->ecx)
12183 {
12184 case MSR_IA32_SYSENTER_CS:
12185 case MSR_IA32_SYSENTER_EIP:
12186 case MSR_IA32_SYSENTER_ESP:
12187 case MSR_K8_FS_BASE:
12188 case MSR_K8_GS_BASE:
12189 {
12190 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12191 HMVMX_RETURN_UNEXPECTED_EXIT();
12192 }
12193
12194 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12195 default:
12196 {
12197 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12198 {
12199 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12200 if (pMixedCtx->ecx != MSR_K6_EFER)
12201 {
12202 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12203 pMixedCtx->ecx));
12204 HMVMX_RETURN_UNEXPECTED_EXIT();
12205 }
12206 }
12207
12208 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12209 {
12210 VMXMSREXITREAD enmRead;
12211 VMXMSREXITWRITE enmWrite;
12212 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12213 AssertRCReturn(rc2, rc2);
12214 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12215 {
12216 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12217 HMVMX_RETURN_UNEXPECTED_EXIT();
12218 }
12219 }
12220 break;
12221 }
12222 }
12223 }
12224#endif /* VBOX_STRICT */
12225 }
12226 return rc;
12227}
12228
12229
12230/**
12231 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12232 */
12233HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12234{
12235 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12236
12237 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12238 return VINF_EM_RAW_INTERRUPT;
12239}
12240
12241
12242/**
12243 * VM-exit handler for when the TPR value is lowered below the specified
12244 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12245 */
12246HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12247{
12248 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12249 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12250
12251 /*
12252 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12253 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12254 */
12255 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12256 return VINF_SUCCESS;
12257}
12258
12259
12260/**
12261 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12262 * VM-exit.
12263 *
12264 * @retval VINF_SUCCESS when guest execution can continue.
12265 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12266 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12267 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12268 * interpreter.
12269 */
12270HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12271{
12272 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12273 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12274 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12275 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12276 AssertRCReturn(rc, rc);
12277
12278 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12279 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12280 PVM pVM = pVCpu->CTX_SUFF(pVM);
12281 VBOXSTRICTRC rcStrict;
12282 rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, true /*fNeedRsp*/);
12283 switch (uAccessType)
12284 {
12285 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12286 {
12287 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12288 AssertRCReturn(rc, rc);
12289
12290 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12291 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12292 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12293 AssertMsg( rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE
12294 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12295 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12296 {
12297 case 0: /* CR0 */
12298 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12299 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12300 break;
12301 case 2: /* CR2 */
12302 /* Nothing to do here, CR2 it's not part of the VMCS. */
12303 break;
12304 case 3: /* CR3 */
12305 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
12306 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12307 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12308 break;
12309 case 4: /* CR4 */
12310 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12311 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12312 VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12313 break;
12314 case 8: /* CR8 */
12315 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12316 /* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12317 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12318 break;
12319 default:
12320 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12321 break;
12322 }
12323
12324 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12325 break;
12326 }
12327
12328 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12329 {
12330 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12331 AssertRCReturn(rc, rc);
12332
12333 Assert( !pVM->hm.s.fNestedPaging
12334 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12335 || pVCpu->hm.s.fUsingDebugLoop
12336 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12337
12338 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12339 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12340 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12341
12342 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12343 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12344 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12345 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12346 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12347 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12348 VBOXSTRICTRC_VAL(rcStrict)));
12349 break;
12350 }
12351
12352 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12353 {
12354 AssertRCReturn(rc, rc);
12355 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12356 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12357 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12358 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12359 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12360 break;
12361 }
12362
12363 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12364 {
12365 AssertRCReturn(rc, rc);
12366 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12367 VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12368 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE,
12369 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12370 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12371 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12372 break;
12373 }
12374
12375 default:
12376 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12377 VERR_VMX_UNEXPECTED_EXCEPTION);
12378 }
12379
12380 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12381 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12382 NOREF(pVM);
12383 return rcStrict;
12384}
12385
12386
12387/**
12388 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12389 * VM-exit.
12390 */
12391HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12392{
12393 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12394 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12395
12396 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12397 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12398 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12399 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12400 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12401 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12402 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12403 AssertRCReturn(rc2, rc2);
12404
12405 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12406 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12407 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12408 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12409 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12410 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12411 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12412 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12413 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12414
12415 /* I/O operation lookup arrays. */
12416 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12417 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12418
12419 VBOXSTRICTRC rcStrict;
12420 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12421 uint32_t const cbInstr = pVmxTransient->cbInstr;
12422 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12423 PVM pVM = pVCpu->CTX_SUFF(pVM);
12424 if (fIOString)
12425 {
12426#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12427 See @bugref{5752#c158}. Should work now. */
12428 /*
12429 * INS/OUTS - I/O String instruction.
12430 *
12431 * Use instruction-information if available, otherwise fall back on
12432 * interpreting the instruction.
12433 */
12434 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12435 fIOWrite ? 'w' : 'r'));
12436 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12437 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12438 {
12439 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12440 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12441 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12442 AssertRCReturn(rc2, rc2);
12443 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12444 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12445 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12446 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12447 if (fIOWrite)
12448 {
12449 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12450 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12451 }
12452 else
12453 {
12454 /*
12455 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12456 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12457 * See Intel Instruction spec. for "INS".
12458 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12459 */
12460 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12461 }
12462 }
12463 else
12464 {
12465 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12466 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12467 AssertRCReturn(rc2, rc2);
12468 rcStrict = IEMExecOne(pVCpu);
12469 }
12470 /** @todo IEM needs to be setting these flags somehow. */
12471 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12472 fUpdateRipAlready = true;
12473#else
12474 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12475 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12476 if (RT_SUCCESS(rcStrict))
12477 {
12478 if (fIOWrite)
12479 {
12480 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12481 (DISCPUMODE)pDis->uAddrMode, cbValue);
12482 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12483 }
12484 else
12485 {
12486 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12487 (DISCPUMODE)pDis->uAddrMode, cbValue);
12488 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12489 }
12490 }
12491 else
12492 {
12493 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12494 pMixedCtx->rip));
12495 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12496 }
12497#endif
12498 }
12499 else
12500 {
12501 /*
12502 * IN/OUT - I/O instruction.
12503 */
12504 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12505 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12506 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12507 if (fIOWrite)
12508 {
12509 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12510 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12511 }
12512 else
12513 {
12514 uint32_t u32Result = 0;
12515 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12516 if (IOM_SUCCESS(rcStrict))
12517 {
12518 /* Save result of I/O IN instr. in AL/AX/EAX. */
12519 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12520 }
12521 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12522 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12523 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12524 }
12525 }
12526
12527 if (IOM_SUCCESS(rcStrict))
12528 {
12529 if (!fUpdateRipAlready)
12530 {
12531 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12532 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12533 }
12534
12535 /*
12536 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12537 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12538 */
12539 if (fIOString)
12540 {
12541 /** @todo Single-step for INS/OUTS with REP prefix? */
12542 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12543 }
12544 else if ( !fDbgStepping
12545 && fGstStepping)
12546 {
12547 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12548 }
12549
12550 /*
12551 * If any I/O breakpoints are armed, we need to check if one triggered
12552 * and take appropriate action.
12553 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12554 */
12555 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12556 AssertRCReturn(rc2, rc2);
12557
12558 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12559 * execution engines about whether hyper BPs and such are pending. */
12560 uint32_t const uDr7 = pMixedCtx->dr[7];
12561 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12562 && X86_DR7_ANY_RW_IO(uDr7)
12563 && (pMixedCtx->cr4 & X86_CR4_DE))
12564 || DBGFBpIsHwIoArmed(pVM)))
12565 {
12566 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12567
12568 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12569 VMMRZCallRing3Disable(pVCpu);
12570 HM_DISABLE_PREEMPT();
12571
12572 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12573
12574 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12575 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12576 {
12577 /* Raise #DB. */
12578 if (fIsGuestDbgActive)
12579 ASMSetDR6(pMixedCtx->dr[6]);
12580 if (pMixedCtx->dr[7] != uDr7)
12581 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12582
12583 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12584 }
12585 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12586 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12587 else if ( rcStrict2 != VINF_SUCCESS
12588 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12589 rcStrict = rcStrict2;
12590 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12591
12592 HM_RESTORE_PREEMPT();
12593 VMMRZCallRing3Enable(pVCpu);
12594 }
12595 }
12596
12597#ifdef VBOX_STRICT
12598 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12599 Assert(!fIOWrite);
12600 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12601 Assert(fIOWrite);
12602 else
12603 {
12604#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12605 * statuses, that the VMM device and some others may return. See
12606 * IOM_SUCCESS() for guidance. */
12607 AssertMsg( RT_FAILURE(rcStrict)
12608 || rcStrict == VINF_SUCCESS
12609 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12610 || rcStrict == VINF_EM_DBG_BREAKPOINT
12611 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12612 || rcStrict == VINF_EM_RAW_TO_R3
12613 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12614#endif
12615 }
12616#endif
12617
12618 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12619 return rcStrict;
12620}
12621
12622
12623/**
12624 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12625 * VM-exit.
12626 */
12627HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12628{
12629 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12630
12631 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12632 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12633 AssertRCReturn(rc, rc);
12634 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12635 {
12636 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12637 AssertRCReturn(rc, rc);
12638 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12639 {
12640 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12641
12642 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12643 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12644
12645 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
12646 Assert(!pVCpu->hm.s.Event.fPending);
12647 pVCpu->hm.s.Event.fPending = true;
12648 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
12649 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12650 AssertRCReturn(rc, rc);
12651 if (fErrorCodeValid)
12652 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12653 else
12654 pVCpu->hm.s.Event.u32ErrCode = 0;
12655 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12656 && uVector == X86_XCPT_PF)
12657 {
12658 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
12659 }
12660
12661 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12662 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12663 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12664 }
12665 }
12666
12667 /* Fall back to the interpreter to emulate the task-switch. */
12668 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12669 return VERR_EM_INTERPRETER;
12670}
12671
12672
12673/**
12674 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12675 */
12676HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12677{
12678 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12679 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12680 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12681 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12682 AssertRCReturn(rc, rc);
12683 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12684 return VINF_EM_DBG_STEPPED;
12685}
12686
12687
12688/**
12689 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12690 */
12691HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12692{
12693 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12694
12695 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12696
12697 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12698 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12699 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12700 {
12701 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12702 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12703 {
12704 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12705 return VERR_EM_INTERPRETER;
12706 }
12707 }
12708 else
12709 {
12710 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12711 rcStrict1 = VINF_SUCCESS;
12712 return rcStrict1;
12713 }
12714
12715#if 0
12716 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
12717 * just sync the whole thing. */
12718 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12719#else
12720 /* Aggressive state sync. for now. */
12721 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12722 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12723 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12724#endif
12725 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12726 AssertRCReturn(rc, rc);
12727
12728 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12729 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12730 VBOXSTRICTRC rcStrict2;
12731 switch (uAccessType)
12732 {
12733 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12734 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12735 {
12736 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12737 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
12738 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12739
12740 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12741 GCPhys &= PAGE_BASE_GC_MASK;
12742 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12743 PVM pVM = pVCpu->CTX_SUFF(pVM);
12744 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12745 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12746
12747 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12748 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12749 CPUMCTX2CORE(pMixedCtx), GCPhys);
12750 Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
12751 if ( rcStrict2 == VINF_SUCCESS
12752 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12753 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12754 {
12755 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12756 | HM_CHANGED_GUEST_RSP
12757 | HM_CHANGED_GUEST_RFLAGS
12758 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12759 rcStrict2 = VINF_SUCCESS;
12760 }
12761 break;
12762 }
12763
12764 default:
12765 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
12766 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12767 break;
12768 }
12769
12770 if (rcStrict2 != VINF_SUCCESS)
12771 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12772 return rcStrict2;
12773}
12774
12775
12776/**
12777 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12778 * VM-exit.
12779 */
12780HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12781{
12782 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12783
12784 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12785 if (pVmxTransient->fWasGuestDebugStateActive)
12786 {
12787 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12788 HMVMX_RETURN_UNEXPECTED_EXIT();
12789 }
12790
12791 if ( !pVCpu->hm.s.fSingleInstruction
12792 && !pVmxTransient->fWasHyperDebugStateActive)
12793 {
12794 Assert(!DBGFIsStepping(pVCpu));
12795 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12796
12797 /* Don't intercept MOV DRx any more. */
12798 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12799 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12800 AssertRCReturn(rc, rc);
12801
12802 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12803 VMMRZCallRing3Disable(pVCpu);
12804 HM_DISABLE_PREEMPT();
12805
12806 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12807 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12808 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12809
12810 HM_RESTORE_PREEMPT();
12811 VMMRZCallRing3Enable(pVCpu);
12812
12813#ifdef VBOX_WITH_STATISTICS
12814 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12815 AssertRCReturn(rc, rc);
12816 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12817 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12818 else
12819 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12820#endif
12821 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12822 return VINF_SUCCESS;
12823 }
12824
12825 /*
12826 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12827 * Update the segment registers and DR7 from the CPU.
12828 */
12829 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12830 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12831 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12832 AssertRCReturn(rc, rc);
12833 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12834
12835 PVM pVM = pVCpu->CTX_SUFF(pVM);
12836 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12837 {
12838 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12839 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
12840 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
12841 if (RT_SUCCESS(rc))
12842 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12843 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12844 }
12845 else
12846 {
12847 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12848 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
12849 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
12850 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12851 }
12852
12853 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12854 if (RT_SUCCESS(rc))
12855 {
12856 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12857 AssertRCReturn(rc2, rc2);
12858 return VINF_SUCCESS;
12859 }
12860 return rc;
12861}
12862
12863
12864/**
12865 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12866 * Conditional VM-exit.
12867 */
12868HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12869{
12870 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12871 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12872
12873 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12874 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12875 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12876 {
12877 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12878 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12879 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12880 {
12881 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12882 return VERR_EM_INTERPRETER;
12883 }
12884 }
12885 else
12886 {
12887 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12888 rcStrict1 = VINF_SUCCESS;
12889 return rcStrict1;
12890 }
12891
12892 RTGCPHYS GCPhys = 0;
12893 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12894
12895#if 0
12896 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12897#else
12898 /* Aggressive state sync. for now. */
12899 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12900 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12901 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12902#endif
12903 AssertRCReturn(rc, rc);
12904
12905 /*
12906 * If we succeed, resume guest execution.
12907 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12908 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12909 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12910 * weird case. See @bugref{6043}.
12911 */
12912 PVM pVM = pVCpu->CTX_SUFF(pVM);
12913 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12914 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
12915 if ( rcStrict2 == VINF_SUCCESS
12916 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12917 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12918 {
12919 /* Successfully handled MMIO operation. */
12920 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12921 | HM_CHANGED_GUEST_RSP
12922 | HM_CHANGED_GUEST_RFLAGS
12923 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12924 return VINF_SUCCESS;
12925 }
12926 return rcStrict2;
12927}
12928
12929
12930/**
12931 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12932 * VM-exit.
12933 */
12934HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12935{
12936 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12937 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12938
12939 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12940 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12941 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12942 {
12943 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12944 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12945 Log4(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12946 }
12947 else
12948 {
12949 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12950 rcStrict1 = VINF_SUCCESS;
12951 return rcStrict1;
12952 }
12953
12954 RTGCPHYS GCPhys = 0;
12955 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12956 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12957#if 0
12958 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12959#else
12960 /* Aggressive state sync. for now. */
12961 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12962 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12963 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12964#endif
12965 AssertRCReturn(rc, rc);
12966
12967 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12968 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12969
12970 RTGCUINT uErrorCode = 0;
12971 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
12972 uErrorCode |= X86_TRAP_PF_ID;
12973 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
12974 uErrorCode |= X86_TRAP_PF_RW;
12975 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
12976 uErrorCode |= X86_TRAP_PF_P;
12977
12978 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12979
12980 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12981 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12982
12983 /* Handle the pagefault trap for the nested shadow table. */
12984 PVM pVM = pVCpu->CTX_SUFF(pVM);
12985 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12986 TRPMResetTrap(pVCpu);
12987
12988 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12989 if ( rcStrict2 == VINF_SUCCESS
12990 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12991 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12992 {
12993 /* Successfully synced our nested page tables. */
12994 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12995 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12996 | HM_CHANGED_GUEST_RSP
12997 | HM_CHANGED_GUEST_RFLAGS);
12998 return VINF_SUCCESS;
12999 }
13000
13001 Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
13002 return rcStrict2;
13003}
13004
13005/** @} */
13006
13007/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13008/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
13009/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13010
13011/** @name VM-exit exception handlers.
13012 * @{
13013 */
13014
13015/**
13016 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
13017 */
13018static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13019{
13020 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13021 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13022
13023 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13024 AssertRCReturn(rc, rc);
13025
13026 if (!(pMixedCtx->cr0 & X86_CR0_NE))
13027 {
13028 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13029 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13030
13031 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13032 * provides VM-exit instruction length. If this causes problem later,
13033 * disassemble the instruction like it's done on AMD-V. */
13034 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13035 AssertRCReturn(rc2, rc2);
13036 return rc;
13037 }
13038
13039 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13040 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13041 return rc;
13042}
13043
13044
13045/**
13046 * VM-exit exception handler for \#BP (Breakpoint exception).
13047 */
13048static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13049{
13050 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13051 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13052
13053 /** @todo Try optimize this by not saving the entire guest state unless
13054 * really needed. */
13055 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13056 AssertRCReturn(rc, rc);
13057
13058 PVM pVM = pVCpu->CTX_SUFF(pVM);
13059 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
13060 if (rc == VINF_EM_RAW_GUEST_TRAP)
13061 {
13062 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13063 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13064 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13065 AssertRCReturn(rc, rc);
13066
13067 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13068 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13069 }
13070
13071 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
13072 return rc;
13073}
13074
13075
13076/**
13077 * VM-exit exception handler for \#AC (alignment check exception).
13078 */
13079static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13080{
13081 RT_NOREF_PV(pMixedCtx);
13082 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13083
13084 /*
13085 * Re-inject it. We'll detect any nesting before getting here.
13086 */
13087 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13088 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13089 AssertRCReturn(rc, rc);
13090 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13091
13092 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13093 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13094 return VINF_SUCCESS;
13095}
13096
13097
13098/**
13099 * VM-exit exception handler for \#DB (Debug exception).
13100 */
13101static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13102{
13103 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13104 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13105 Log6(("XcptDB\n"));
13106
13107 /*
13108 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13109 * for processing.
13110 */
13111 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13112 AssertRCReturn(rc, rc);
13113
13114 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13115 uint64_t uDR6 = X86_DR6_INIT_VAL;
13116 uDR6 |= ( pVmxTransient->uExitQualification
13117 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
13118
13119 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13120 if (rc == VINF_EM_RAW_GUEST_TRAP)
13121 {
13122 /*
13123 * The exception was for the guest. Update DR6, DR7.GD and
13124 * IA32_DEBUGCTL.LBR before forwarding it.
13125 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13126 */
13127 VMMRZCallRing3Disable(pVCpu);
13128 HM_DISABLE_PREEMPT();
13129
13130 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
13131 pMixedCtx->dr[6] |= uDR6;
13132 if (CPUMIsGuestDebugStateActive(pVCpu))
13133 ASMSetDR6(pMixedCtx->dr[6]);
13134
13135 HM_RESTORE_PREEMPT();
13136 VMMRZCallRing3Enable(pVCpu);
13137
13138 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13139 AssertRCReturn(rc, rc);
13140
13141 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13142 pMixedCtx->dr[7] &= ~X86_DR7_GD;
13143
13144 /* Paranoia. */
13145 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13146 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
13147
13148 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
13149 AssertRCReturn(rc, rc);
13150
13151 /*
13152 * Raise #DB in the guest.
13153 *
13154 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
13155 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
13156 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
13157 *
13158 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
13159 */
13160 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13161 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13162 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13163 AssertRCReturn(rc, rc);
13164 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13165 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13166 return VINF_SUCCESS;
13167 }
13168
13169 /*
13170 * Not a guest trap, must be a hypervisor related debug event then.
13171 * Update DR6 in case someone is interested in it.
13172 */
13173 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13174 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13175 CPUMSetHyperDR6(pVCpu, uDR6);
13176
13177 return rc;
13178}
13179
13180
13181/**
13182 * VM-exit exception handler for \#NM (Device-not-available exception: floating
13183 * point exception).
13184 */
13185static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13186{
13187 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13188
13189 /* We require CR0 and EFER. EFER is always up-to-date. */
13190 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13191 AssertRCReturn(rc, rc);
13192
13193 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
13194 VMMRZCallRing3Disable(pVCpu);
13195 HM_DISABLE_PREEMPT();
13196
13197 /* If the guest FPU was active at the time of the #NM VM-exit, then it's a guest fault. */
13198 if (pVmxTransient->fWasGuestFPUStateActive)
13199 {
13200 rc = VINF_EM_RAW_GUEST_TRAP;
13201 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
13202 }
13203 else
13204 {
13205#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13206 Assert(!pVmxTransient->fWasGuestFPUStateActive || pVCpu->hm.s.fUsingDebugLoop);
13207#endif
13208 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu);
13209 Assert( rc == VINF_EM_RAW_GUEST_TRAP
13210 || ((rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
13211 if (rc == VINF_CPUM_HOST_CR0_MODIFIED)
13212 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
13213 }
13214
13215 HM_RESTORE_PREEMPT();
13216 VMMRZCallRing3Enable(pVCpu);
13217
13218 if (rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED)
13219 {
13220 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
13221 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13222 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
13223 pVCpu->hm.s.fPreloadGuestFpu = true;
13224 }
13225 else
13226 {
13227 /* Forward #NM to the guest. */
13228 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
13229 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13230 AssertRCReturn(rc, rc);
13231 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13232 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
13233 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13234 }
13235
13236 return VINF_SUCCESS;
13237}
13238
13239
13240/**
13241 * VM-exit exception handler for \#GP (General-protection exception).
13242 *
13243 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13244 */
13245static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13246{
13247 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13248 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13249
13250 int rc;
13251 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13252 { /* likely */ }
13253 else
13254 {
13255#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13256 Assert(pVCpu->hm.s.fUsingDebugLoop);
13257#endif
13258 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13259 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13260 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13261 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13262 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13263 AssertRCReturn(rc, rc);
13264 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13265 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13266 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13267 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13268 return rc;
13269 }
13270
13271 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13272 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13273
13274 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13275 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13276 AssertRCReturn(rc, rc);
13277
13278 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13279 uint32_t cbOp = 0;
13280 PVM pVM = pVCpu->CTX_SUFF(pVM);
13281 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13282 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13283 if (RT_SUCCESS(rc))
13284 {
13285 rc = VINF_SUCCESS;
13286 Assert(cbOp == pDis->cbInstr);
13287 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13288 switch (pDis->pCurInstr->uOpcode)
13289 {
13290 case OP_CLI:
13291 {
13292 pMixedCtx->eflags.Bits.u1IF = 0;
13293 pMixedCtx->eflags.Bits.u1RF = 0;
13294 pMixedCtx->rip += pDis->cbInstr;
13295 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13296 if ( !fDbgStepping
13297 && pMixedCtx->eflags.Bits.u1TF)
13298 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13299 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13300 break;
13301 }
13302
13303 case OP_STI:
13304 {
13305 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13306 pMixedCtx->eflags.Bits.u1IF = 1;
13307 pMixedCtx->eflags.Bits.u1RF = 0;
13308 pMixedCtx->rip += pDis->cbInstr;
13309 if (!fOldIF)
13310 {
13311 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13312 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13313 }
13314 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13315 if ( !fDbgStepping
13316 && pMixedCtx->eflags.Bits.u1TF)
13317 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13318 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13319 break;
13320 }
13321
13322 case OP_HLT:
13323 {
13324 rc = VINF_EM_HALT;
13325 pMixedCtx->rip += pDis->cbInstr;
13326 pMixedCtx->eflags.Bits.u1RF = 0;
13327 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13328 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13329 break;
13330 }
13331
13332 case OP_POPF:
13333 {
13334 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13335 uint32_t cbParm;
13336 uint32_t uMask;
13337 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13338 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13339 {
13340 cbParm = 4;
13341 uMask = 0xffffffff;
13342 }
13343 else
13344 {
13345 cbParm = 2;
13346 uMask = 0xffff;
13347 }
13348
13349 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13350 RTGCPTR GCPtrStack = 0;
13351 X86EFLAGS Eflags;
13352 Eflags.u32 = 0;
13353 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13354 &GCPtrStack);
13355 if (RT_SUCCESS(rc))
13356 {
13357 Assert(sizeof(Eflags.u32) >= cbParm);
13358 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13359 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13360 }
13361 if (RT_FAILURE(rc))
13362 {
13363 rc = VERR_EM_INTERPRETER;
13364 break;
13365 }
13366 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13367 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13368 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13369 pMixedCtx->esp += cbParm;
13370 pMixedCtx->esp &= uMask;
13371 pMixedCtx->rip += pDis->cbInstr;
13372 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13373 | HM_CHANGED_GUEST_RSP
13374 | HM_CHANGED_GUEST_RFLAGS);
13375 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13376 POPF restores EFLAGS.TF. */
13377 if ( !fDbgStepping
13378 && fGstStepping)
13379 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13380 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13381 break;
13382 }
13383
13384 case OP_PUSHF:
13385 {
13386 uint32_t cbParm;
13387 uint32_t uMask;
13388 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13389 {
13390 cbParm = 4;
13391 uMask = 0xffffffff;
13392 }
13393 else
13394 {
13395 cbParm = 2;
13396 uMask = 0xffff;
13397 }
13398
13399 /* Get the stack pointer & push the contents of eflags onto the stack. */
13400 RTGCPTR GCPtrStack = 0;
13401 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13402 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13403 if (RT_FAILURE(rc))
13404 {
13405 rc = VERR_EM_INTERPRETER;
13406 break;
13407 }
13408 X86EFLAGS Eflags = pMixedCtx->eflags;
13409 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13410 Eflags.Bits.u1RF = 0;
13411 Eflags.Bits.u1VM = 0;
13412
13413 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13414 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13415 {
13416 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13417 rc = VERR_EM_INTERPRETER;
13418 break;
13419 }
13420 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13421 pMixedCtx->esp -= cbParm;
13422 pMixedCtx->esp &= uMask;
13423 pMixedCtx->rip += pDis->cbInstr;
13424 pMixedCtx->eflags.Bits.u1RF = 0;
13425 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13426 | HM_CHANGED_GUEST_RSP
13427 | HM_CHANGED_GUEST_RFLAGS);
13428 if ( !fDbgStepping
13429 && pMixedCtx->eflags.Bits.u1TF)
13430 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13431 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13432 break;
13433 }
13434
13435 case OP_IRET:
13436 {
13437 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13438 * instruction reference. */
13439 RTGCPTR GCPtrStack = 0;
13440 uint32_t uMask = 0xffff;
13441 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13442 uint16_t aIretFrame[3];
13443 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13444 {
13445 rc = VERR_EM_INTERPRETER;
13446 break;
13447 }
13448 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13449 &GCPtrStack);
13450 if (RT_SUCCESS(rc))
13451 {
13452 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13453 PGMACCESSORIGIN_HM));
13454 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13455 }
13456 if (RT_FAILURE(rc))
13457 {
13458 rc = VERR_EM_INTERPRETER;
13459 break;
13460 }
13461 pMixedCtx->eip = 0;
13462 pMixedCtx->ip = aIretFrame[0];
13463 pMixedCtx->cs.Sel = aIretFrame[1];
13464 pMixedCtx->cs.ValidSel = aIretFrame[1];
13465 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13466 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13467 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13468 pMixedCtx->sp += sizeof(aIretFrame);
13469 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13470 | HM_CHANGED_GUEST_SEGMENT_REGS
13471 | HM_CHANGED_GUEST_RSP
13472 | HM_CHANGED_GUEST_RFLAGS);
13473 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13474 if ( !fDbgStepping
13475 && fGstStepping)
13476 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13477 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13478 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13479 break;
13480 }
13481
13482 case OP_INT:
13483 {
13484 uint16_t uVector = pDis->Param1.uValue & 0xff;
13485 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13486 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13487 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13488 break;
13489 }
13490
13491 case OP_INTO:
13492 {
13493 if (pMixedCtx->eflags.Bits.u1OF)
13494 {
13495 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13496 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13497 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13498 }
13499 else
13500 {
13501 pMixedCtx->eflags.Bits.u1RF = 0;
13502 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13503 }
13504 break;
13505 }
13506
13507 default:
13508 {
13509 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13510 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13511 EMCODETYPE_SUPERVISOR);
13512 rc = VBOXSTRICTRC_VAL(rc2);
13513 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13514 /** @todo We have to set pending-debug exceptions here when the guest is
13515 * single-stepping depending on the instruction that was interpreted. */
13516 Log4(("#GP rc=%Rrc\n", rc));
13517 break;
13518 }
13519 }
13520 }
13521 else
13522 rc = VERR_EM_INTERPRETER;
13523
13524 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13525 ("#GP Unexpected rc=%Rrc\n", rc));
13526 return rc;
13527}
13528
13529
13530/**
13531 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13532 * the exception reported in the VMX transient structure back into the VM.
13533 *
13534 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13535 * up-to-date.
13536 */
13537static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13538{
13539 RT_NOREF_PV(pMixedCtx);
13540 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13541#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13542 Assert(pVCpu->hm.s.fUsingDebugLoop);
13543#endif
13544
13545 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13546 hmR0VmxCheckExitDueToEventDelivery(). */
13547 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13548 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13549 AssertRCReturn(rc, rc);
13550 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13551
13552#ifdef DEBUG_ramshankar
13553 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13554 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13555 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13556#endif
13557
13558 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13559 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13560 return VINF_SUCCESS;
13561}
13562
13563
13564/**
13565 * VM-exit exception handler for \#PF (Page-fault exception).
13566 */
13567static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13568{
13569 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13570 PVM pVM = pVCpu->CTX_SUFF(pVM);
13571 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13572 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13573 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13574 AssertRCReturn(rc, rc);
13575
13576 if (!pVM->hm.s.fNestedPaging)
13577 { /* likely */ }
13578 else
13579 {
13580#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13581 Assert(pVCpu->hm.s.fUsingDebugLoop);
13582#endif
13583 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13584 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13585 {
13586 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13587 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13588 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13589 }
13590 else
13591 {
13592 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13593 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13594 Log4(("Pending #DF due to vectoring #PF. NP\n"));
13595 }
13596 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13597 return rc;
13598 }
13599
13600 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13601 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13602 if (pVmxTransient->fVectoringPF)
13603 {
13604 Assert(pVCpu->hm.s.Event.fPending);
13605 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13606 }
13607
13608 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13609 AssertRCReturn(rc, rc);
13610
13611 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13612 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13613
13614 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13615 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13616 (RTGCPTR)pVmxTransient->uExitQualification);
13617
13618 Log4(("#PF: rc=%Rrc\n", rc));
13619 if (rc == VINF_SUCCESS)
13620 {
13621#if 0
13622 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13623 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
13624 * memory? We don't update the whole state here... */
13625 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13626 | HM_CHANGED_GUEST_RSP
13627 | HM_CHANGED_GUEST_RFLAGS
13628 | HM_CHANGED_VMX_GUEST_APIC_STATE);
13629#else
13630 /*
13631 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13632 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13633 */
13634 /** @todo take advantage of CPUM changed flags instead of brute forcing. */
13635 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13636#endif
13637 TRPMResetTrap(pVCpu);
13638 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13639 return rc;
13640 }
13641
13642 if (rc == VINF_EM_RAW_GUEST_TRAP)
13643 {
13644 if (!pVmxTransient->fVectoringDoublePF)
13645 {
13646 /* It's a guest page fault and needs to be reflected to the guest. */
13647 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13648 TRPMResetTrap(pVCpu);
13649 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13650 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13651 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13652 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13653 }
13654 else
13655 {
13656 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13657 TRPMResetTrap(pVCpu);
13658 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13659 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13660 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
13661 }
13662
13663 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13664 return VINF_SUCCESS;
13665 }
13666
13667 TRPMResetTrap(pVCpu);
13668 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13669 return rc;
13670}
13671
13672/** @} */
13673
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette