VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 63127

Last change on this file since 63127 was 63127, checked in by vboxsync, 8 years ago

VMM: warnings
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 583.2 KB
Line 
1/* $Id: HMVMXR0.cpp 63127 2016-08-07 13:03:12Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2016 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#include <iprt/x86.h>
24#include <iprt/asm-amd64-x86.h>
25#include <iprt/thread.h>
26
27#include <VBox/vmm/pdmapi.h>
28#include <VBox/vmm/dbgf.h>
29#include <VBox/vmm/iem.h>
30#include <VBox/vmm/iom.h>
31#include <VBox/vmm/selm.h>
32#include <VBox/vmm/tm.h>
33#include <VBox/vmm/gim.h>
34#ifdef VBOX_WITH_REM
35# include <VBox/vmm/rem.h>
36#endif
37#ifdef VBOX_WITH_NEW_APIC
38# include <VBox/vmm/apic.h>
39#endif
40#include "HMInternal.h"
41#include <VBox/vmm/vm.h>
42#include "HMVMXR0.h"
43#include "dtrace/VBoxVMM.h"
44
45#ifdef DEBUG_ramshankar
46# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
47# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
49# define HMVMX_ALWAYS_CHECK_GUEST_STATE
50# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
51# define HMVMX_ALWAYS_TRAP_PF
52# define HMVMX_ALWAYS_SWAP_FPU_STATE
53# define HMVMX_ALWAYS_FLUSH_TLB
54# define HMVMX_ALWAYS_SWAP_EFER
55#endif
56
57
58/*********************************************************************************************************************************
59* Defined Constants And Macros *
60*********************************************************************************************************************************/
61/** Use the function table. */
62#define HMVMX_USE_FUNCTION_TABLE
63
64/** Determine which tagged-TLB flush handler to use. */
65#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70/** @name Updated-guest-state flags.
71 * @{ */
72#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
73#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
74#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
75#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
76#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
77#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
78#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
79#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
80#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
81#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
82#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
83#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
84#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
85#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
86#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
87#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
88#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
89#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
90#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
91#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
92#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
93 | HMVMX_UPDATED_GUEST_RSP \
94 | HMVMX_UPDATED_GUEST_RFLAGS \
95 | HMVMX_UPDATED_GUEST_CR0 \
96 | HMVMX_UPDATED_GUEST_CR3 \
97 | HMVMX_UPDATED_GUEST_CR4 \
98 | HMVMX_UPDATED_GUEST_GDTR \
99 | HMVMX_UPDATED_GUEST_IDTR \
100 | HMVMX_UPDATED_GUEST_LDTR \
101 | HMVMX_UPDATED_GUEST_TR \
102 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
103 | HMVMX_UPDATED_GUEST_DEBUG \
104 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
105 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
106 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
107 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
108 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
109 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
110 | HMVMX_UPDATED_GUEST_INTR_STATE \
111 | HMVMX_UPDATED_GUEST_APIC_STATE)
112/** @} */
113
114/** @name
115 * Flags to skip redundant reads of some common VMCS fields that are not part of
116 * the guest-CPU state but are in the transient structure.
117 */
118#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
119#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
120#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
123#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
124#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
125/** @} */
126
127/** @name
128 * States of the VMCS.
129 *
130 * This does not reflect all possible VMCS states but currently only those
131 * needed for maintaining the VMCS consistently even when thread-context hooks
132 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
133 */
134#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
135#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
136#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
137/** @} */
138
139/**
140 * Exception bitmap mask for real-mode guests (real-on-v86).
141 *
142 * We need to intercept all exceptions manually except:
143 * - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
144 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
145 * due to bugs in Intel CPUs.
146 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
147 * support.
148 */
149#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
150 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
151 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
152 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
153 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
154 /* RT_BIT(X86_XCPT_MF) always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
155 | RT_BIT(X86_XCPT_XF))
156
157/**
158 * Exception bitmap mask for all contributory exceptions.
159 *
160 * Page fault is deliberately excluded here as it's conditional as to whether
161 * it's contributory or benign. Page faults are handled separately.
162 */
163#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
164 | RT_BIT(X86_XCPT_DE))
165
166/** Maximum VM-instruction error number. */
167#define HMVMX_INSTR_ERROR_MAX 28
168
169/** Profiling macro. */
170#ifdef HM_PROFILE_EXIT_DISPATCH
171# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
172# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
173#else
174# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
175# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
176#endif
177
178/** Assert that preemption is disabled or covered by thread-context hooks. */
179#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
180 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
181
182/** Assert that we haven't migrated CPUs when thread-context hooks are not
183 * used. */
184#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
185 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
186 ("Illegal migration! Entered on CPU %u Current %u\n", \
187 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
188
189/** Helper macro for VM-exit handlers called unexpectedly. */
190#define HMVMX_RETURN_UNEXPECTED_EXIT() \
191 do { \
192 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
193 return VERR_VMX_UNEXPECTED_EXIT; \
194 } while (0)
195
196
197/*********************************************************************************************************************************
198* Structures and Typedefs *
199*********************************************************************************************************************************/
200/**
201 * VMX transient state.
202 *
203 * A state structure for holding miscellaneous information across
204 * VMX non-root operation and restored after the transition.
205 */
206typedef struct VMXTRANSIENT
207{
208 /** The host's rflags/eflags. */
209 RTCCUINTREG fEFlags;
210#if HC_ARCH_BITS == 32
211 uint32_t u32Alignment0;
212#endif
213 /** The guest's TPR value used for TPR shadowing. */
214 uint8_t u8GuestTpr;
215 /** Alignment. */
216 uint8_t abAlignment0[7];
217
218 /** The basic VM-exit reason. */
219 uint16_t uExitReason;
220 /** Alignment. */
221 uint16_t u16Alignment0;
222 /** The VM-exit interruption error code. */
223 uint32_t uExitIntErrorCode;
224 /** The VM-exit exit code qualification. */
225 uint64_t uExitQualification;
226
227 /** The VM-exit interruption-information field. */
228 uint32_t uExitIntInfo;
229 /** The VM-exit instruction-length field. */
230 uint32_t cbInstr;
231 /** The VM-exit instruction-information field. */
232 union
233 {
234 /** Plain unsigned int representation. */
235 uint32_t u;
236 /** INS and OUTS information. */
237 struct
238 {
239 uint32_t u7Reserved0 : 7;
240 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
241 uint32_t u3AddrSize : 3;
242 uint32_t u5Reserved1 : 5;
243 /** The segment register (X86_SREG_XXX). */
244 uint32_t iSegReg : 3;
245 uint32_t uReserved2 : 14;
246 } StrIo;
247 } ExitInstrInfo;
248 /** Whether the VM-entry failed or not. */
249 bool fVMEntryFailed;
250 /** Alignment. */
251 uint8_t abAlignment1[3];
252
253 /** The VM-entry interruption-information field. */
254 uint32_t uEntryIntInfo;
255 /** The VM-entry exception error code field. */
256 uint32_t uEntryXcptErrorCode;
257 /** The VM-entry instruction length field. */
258 uint32_t cbEntryInstr;
259
260 /** IDT-vectoring information field. */
261 uint32_t uIdtVectoringInfo;
262 /** IDT-vectoring error code. */
263 uint32_t uIdtVectoringErrorCode;
264
265 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
266 uint32_t fVmcsFieldsRead;
267
268 /** Whether the guest FPU was active at the time of VM-exit. */
269 bool fWasGuestFPUStateActive;
270 /** Whether the guest debug state was active at the time of VM-exit. */
271 bool fWasGuestDebugStateActive;
272 /** Whether the hyper debug state was active at the time of VM-exit. */
273 bool fWasHyperDebugStateActive;
274 /** Whether TSC-offsetting should be setup before VM-entry. */
275 bool fUpdateTscOffsettingAndPreemptTimer;
276 /** Whether the VM-exit was caused by a page-fault during delivery of a
277 * contributory exception or a page-fault. */
278 bool fVectoringDoublePF;
279 /** Whether the VM-exit was caused by a page-fault during delivery of an
280 * external interrupt or NMI. */
281 bool fVectoringPF;
282} VMXTRANSIENT;
283AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
284AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
285AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
286AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
287AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
288/** Pointer to VMX transient state. */
289typedef VMXTRANSIENT *PVMXTRANSIENT;
290
291
292/**
293 * MSR-bitmap read permissions.
294 */
295typedef enum VMXMSREXITREAD
296{
297 /** Reading this MSR causes a VM-exit. */
298 VMXMSREXIT_INTERCEPT_READ = 0xb,
299 /** Reading this MSR does not cause a VM-exit. */
300 VMXMSREXIT_PASSTHRU_READ
301} VMXMSREXITREAD;
302/** Pointer to MSR-bitmap read permissions. */
303typedef VMXMSREXITREAD* PVMXMSREXITREAD;
304
305/**
306 * MSR-bitmap write permissions.
307 */
308typedef enum VMXMSREXITWRITE
309{
310 /** Writing to this MSR causes a VM-exit. */
311 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
312 /** Writing to this MSR does not cause a VM-exit. */
313 VMXMSREXIT_PASSTHRU_WRITE
314} VMXMSREXITWRITE;
315/** Pointer to MSR-bitmap write permissions. */
316typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
317
318
319/**
320 * VMX VM-exit handler.
321 *
322 * @returns Strict VBox status code (i.e. informational status codes too).
323 * @param pVCpu The cross context virtual CPU structure.
324 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
325 * out-of-sync. Make sure to update the required
326 * fields before using them.
327 * @param pVmxTransient Pointer to the VMX-transient structure.
328 */
329#ifndef HMVMX_USE_FUNCTION_TABLE
330typedef DECLINLINE(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
331#else
332typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
333/** Pointer to VM-exit handler. */
334typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
335#endif
336
337/**
338 * VMX VM-exit handler, non-strict status code.
339 *
340 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
341 *
342 * @returns VBox status code, no informational status code returned.
343 * @param pVCpu The cross context virtual CPU structure.
344 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
345 * out-of-sync. Make sure to update the required
346 * fields before using them.
347 * @param pVmxTransient Pointer to the VMX-transient structure.
348 *
349 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
350 * use of that status code will be replaced with VINF_EM_SOMETHING
351 * later when switching over to IEM.
352 */
353#ifndef HMVMX_USE_FUNCTION_TABLE
354typedef DECLINLINE(int) FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
355#else
356typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
357#endif
358
359
360/*********************************************************************************************************************************
361* Internal Functions *
362*********************************************************************************************************************************/
363static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
364static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
365static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
366static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
367 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
368 bool fStepping, uint32_t *puIntState);
369#if HC_ARCH_BITS == 32
370static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
371#endif
372#ifndef HMVMX_USE_FUNCTION_TABLE
373DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
374# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
375# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
376#else
377# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
378# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
379#endif
380
381
382/** @name VM-exit handlers.
383 * @{
384 */
385static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
386static FNVMXEXITHANDLER hmR0VmxExitExtInt;
387static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
388static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
389static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
390static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
391static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
392static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
393static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
394static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
395static FNVMXEXITHANDLER hmR0VmxExitCpuid;
396static FNVMXEXITHANDLER hmR0VmxExitGetsec;
397static FNVMXEXITHANDLER hmR0VmxExitHlt;
398static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
399static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
400static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
401static FNVMXEXITHANDLER hmR0VmxExitVmcall;
402static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
403static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
404static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
405static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
406static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
407static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
408static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
409static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
410static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
411static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
412static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
413static FNVMXEXITHANDLER hmR0VmxExitMwait;
414static FNVMXEXITHANDLER hmR0VmxExitMtf;
415static FNVMXEXITHANDLER hmR0VmxExitMonitor;
416static FNVMXEXITHANDLER hmR0VmxExitPause;
417static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
418static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
419static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
420static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
421static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
422static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
423static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
424static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
425static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
426static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
427static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
428static FNVMXEXITHANDLER hmR0VmxExitRdrand;
429static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
430/** @} */
431
432static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
433static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
434static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
435static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
436static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
437static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
438static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
439static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
440static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
441
442
443/*********************************************************************************************************************************
444* Global Variables *
445*********************************************************************************************************************************/
446#ifdef HMVMX_USE_FUNCTION_TABLE
447
448/**
449 * VMX_EXIT dispatch table.
450 */
451static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
452{
453 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
454 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
455 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
456 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
457 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
458 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
459 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
460 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
461 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
462 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
463 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
464 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
465 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
466 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
467 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
468 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
469 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
470 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
471 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
472 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
473 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
474 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
475 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
476 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
477 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
478 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
479 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
480 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
481 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
482 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
483 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
484 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
485 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
486 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
487 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
488 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
489 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
490 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
491 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
492 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
493 /* 40 UNDEFINED */ hmR0VmxExitPause,
494 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
495 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
496 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
497 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
498 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
499 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
500 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
501 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
502 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
503 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
504 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
505 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
506 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
507 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
508 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
509 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
510 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
511 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
512 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
513 /* 60 VMX_EXIT_RESERVED_60 */ hmR0VmxExitErrUndefined,
514 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
515 /* 62 VMX_EXIT_RESERVED_62 */ hmR0VmxExitErrUndefined,
516 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
517 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
518};
519#endif /* HMVMX_USE_FUNCTION_TABLE */
520
521#ifdef VBOX_STRICT
522static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
523{
524 /* 0 */ "(Not Used)",
525 /* 1 */ "VMCALL executed in VMX root operation.",
526 /* 2 */ "VMCLEAR with invalid physical address.",
527 /* 3 */ "VMCLEAR with VMXON pointer.",
528 /* 4 */ "VMLAUNCH with non-clear VMCS.",
529 /* 5 */ "VMRESUME with non-launched VMCS.",
530 /* 6 */ "VMRESUME after VMXOFF",
531 /* 7 */ "VM-entry with invalid control fields.",
532 /* 8 */ "VM-entry with invalid host state fields.",
533 /* 9 */ "VMPTRLD with invalid physical address.",
534 /* 10 */ "VMPTRLD with VMXON pointer.",
535 /* 11 */ "VMPTRLD with incorrect revision identifier.",
536 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
537 /* 13 */ "VMWRITE to read-only VMCS component.",
538 /* 14 */ "(Not Used)",
539 /* 15 */ "VMXON executed in VMX root operation.",
540 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
541 /* 17 */ "VM-entry with non-launched executing VMCS.",
542 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
543 /* 19 */ "VMCALL with non-clear VMCS.",
544 /* 20 */ "VMCALL with invalid VM-exit control fields.",
545 /* 21 */ "(Not Used)",
546 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
547 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
548 /* 24 */ "VMCALL with invalid SMM-monitor features.",
549 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
550 /* 26 */ "VM-entry with events blocked by MOV SS.",
551 /* 27 */ "(Not Used)",
552 /* 28 */ "Invalid operand to INVEPT/INVVPID."
553};
554#endif /* VBOX_STRICT */
555
556
557
558/**
559 * Updates the VM's last error record.
560 *
561 * If there was a VMX instruction error, reads the error data from the VMCS and
562 * updates VCPU's last error record as well.
563 *
564 * @param pVM The cross context VM structure.
565 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
566 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
567 * VERR_VMX_INVALID_VMCS_FIELD.
568 * @param rc The error code.
569 */
570static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
571{
572 AssertPtr(pVM);
573 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
574 || rc == VERR_VMX_UNABLE_TO_START_VM)
575 {
576 AssertPtrReturnVoid(pVCpu);
577 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
578 }
579 pVM->hm.s.lLastError = rc;
580}
581
582
583/**
584 * Reads the VM-entry interruption-information field from the VMCS into the VMX
585 * transient structure.
586 *
587 * @returns VBox status code.
588 * @param pVmxTransient Pointer to the VMX transient structure.
589 *
590 * @remarks No-long-jump zone!!!
591 */
592DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
593{
594 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
595 AssertRCReturn(rc, rc);
596 return VINF_SUCCESS;
597}
598
599
600/**
601 * Reads the VM-entry exception error code field from the VMCS into
602 * the VMX transient structure.
603 *
604 * @returns VBox status code.
605 * @param pVmxTransient Pointer to the VMX transient structure.
606 *
607 * @remarks No-long-jump zone!!!
608 */
609DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
610{
611 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
612 AssertRCReturn(rc, rc);
613 return VINF_SUCCESS;
614}
615
616
617/**
618 * Reads the VM-entry exception error code field from the VMCS into
619 * the VMX transient structure.
620 *
621 * @returns VBox status code.
622 * @param pVmxTransient Pointer to the VMX transient structure.
623 *
624 * @remarks No-long-jump zone!!!
625 */
626DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
627{
628 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
629 AssertRCReturn(rc, rc);
630 return VINF_SUCCESS;
631}
632
633
634/**
635 * Reads the VM-exit interruption-information field from the VMCS into the VMX
636 * transient structure.
637 *
638 * @returns VBox status code.
639 * @param pVmxTransient Pointer to the VMX transient structure.
640 */
641DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
642{
643 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
644 {
645 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
646 AssertRCReturn(rc, rc);
647 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
648 }
649 return VINF_SUCCESS;
650}
651
652
653/**
654 * Reads the VM-exit interruption error code from the VMCS into the VMX
655 * transient structure.
656 *
657 * @returns VBox status code.
658 * @param pVmxTransient Pointer to the VMX transient structure.
659 */
660DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
661{
662 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
663 {
664 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
665 AssertRCReturn(rc, rc);
666 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
667 }
668 return VINF_SUCCESS;
669}
670
671
672/**
673 * Reads the VM-exit instruction length field from the VMCS into the VMX
674 * transient structure.
675 *
676 * @returns VBox status code.
677 * @param pVmxTransient Pointer to the VMX transient structure.
678 */
679DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
680{
681 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
682 {
683 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
684 AssertRCReturn(rc, rc);
685 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
686 }
687 return VINF_SUCCESS;
688}
689
690
691/**
692 * Reads the VM-exit instruction-information field from the VMCS into
693 * the VMX transient structure.
694 *
695 * @returns VBox status code.
696 * @param pVmxTransient Pointer to the VMX transient structure.
697 */
698DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
699{
700 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
701 {
702 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
703 AssertRCReturn(rc, rc);
704 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
705 }
706 return VINF_SUCCESS;
707}
708
709
710/**
711 * Reads the exit code qualification from the VMCS into the VMX transient
712 * structure.
713 *
714 * @returns VBox status code.
715 * @param pVCpu The cross context virtual CPU structure of the
716 * calling EMT. (Required for the VMCS cache case.)
717 * @param pVmxTransient Pointer to the VMX transient structure.
718 */
719DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
720{
721 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
722 {
723 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
724 AssertRCReturn(rc, rc);
725 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
726 }
727 return VINF_SUCCESS;
728}
729
730
731/**
732 * Reads the IDT-vectoring information field from the VMCS into the VMX
733 * transient structure.
734 *
735 * @returns VBox status code.
736 * @param pVmxTransient Pointer to the VMX transient structure.
737 *
738 * @remarks No-long-jump zone!!!
739 */
740DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
741{
742 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
743 {
744 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
745 AssertRCReturn(rc, rc);
746 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
747 }
748 return VINF_SUCCESS;
749}
750
751
752/**
753 * Reads the IDT-vectoring error code from the VMCS into the VMX
754 * transient structure.
755 *
756 * @returns VBox status code.
757 * @param pVmxTransient Pointer to the VMX transient structure.
758 */
759DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
760{
761 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
762 {
763 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
764 AssertRCReturn(rc, rc);
765 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
766 }
767 return VINF_SUCCESS;
768}
769
770
771/**
772 * Enters VMX root mode operation on the current CPU.
773 *
774 * @returns VBox status code.
775 * @param pVM The cross context VM structure. Can be
776 * NULL, after a resume.
777 * @param HCPhysCpuPage Physical address of the VMXON region.
778 * @param pvCpuPage Pointer to the VMXON region.
779 */
780static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
781{
782 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
783 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
784 Assert(pvCpuPage);
785 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
786
787 if (pVM)
788 {
789 /* Write the VMCS revision dword to the VMXON region. */
790 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
791 }
792
793 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
794 RTCCUINTREG fEFlags = ASMIntDisableFlags();
795
796 /* Enable the VMX bit in CR4 if necessary. */
797 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
798
799 /* Enter VMX root mode. */
800 int rc = VMXEnable(HCPhysCpuPage);
801 if (RT_FAILURE(rc))
802 {
803 if (!(uOldCr4 & X86_CR4_VMXE))
804 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
805
806 if (pVM)
807 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
808 }
809
810 /* Restore interrupts. */
811 ASMSetFlags(fEFlags);
812 return rc;
813}
814
815
816/**
817 * Exits VMX root mode operation on the current CPU.
818 *
819 * @returns VBox status code.
820 */
821static int hmR0VmxLeaveRootMode(void)
822{
823 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
824
825 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
826 RTCCUINTREG fEFlags = ASMIntDisableFlags();
827
828 /* If we're for some reason not in VMX root mode, then don't leave it. */
829 RTCCUINTREG uHostCR4 = ASMGetCR4();
830
831 int rc;
832 if (uHostCR4 & X86_CR4_VMXE)
833 {
834 /* Exit VMX root mode and clear the VMX bit in CR4. */
835 VMXDisable();
836 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
837 rc = VINF_SUCCESS;
838 }
839 else
840 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
841
842 /* Restore interrupts. */
843 ASMSetFlags(fEFlags);
844 return rc;
845}
846
847
848/**
849 * Allocates and maps one physically contiguous page. The allocated page is
850 * zero'd out. (Used by various VT-x structures).
851 *
852 * @returns IPRT status code.
853 * @param pMemObj Pointer to the ring-0 memory object.
854 * @param ppVirt Where to store the virtual address of the
855 * allocation.
856 * @param pHCPhys Where to store the physical address of the
857 * allocation.
858 */
859DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
860{
861 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
862 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
863 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
864
865 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
866 if (RT_FAILURE(rc))
867 return rc;
868 *ppVirt = RTR0MemObjAddress(*pMemObj);
869 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
870 ASMMemZero32(*ppVirt, PAGE_SIZE);
871 return VINF_SUCCESS;
872}
873
874
875/**
876 * Frees and unmaps an allocated physical page.
877 *
878 * @param pMemObj Pointer to the ring-0 memory object.
879 * @param ppVirt Where to re-initialize the virtual address of
880 * allocation as 0.
881 * @param pHCPhys Where to re-initialize the physical address of the
882 * allocation as 0.
883 */
884DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
885{
886 AssertPtr(pMemObj);
887 AssertPtr(ppVirt);
888 AssertPtr(pHCPhys);
889 if (*pMemObj != NIL_RTR0MEMOBJ)
890 {
891 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
892 AssertRC(rc);
893 *pMemObj = NIL_RTR0MEMOBJ;
894 *ppVirt = 0;
895 *pHCPhys = 0;
896 }
897}
898
899
900/**
901 * Worker function to free VT-x related structures.
902 *
903 * @returns IPRT status code.
904 * @param pVM The cross context VM structure.
905 */
906static void hmR0VmxStructsFree(PVM pVM)
907{
908 for (VMCPUID i = 0; i < pVM->cCpus; i++)
909 {
910 PVMCPU pVCpu = &pVM->aCpus[i];
911 AssertPtr(pVCpu);
912
913 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
914 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
915
916 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
917 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
918
919 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
920 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
921 }
922
923 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
924#ifdef VBOX_WITH_CRASHDUMP_MAGIC
925 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
926#endif
927}
928
929
930/**
931 * Worker function to allocate VT-x related VM structures.
932 *
933 * @returns IPRT status code.
934 * @param pVM The cross context VM structure.
935 */
936static int hmR0VmxStructsAlloc(PVM pVM)
937{
938 /*
939 * Initialize members up-front so we can cleanup properly on allocation failure.
940 */
941#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
942 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
943 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
944 pVM->hm.s.vmx.HCPhys##a_Name = 0;
945
946#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
947 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
948 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
949 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
950
951#ifdef VBOX_WITH_CRASHDUMP_MAGIC
952 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
953#endif
954 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
955
956 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
957 for (VMCPUID i = 0; i < pVM->cCpus; i++)
958 {
959 PVMCPU pVCpu = &pVM->aCpus[i];
960 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
961 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
962 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
963 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
964 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
965 }
966#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
967#undef VMXLOCAL_INIT_VM_MEMOBJ
968
969 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
970 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
971 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
972 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
973
974 /*
975 * Allocate all the VT-x structures.
976 */
977 int rc = VINF_SUCCESS;
978#ifdef VBOX_WITH_CRASHDUMP_MAGIC
979 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
980 if (RT_FAILURE(rc))
981 goto cleanup;
982 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
983 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
984#endif
985
986 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
987 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
988 {
989 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
990 &pVM->hm.s.vmx.HCPhysApicAccess);
991 if (RT_FAILURE(rc))
992 goto cleanup;
993 }
994
995 /*
996 * Initialize per-VCPU VT-x structures.
997 */
998 for (VMCPUID i = 0; i < pVM->cCpus; i++)
999 {
1000 PVMCPU pVCpu = &pVM->aCpus[i];
1001 AssertPtr(pVCpu);
1002
1003 /* Allocate the VM control structure (VMCS). */
1004 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1005 if (RT_FAILURE(rc))
1006 goto cleanup;
1007
1008 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
1009 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1010 {
1011 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1012 &pVCpu->hm.s.vmx.HCPhysVirtApic);
1013 if (RT_FAILURE(rc))
1014 goto cleanup;
1015 }
1016
1017 /*
1018 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1019 * transparent accesses of specific MSRs.
1020 *
1021 * If the condition for enabling MSR bitmaps changes here, don't forget to
1022 * update HMAreMsrBitmapsAvailable().
1023 */
1024 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1025 {
1026 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1027 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1028 if (RT_FAILURE(rc))
1029 goto cleanup;
1030 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1031 }
1032
1033 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1034 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1035 if (RT_FAILURE(rc))
1036 goto cleanup;
1037
1038 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1039 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1040 if (RT_FAILURE(rc))
1041 goto cleanup;
1042 }
1043
1044 return VINF_SUCCESS;
1045
1046cleanup:
1047 hmR0VmxStructsFree(pVM);
1048 return rc;
1049}
1050
1051
1052/**
1053 * Does global VT-x initialization (called during module initialization).
1054 *
1055 * @returns VBox status code.
1056 */
1057VMMR0DECL(int) VMXR0GlobalInit(void)
1058{
1059#ifdef HMVMX_USE_FUNCTION_TABLE
1060 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1061# ifdef VBOX_STRICT
1062 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1063 Assert(g_apfnVMExitHandlers[i]);
1064# endif
1065#endif
1066 return VINF_SUCCESS;
1067}
1068
1069
1070/**
1071 * Does global VT-x termination (called during module termination).
1072 */
1073VMMR0DECL(void) VMXR0GlobalTerm()
1074{
1075 /* Nothing to do currently. */
1076}
1077
1078
1079/**
1080 * Sets up and activates VT-x on the current CPU.
1081 *
1082 * @returns VBox status code.
1083 * @param pCpu Pointer to the global CPU info struct.
1084 * @param pVM The cross context VM structure. Can be
1085 * NULL after a host resume operation.
1086 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1087 * fEnabledByHost is @c true).
1088 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1089 * @a fEnabledByHost is @c true).
1090 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1091 * enable VT-x on the host.
1092 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1093 */
1094VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1095 void *pvMsrs)
1096{
1097 Assert(pCpu);
1098 Assert(pvMsrs);
1099 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1100
1101 /* Enable VT-x if it's not already enabled by the host. */
1102 if (!fEnabledByHost)
1103 {
1104 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1105 if (RT_FAILURE(rc))
1106 return rc;
1107 }
1108
1109 /*
1110 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1111 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1112 */
1113 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1114 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1115 {
1116 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1117 pCpu->fFlushAsidBeforeUse = false;
1118 }
1119 else
1120 pCpu->fFlushAsidBeforeUse = true;
1121
1122 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1123 ++pCpu->cTlbFlushes;
1124
1125 return VINF_SUCCESS;
1126}
1127
1128
1129/**
1130 * Deactivates VT-x on the current CPU.
1131 *
1132 * @returns VBox status code.
1133 * @param pCpu Pointer to the global CPU info struct.
1134 * @param pvCpuPage Pointer to the VMXON region.
1135 * @param HCPhysCpuPage Physical address of the VMXON region.
1136 *
1137 * @remarks This function should never be called when SUPR0EnableVTx() or
1138 * similar was used to enable VT-x on the host.
1139 */
1140VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1141{
1142 NOREF(pCpu);
1143 NOREF(pvCpuPage);
1144 NOREF(HCPhysCpuPage);
1145
1146 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1147 return hmR0VmxLeaveRootMode();
1148}
1149
1150
1151/**
1152 * Sets the permission bits for the specified MSR in the MSR bitmap.
1153 *
1154 * @param pVCpu The cross context virtual CPU structure.
1155 * @param uMsr The MSR value.
1156 * @param enmRead Whether reading this MSR causes a VM-exit.
1157 * @param enmWrite Whether writing this MSR causes a VM-exit.
1158 */
1159static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1160{
1161 int32_t iBit;
1162 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1163
1164 /*
1165 * Layout:
1166 * 0x000 - 0x3ff - Low MSR read bits
1167 * 0x400 - 0x7ff - High MSR read bits
1168 * 0x800 - 0xbff - Low MSR write bits
1169 * 0xc00 - 0xfff - High MSR write bits
1170 */
1171 if (uMsr <= 0x00001FFF)
1172 iBit = uMsr;
1173 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1174 {
1175 iBit = uMsr - UINT32_C(0xC0000000);
1176 pbMsrBitmap += 0x400;
1177 }
1178 else
1179 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1180
1181 Assert(iBit <= 0x1fff);
1182 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1183 ASMBitSet(pbMsrBitmap, iBit);
1184 else
1185 ASMBitClear(pbMsrBitmap, iBit);
1186
1187 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1188 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1189 else
1190 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1191}
1192
1193
1194#ifdef VBOX_STRICT
1195/**
1196 * Gets the permission bits for the specified MSR in the MSR bitmap.
1197 *
1198 * @returns VBox status code.
1199 * @retval VINF_SUCCESS if the specified MSR is found.
1200 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1201 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1202 *
1203 * @param pVCpu The cross context virtual CPU structure.
1204 * @param uMsr The MSR.
1205 * @param penmRead Where to store the read permissions.
1206 * @param penmWrite Where to store the write permissions.
1207 */
1208static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1209{
1210 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1211 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1212 int32_t iBit;
1213 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1214
1215 /* See hmR0VmxSetMsrPermission() for the layout. */
1216 if (uMsr <= 0x00001FFF)
1217 iBit = uMsr;
1218 else if ( uMsr >= 0xC0000000
1219 && uMsr <= 0xC0001FFF)
1220 {
1221 iBit = (uMsr - 0xC0000000);
1222 pbMsrBitmap += 0x400;
1223 }
1224 else
1225 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1226
1227 Assert(iBit <= 0x1fff);
1228 if (ASMBitTest(pbMsrBitmap, iBit))
1229 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1230 else
1231 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1232
1233 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1234 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1235 else
1236 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1237 return VINF_SUCCESS;
1238}
1239#endif /* VBOX_STRICT */
1240
1241
1242/**
1243 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1244 * area.
1245 *
1246 * @returns VBox status code.
1247 * @param pVCpu The cross context virtual CPU structure.
1248 * @param cMsrs The number of MSRs.
1249 */
1250DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1251{
1252 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1253 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1254 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1255 {
1256 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1257 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1258 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1259 }
1260
1261 /* Update number of guest MSRs to load/store across the world-switch. */
1262 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1263 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1264
1265 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1266 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1267 AssertRCReturn(rc, rc);
1268
1269 /* Update the VCPU's copy of the MSR count. */
1270 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1271
1272 return VINF_SUCCESS;
1273}
1274
1275
1276/**
1277 * Adds a new (or updates the value of an existing) guest/host MSR
1278 * pair to be swapped during the world-switch as part of the
1279 * auto-load/store MSR area in the VMCS.
1280 *
1281 * @returns VBox status code.
1282 * @param pVCpu The cross context virtual CPU structure.
1283 * @param uMsr The MSR.
1284 * @param uGuestMsrValue Value of the guest MSR.
1285 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1286 * necessary.
1287 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1288 * its value was updated. Optional, can be NULL.
1289 */
1290static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1291 bool *pfAddedAndUpdated)
1292{
1293 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1294 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1295 uint32_t i;
1296 for (i = 0; i < cMsrs; i++)
1297 {
1298 if (pGuestMsr->u32Msr == uMsr)
1299 break;
1300 pGuestMsr++;
1301 }
1302
1303 bool fAdded = false;
1304 if (i == cMsrs)
1305 {
1306 ++cMsrs;
1307 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1308 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1309
1310 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1311 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1312 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1313
1314 fAdded = true;
1315 }
1316
1317 /* Update the MSR values in the auto-load/store MSR area. */
1318 pGuestMsr->u32Msr = uMsr;
1319 pGuestMsr->u64Value = uGuestMsrValue;
1320
1321 /* Create/update the MSR slot in the host MSR area. */
1322 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1323 pHostMsr += i;
1324 pHostMsr->u32Msr = uMsr;
1325
1326 /*
1327 * Update the host MSR only when requested by the caller AND when we're
1328 * adding it to the auto-load/store area. Otherwise, it would have been
1329 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1330 */
1331 bool fUpdatedMsrValue = false;
1332 if ( fAdded
1333 && fUpdateHostMsr)
1334 {
1335 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1336 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1337 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1338 fUpdatedMsrValue = true;
1339 }
1340
1341 if (pfAddedAndUpdated)
1342 *pfAddedAndUpdated = fUpdatedMsrValue;
1343 return VINF_SUCCESS;
1344}
1345
1346
1347/**
1348 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1349 * auto-load/store MSR area in the VMCS.
1350 *
1351 * @returns VBox status code.
1352 * @param pVCpu The cross context virtual CPU structure.
1353 * @param uMsr The MSR.
1354 */
1355static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1356{
1357 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1358 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1359 for (uint32_t i = 0; i < cMsrs; i++)
1360 {
1361 /* Find the MSR. */
1362 if (pGuestMsr->u32Msr == uMsr)
1363 {
1364 /* If it's the last MSR, simply reduce the count. */
1365 if (i == cMsrs - 1)
1366 {
1367 --cMsrs;
1368 break;
1369 }
1370
1371 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1372 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1373 pLastGuestMsr += cMsrs - 1;
1374 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1375 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1376
1377 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1378 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1379 pLastHostMsr += cMsrs - 1;
1380 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1381 pHostMsr->u64Value = pLastHostMsr->u64Value;
1382 --cMsrs;
1383 break;
1384 }
1385 pGuestMsr++;
1386 }
1387
1388 /* Update the VMCS if the count changed (meaning the MSR was found). */
1389 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1390 {
1391 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1392 AssertRCReturn(rc, rc);
1393
1394 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1395 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1396 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1397
1398 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1399 return VINF_SUCCESS;
1400 }
1401
1402 return VERR_NOT_FOUND;
1403}
1404
1405
1406/**
1407 * Checks if the specified guest MSR is part of the auto-load/store area in
1408 * the VMCS.
1409 *
1410 * @returns true if found, false otherwise.
1411 * @param pVCpu The cross context virtual CPU structure.
1412 * @param uMsr The MSR to find.
1413 */
1414static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1415{
1416 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1417 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1418
1419 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1420 {
1421 if (pGuestMsr->u32Msr == uMsr)
1422 return true;
1423 }
1424 return false;
1425}
1426
1427
1428/**
1429 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1430 *
1431 * @param pVCpu The cross context virtual CPU structure.
1432 *
1433 * @remarks No-long-jump zone!!!
1434 */
1435static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1436{
1437 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1438 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1439 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1440 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1441
1442 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1443 {
1444 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1445
1446 /*
1447 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1448 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1449 */
1450 if (pHostMsr->u32Msr == MSR_K6_EFER)
1451 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1452 else
1453 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1454 }
1455
1456 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1457}
1458
1459
1460/**
1461 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1462 * perform lazy restoration of the host MSRs while leaving VT-x.
1463 *
1464 * @param pVCpu The cross context virtual CPU structure.
1465 *
1466 * @remarks No-long-jump zone!!!
1467 */
1468static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1469{
1470 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1471
1472 /*
1473 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1474 */
1475 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1476 {
1477 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1478#if HC_ARCH_BITS == 64
1479 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1480 {
1481 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1482 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1483 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1484 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1485 }
1486#endif
1487 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1488 }
1489}
1490
1491
1492/**
1493 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1494 * lazily while leaving VT-x.
1495 *
1496 * @returns true if it does, false otherwise.
1497 * @param pVCpu The cross context virtual CPU structure.
1498 * @param uMsr The MSR to check.
1499 */
1500static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1501{
1502 NOREF(pVCpu);
1503#if HC_ARCH_BITS == 64
1504 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1505 {
1506 switch (uMsr)
1507 {
1508 case MSR_K8_LSTAR:
1509 case MSR_K6_STAR:
1510 case MSR_K8_SF_MASK:
1511 case MSR_K8_KERNEL_GS_BASE:
1512 return true;
1513 }
1514 }
1515#else
1516 RT_NOREF(pVCpu, uMsr);
1517#endif
1518 return false;
1519}
1520
1521
1522/**
1523 * Saves a set of guest MSRs back into the guest-CPU context.
1524 *
1525 * @param pVCpu The cross context virtual CPU structure.
1526 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1527 * out-of-sync. Make sure to update the required fields
1528 * before using them.
1529 *
1530 * @remarks No-long-jump zone!!!
1531 */
1532static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1533{
1534 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1535 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1536
1537 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1538 {
1539 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1540#if HC_ARCH_BITS == 64
1541 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1542 {
1543 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1544 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1545 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1546 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1547 }
1548#else
1549 NOREF(pMixedCtx);
1550#endif
1551 }
1552}
1553
1554
1555/**
1556 * Loads a set of guests MSRs to allow read/passthru to the guest.
1557 *
1558 * The name of this function is slightly confusing. This function does NOT
1559 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1560 * common prefix for functions dealing with "lazy restoration" of the shared
1561 * MSRs.
1562 *
1563 * @param pVCpu The cross context virtual CPU structure.
1564 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1565 * out-of-sync. Make sure to update the required fields
1566 * before using them.
1567 *
1568 * @remarks No-long-jump zone!!!
1569 */
1570static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1571{
1572 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1573 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1574
1575#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1576 do { \
1577 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1578 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1579 else \
1580 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1581 } while (0)
1582
1583 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1584 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1585 {
1586#if HC_ARCH_BITS == 64
1587 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1588 {
1589 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1590 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1591 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1592 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1593 }
1594#else
1595 RT_NOREF(pMixedCtx);
1596#endif
1597 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1598 }
1599
1600#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1601}
1602
1603
1604/**
1605 * Performs lazy restoration of the set of host MSRs if they were previously
1606 * loaded with guest MSR values.
1607 *
1608 * @param pVCpu The cross context virtual CPU structure.
1609 *
1610 * @remarks No-long-jump zone!!!
1611 * @remarks The guest MSRs should have been saved back into the guest-CPU
1612 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1613 */
1614static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1615{
1616 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1617 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1618
1619 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1620 {
1621 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1622#if HC_ARCH_BITS == 64
1623 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1624 {
1625 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1626 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1627 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1628 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1629 }
1630#endif
1631 }
1632 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1633}
1634
1635
1636/**
1637 * Verifies that our cached values of the VMCS controls are all
1638 * consistent with what's actually present in the VMCS.
1639 *
1640 * @returns VBox status code.
1641 * @param pVCpu The cross context virtual CPU structure.
1642 */
1643static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1644{
1645 uint32_t u32Val;
1646 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1647 AssertRCReturn(rc, rc);
1648 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1649 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1650
1651 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1652 AssertRCReturn(rc, rc);
1653 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1654 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1655
1656 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1657 AssertRCReturn(rc, rc);
1658 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1659 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1660
1661 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1662 AssertRCReturn(rc, rc);
1663 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1664 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1665
1666 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1667 {
1668 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1669 AssertRCReturn(rc, rc);
1670 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1671 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1672 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1673 }
1674
1675 return VINF_SUCCESS;
1676}
1677
1678
1679#ifdef VBOX_STRICT
1680/**
1681 * Verifies that our cached host EFER value has not changed
1682 * since we cached it.
1683 *
1684 * @param pVCpu The cross context virtual CPU structure.
1685 */
1686static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1687{
1688 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1689
1690 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1691 {
1692 uint64_t u64Val;
1693 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1694 AssertRC(rc);
1695
1696 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1697 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1698 }
1699}
1700
1701
1702/**
1703 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1704 * VMCS are correct.
1705 *
1706 * @param pVCpu The cross context virtual CPU structure.
1707 */
1708static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1709{
1710 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1711
1712 /* Verify MSR counts in the VMCS are what we think it should be. */
1713 uint32_t cMsrs;
1714 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1715 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1716
1717 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1718 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1719
1720 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1721 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1722
1723 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1724 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1725 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1726 {
1727 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1728 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1729 pGuestMsr->u32Msr, cMsrs));
1730
1731 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1732 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1733 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1734
1735 /* Verify that the permissions are as expected in the MSR bitmap. */
1736 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1737 {
1738 VMXMSREXITREAD enmRead;
1739 VMXMSREXITWRITE enmWrite;
1740 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1741 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1742 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1743 {
1744 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1745 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1746 }
1747 else
1748 {
1749 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1750 pGuestMsr->u32Msr, cMsrs));
1751 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1752 pGuestMsr->u32Msr, cMsrs));
1753 }
1754 }
1755 }
1756}
1757#endif /* VBOX_STRICT */
1758
1759
1760/**
1761 * Flushes the TLB using EPT.
1762 *
1763 * @returns VBox status code.
1764 * @param pVCpu The cross context virtual CPU structure of the calling
1765 * EMT. Can be NULL depending on @a enmFlush.
1766 * @param enmFlush Type of flush.
1767 *
1768 * @remarks Caller is responsible for making sure this function is called only
1769 * when NestedPaging is supported and providing @a enmFlush that is
1770 * supported by the CPU.
1771 * @remarks Can be called with interrupts disabled.
1772 */
1773static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1774{
1775 uint64_t au64Descriptor[2];
1776 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1777 au64Descriptor[0] = 0;
1778 else
1779 {
1780 Assert(pVCpu);
1781 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1782 }
1783 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1784
1785 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1786 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1787 rc));
1788 if ( RT_SUCCESS(rc)
1789 && pVCpu)
1790 {
1791 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1792 }
1793}
1794
1795
1796/**
1797 * Flushes the TLB using VPID.
1798 *
1799 * @returns VBox status code.
1800 * @param pVM The cross context VM structure.
1801 * @param pVCpu The cross context virtual CPU structure of the calling
1802 * EMT. Can be NULL depending on @a enmFlush.
1803 * @param enmFlush Type of flush.
1804 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1805 * on @a enmFlush).
1806 *
1807 * @remarks Can be called with interrupts disabled.
1808 */
1809static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1810{
1811 NOREF(pVM);
1812 AssertPtr(pVM);
1813 Assert(pVM->hm.s.vmx.fVpid);
1814
1815 uint64_t au64Descriptor[2];
1816 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1817 {
1818 au64Descriptor[0] = 0;
1819 au64Descriptor[1] = 0;
1820 }
1821 else
1822 {
1823 AssertPtr(pVCpu);
1824 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1825 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1826 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1827 au64Descriptor[1] = GCPtr;
1828 }
1829
1830 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1831 AssertMsg(rc == VINF_SUCCESS,
1832 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1833 if ( RT_SUCCESS(rc)
1834 && pVCpu)
1835 {
1836 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1837 }
1838}
1839
1840
1841/**
1842 * Invalidates a guest page by guest virtual address. Only relevant for
1843 * EPT/VPID, otherwise there is nothing really to invalidate.
1844 *
1845 * @returns VBox status code.
1846 * @param pVM The cross context VM structure.
1847 * @param pVCpu The cross context virtual CPU structure.
1848 * @param GCVirt Guest virtual address of the page to invalidate.
1849 */
1850VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1851{
1852 AssertPtr(pVM);
1853 AssertPtr(pVCpu);
1854 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1855
1856 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1857 if (!fFlushPending)
1858 {
1859 /*
1860 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1861 * See @bugref{6043} and @bugref{6177}.
1862 *
1863 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1864 * function maybe called in a loop with individual addresses.
1865 */
1866 if (pVM->hm.s.vmx.fVpid)
1867 {
1868 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1869 {
1870 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1871 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1872 }
1873 else
1874 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1875 }
1876 else if (pVM->hm.s.fNestedPaging)
1877 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1878 }
1879
1880 return VINF_SUCCESS;
1881}
1882
1883
1884/**
1885 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1886 * otherwise there is nothing really to invalidate.
1887 *
1888 * @returns VBox status code.
1889 * @param pVM The cross context VM structure.
1890 * @param pVCpu The cross context virtual CPU structure.
1891 * @param GCPhys Guest physical address of the page to invalidate.
1892 */
1893VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1894{
1895 NOREF(pVM); NOREF(GCPhys);
1896 LogFlowFunc(("%RGp\n", GCPhys));
1897
1898 /*
1899 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1900 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1901 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1902 */
1903 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1904 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1905 return VINF_SUCCESS;
1906}
1907
1908
1909/**
1910 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1911 * case where neither EPT nor VPID is supported by the CPU.
1912 *
1913 * @param pVM The cross context VM structure.
1914 * @param pVCpu The cross context virtual CPU structure.
1915 * @param pCpu Pointer to the global HM struct.
1916 *
1917 * @remarks Called with interrupts disabled.
1918 */
1919static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1920{
1921 AssertPtr(pVCpu);
1922 AssertPtr(pCpu);
1923 NOREF(pVM);
1924
1925 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1926
1927 Assert(pCpu->idCpu != NIL_RTCPUID);
1928 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1929 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1930 pVCpu->hm.s.fForceTLBFlush = false;
1931 return;
1932}
1933
1934
1935/**
1936 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1937 *
1938 * @param pVM The cross context VM structure.
1939 * @param pVCpu The cross context virtual CPU structure.
1940 * @param pCpu Pointer to the global HM CPU struct.
1941 * @remarks All references to "ASID" in this function pertains to "VPID" in
1942 * Intel's nomenclature. The reason is, to avoid confusion in compare
1943 * statements since the host-CPU copies are named "ASID".
1944 *
1945 * @remarks Called with interrupts disabled.
1946 */
1947static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1948{
1949#ifdef VBOX_WITH_STATISTICS
1950 bool fTlbFlushed = false;
1951# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1952# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1953 if (!fTlbFlushed) \
1954 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1955 } while (0)
1956#else
1957# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1958# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1959#endif
1960
1961 AssertPtr(pVM);
1962 AssertPtr(pCpu);
1963 AssertPtr(pVCpu);
1964 Assert(pCpu->idCpu != NIL_RTCPUID);
1965
1966 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1967 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1968 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1969
1970 /*
1971 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1972 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1973 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1974 */
1975 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1976 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1977 {
1978 ++pCpu->uCurrentAsid;
1979 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1980 {
1981 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1982 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1983 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1984 }
1985
1986 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1987 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1988 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1989
1990 /*
1991 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1992 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1993 */
1994 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1995 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1996 HMVMX_SET_TAGGED_TLB_FLUSHED();
1997 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1998 }
1999
2000 /* Check for explicit TLB flushes. */
2001 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2002 {
2003 /*
2004 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
2005 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2006 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2007 * but not guest-physical mappings.
2008 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2009 */
2010 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2011 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2012 HMVMX_SET_TAGGED_TLB_FLUSHED();
2013 }
2014
2015 pVCpu->hm.s.fForceTLBFlush = false;
2016 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2017
2018 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2019 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2020 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2021 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2022 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2023 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2024 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2025 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2026 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2027
2028 /* Update VMCS with the VPID. */
2029 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2030 AssertRC(rc);
2031
2032#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2033}
2034
2035
2036/**
2037 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2038 *
2039 * @returns VBox status code.
2040 * @param pVM The cross context VM structure.
2041 * @param pVCpu The cross context virtual CPU structure.
2042 * @param pCpu Pointer to the global HM CPU struct.
2043 *
2044 * @remarks Called with interrupts disabled.
2045 */
2046static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2047{
2048 AssertPtr(pVM);
2049 AssertPtr(pVCpu);
2050 AssertPtr(pCpu);
2051 Assert(pCpu->idCpu != NIL_RTCPUID);
2052 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2053 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2054
2055 /*
2056 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2057 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2058 */
2059 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2060 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2061 {
2062 pVCpu->hm.s.fForceTLBFlush = true;
2063 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2064 }
2065
2066 /* Check for explicit TLB flushes. */
2067 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2068 {
2069 pVCpu->hm.s.fForceTLBFlush = true;
2070 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2071 }
2072
2073 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2074 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2075
2076 if (pVCpu->hm.s.fForceTLBFlush)
2077 {
2078 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2079 pVCpu->hm.s.fForceTLBFlush = false;
2080 }
2081}
2082
2083
2084/**
2085 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2086 *
2087 * @returns VBox status code.
2088 * @param pVM The cross context VM structure.
2089 * @param pVCpu The cross context virtual CPU structure.
2090 * @param pCpu Pointer to the global HM CPU struct.
2091 *
2092 * @remarks Called with interrupts disabled.
2093 */
2094static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2095{
2096 AssertPtr(pVM);
2097 AssertPtr(pVCpu);
2098 AssertPtr(pCpu);
2099 Assert(pCpu->idCpu != NIL_RTCPUID);
2100 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2101 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2102
2103 /*
2104 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2105 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2106 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2107 */
2108 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2109 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2110 {
2111 pVCpu->hm.s.fForceTLBFlush = true;
2112 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2113 }
2114
2115 /* Check for explicit TLB flushes. */
2116 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2117 {
2118 /*
2119 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2120 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2121 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2122 */
2123 pVCpu->hm.s.fForceTLBFlush = true;
2124 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2125 }
2126
2127 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2128 if (pVCpu->hm.s.fForceTLBFlush)
2129 {
2130 ++pCpu->uCurrentAsid;
2131 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2132 {
2133 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2134 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2135 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2136 }
2137
2138 pVCpu->hm.s.fForceTLBFlush = false;
2139 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2140 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2141 if (pCpu->fFlushAsidBeforeUse)
2142 {
2143 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2144 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2145 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2146 {
2147 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2148 pCpu->fFlushAsidBeforeUse = false;
2149 }
2150 else
2151 {
2152 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2153 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2154 }
2155 }
2156 }
2157
2158 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2159 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2160 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2161 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2162 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2163 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2164 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2165
2166 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2167 AssertRC(rc);
2168}
2169
2170
2171/**
2172 * Flushes the guest TLB entry based on CPU capabilities.
2173 *
2174 * @param pVCpu The cross context virtual CPU structure.
2175 * @param pCpu Pointer to the global HM CPU struct.
2176 */
2177DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2178{
2179#ifdef HMVMX_ALWAYS_FLUSH_TLB
2180 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2181#endif
2182 PVM pVM = pVCpu->CTX_SUFF(pVM);
2183 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2184 {
2185 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2186 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2187 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2188 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2189 default:
2190 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2191 break;
2192 }
2193
2194 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2195}
2196
2197
2198/**
2199 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2200 * TLB entries from the host TLB before VM-entry.
2201 *
2202 * @returns VBox status code.
2203 * @param pVM The cross context VM structure.
2204 */
2205static int hmR0VmxSetupTaggedTlb(PVM pVM)
2206{
2207 /*
2208 * Determine optimal flush type for Nested Paging.
2209 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2210 * guest execution (see hmR3InitFinalizeR0()).
2211 */
2212 if (pVM->hm.s.fNestedPaging)
2213 {
2214 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2215 {
2216 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2217 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2218 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2219 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2220 else
2221 {
2222 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2223 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2224 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2225 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2226 }
2227
2228 /* Make sure the write-back cacheable memory type for EPT is supported. */
2229 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2230 {
2231 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2232 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2233 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2234 }
2235
2236 /* EPT requires a page-walk length of 4. */
2237 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2238 {
2239 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2240 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2241 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2242 }
2243 }
2244 else
2245 {
2246 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2247 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2248 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2249 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2250 }
2251 }
2252
2253 /*
2254 * Determine optimal flush type for VPID.
2255 */
2256 if (pVM->hm.s.vmx.fVpid)
2257 {
2258 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2259 {
2260 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2261 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2262 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2263 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2264 else
2265 {
2266 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2267 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2268 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2269 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2270 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2271 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2272 pVM->hm.s.vmx.fVpid = false;
2273 }
2274 }
2275 else
2276 {
2277 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2278 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2279 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2280 pVM->hm.s.vmx.fVpid = false;
2281 }
2282 }
2283
2284 /*
2285 * Setup the handler for flushing tagged-TLBs.
2286 */
2287 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2288 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2289 else if (pVM->hm.s.fNestedPaging)
2290 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2291 else if (pVM->hm.s.vmx.fVpid)
2292 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2293 else
2294 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2295 return VINF_SUCCESS;
2296}
2297
2298
2299/**
2300 * Sets up pin-based VM-execution controls in the VMCS.
2301 *
2302 * @returns VBox status code.
2303 * @param pVM The cross context VM structure.
2304 * @param pVCpu The cross context virtual CPU structure.
2305 */
2306static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2307{
2308 AssertPtr(pVM);
2309 AssertPtr(pVCpu);
2310
2311 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2312 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2313
2314 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2315 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2316
2317 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2318 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2319
2320 /* Enable the VMX preemption timer. */
2321 if (pVM->hm.s.vmx.fUsePreemptTimer)
2322 {
2323 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2324 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2325 }
2326
2327#ifdef VBOX_WITH_NEW_APIC
2328#if 0
2329 /* Enable posted-interrupt processing. */
2330 if (pVM->hm.s.fPostedIntrs)
2331 {
2332 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2333 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2334 val |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2335 }
2336#endif
2337#endif
2338
2339 if ((val & zap) != val)
2340 {
2341 LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2342 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2343 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2344 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2345 }
2346
2347 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2348 AssertRCReturn(rc, rc);
2349
2350 pVCpu->hm.s.vmx.u32PinCtls = val;
2351 return rc;
2352}
2353
2354
2355/**
2356 * Sets up processor-based VM-execution controls in the VMCS.
2357 *
2358 * @returns VBox status code.
2359 * @param pVM The cross context VM structure.
2360 * @param pVCpu The cross context virtual CPU structure.
2361 */
2362static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2363{
2364 AssertPtr(pVM);
2365 AssertPtr(pVCpu);
2366
2367 int rc = VERR_INTERNAL_ERROR_5;
2368 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2369 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2370
2371 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2372 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2373 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2374 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2375 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2376 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2377 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2378
2379 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2380 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2381 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2382 {
2383 LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2384 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2385 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2386 }
2387
2388 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2389 if (!pVM->hm.s.fNestedPaging)
2390 {
2391 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2392 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2393 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2394 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2395 }
2396
2397 /* Use TPR shadowing if supported by the CPU. */
2398 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2399 {
2400 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2401 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2402 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2403 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2404 AssertRCReturn(rc, rc);
2405
2406 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2407 /* CR8 writes cause a VM-exit based on TPR threshold. */
2408 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2409 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2410 }
2411 else
2412 {
2413 /*
2414 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2415 * Set this control only for 64-bit guests.
2416 */
2417 if (pVM->hm.s.fAllow64BitGuests)
2418 {
2419 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2420 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2421 }
2422 }
2423
2424 /* Use MSR-bitmaps if supported by the CPU. */
2425 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2426 {
2427 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2428
2429 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2430 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2431 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2432 AssertRCReturn(rc, rc);
2433
2434 /*
2435 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2436 * automatically using dedicated fields in the VMCS.
2437 */
2438 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2439 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2440 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2441 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2442 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2443
2444#if HC_ARCH_BITS == 64
2445 /*
2446 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2447 */
2448 if (pVM->hm.s.fAllow64BitGuests)
2449 {
2450 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2451 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2452 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2453 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2454 }
2455#endif
2456 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2457 }
2458
2459 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2460 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2461 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2462
2463 if ((val & zap) != val)
2464 {
2465 LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2466 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2467 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2468 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2469 }
2470
2471 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2472 AssertRCReturn(rc, rc);
2473
2474 pVCpu->hm.s.vmx.u32ProcCtls = val;
2475
2476 /*
2477 * Secondary processor-based VM-execution controls.
2478 */
2479 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2480 {
2481 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2482 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2483
2484 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2485 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2486
2487 if (pVM->hm.s.fNestedPaging)
2488 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2489 else
2490 {
2491 /*
2492 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2493 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2494 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2495 */
2496 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2497 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2498 }
2499
2500 if (pVM->hm.s.vmx.fVpid)
2501 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2502
2503 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2504 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2505
2506#ifdef VBOX_WITH_NEW_APIC
2507#if 0
2508 if (pVM->hm.s.fVirtApicRegs)
2509 {
2510 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2511 val |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2512
2513 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2514 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2515 }
2516#endif
2517#endif
2518
2519 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2520 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2521 * done dynamically. */
2522 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2523 {
2524 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2525 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2526 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2527 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2528 AssertRCReturn(rc, rc);
2529 }
2530
2531 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2532 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2533
2534 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2535 && pVM->hm.s.vmx.cPleGapTicks
2536 && pVM->hm.s.vmx.cPleWindowTicks)
2537 {
2538 val |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2539
2540 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2541 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2542 AssertRCReturn(rc, rc);
2543 }
2544
2545 if ((val & zap) != val)
2546 {
2547 LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2548 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2549 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2550 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2551 }
2552
2553 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2554 AssertRCReturn(rc, rc);
2555
2556 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2557 }
2558 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2559 {
2560 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2561 "available\n"));
2562 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2563 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2564 }
2565
2566 return VINF_SUCCESS;
2567}
2568
2569
2570/**
2571 * Sets up miscellaneous (everything other than Pin & Processor-based
2572 * VM-execution) control fields in the VMCS.
2573 *
2574 * @returns VBox status code.
2575 * @param pVM The cross context VM structure.
2576 * @param pVCpu The cross context virtual CPU structure.
2577 */
2578static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2579{
2580 NOREF(pVM);
2581 AssertPtr(pVM);
2582 AssertPtr(pVCpu);
2583
2584 int rc = VERR_GENERAL_FAILURE;
2585
2586 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2587#if 0
2588 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2589 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2590 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2591
2592 /*
2593 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2594 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2595 * We thus use the exception bitmap to control it rather than use both.
2596 */
2597 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2598 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2599
2600 /** @todo Explore possibility of using IO-bitmaps. */
2601 /* All IO & IOIO instructions cause VM-exits. */
2602 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2603 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2604
2605 /* Initialize the MSR-bitmap area. */
2606 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2607 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2608 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2609 AssertRCReturn(rc, rc);
2610#endif
2611
2612 /* Setup MSR auto-load/store area. */
2613 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2614 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2615 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2616 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2617 AssertRCReturn(rc, rc);
2618
2619 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2620 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2621 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2622 AssertRCReturn(rc, rc);
2623
2624 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2625 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2626 AssertRCReturn(rc, rc);
2627
2628 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2629#if 0
2630 /* Setup debug controls */
2631 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2632 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2633 AssertRCReturn(rc, rc);
2634#endif
2635
2636 return rc;
2637}
2638
2639
2640/**
2641 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2642 *
2643 * @returns VBox status code.
2644 * @param pVM The cross context VM structure.
2645 * @param pVCpu The cross context virtual CPU structure.
2646 */
2647static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2648{
2649 AssertPtr(pVM);
2650 AssertPtr(pVCpu);
2651
2652 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2653
2654 uint32_t u32XcptBitmap = pVCpu->hm.s.fGIMTrapXcptUD ? RT_BIT(X86_XCPT_UD) : 0;
2655
2656 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2657 u32XcptBitmap |= RT_BIT_32(X86_XCPT_AC);
2658
2659 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2660 and writes, and because recursive #DBs can cause the CPU hang, we must always
2661 intercept #DB. */
2662 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2663
2664 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2665 if (!pVM->hm.s.fNestedPaging)
2666 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2667
2668 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2669 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2670 AssertRCReturn(rc, rc);
2671 return rc;
2672}
2673
2674
2675/**
2676 * Sets up the initial guest-state mask. The guest-state mask is consulted
2677 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2678 * for the nested virtualization case (as it would cause a VM-exit).
2679 *
2680 * @param pVCpu The cross context virtual CPU structure.
2681 */
2682static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2683{
2684 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2685 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2686 return VINF_SUCCESS;
2687}
2688
2689
2690/**
2691 * Does per-VM VT-x initialization.
2692 *
2693 * @returns VBox status code.
2694 * @param pVM The cross context VM structure.
2695 */
2696VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2697{
2698 LogFlowFunc(("pVM=%p\n", pVM));
2699
2700 int rc = hmR0VmxStructsAlloc(pVM);
2701 if (RT_FAILURE(rc))
2702 {
2703 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2704 return rc;
2705 }
2706
2707 return VINF_SUCCESS;
2708}
2709
2710
2711/**
2712 * Does per-VM VT-x termination.
2713 *
2714 * @returns VBox status code.
2715 * @param pVM The cross context VM structure.
2716 */
2717VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2718{
2719 LogFlowFunc(("pVM=%p\n", pVM));
2720
2721#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2722 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2723 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2724#endif
2725 hmR0VmxStructsFree(pVM);
2726 return VINF_SUCCESS;
2727}
2728
2729
2730/**
2731 * Sets up the VM for execution under VT-x.
2732 * This function is only called once per-VM during initialization.
2733 *
2734 * @returns VBox status code.
2735 * @param pVM The cross context VM structure.
2736 */
2737VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2738{
2739 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2740 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2741
2742 LogFlowFunc(("pVM=%p\n", pVM));
2743
2744 /*
2745 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2746 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2747 */
2748 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2749 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2750 || !pVM->hm.s.vmx.pRealModeTSS))
2751 {
2752 LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2753 return VERR_INTERNAL_ERROR;
2754 }
2755
2756 /* Initialize these always, see hmR3InitFinalizeR0().*/
2757 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2758 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2759
2760 /* Setup the tagged-TLB flush handlers. */
2761 int rc = hmR0VmxSetupTaggedTlb(pVM);
2762 if (RT_FAILURE(rc))
2763 {
2764 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2765 return rc;
2766 }
2767
2768 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2769 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2770#if HC_ARCH_BITS == 64
2771 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2772 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2773 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2774 {
2775 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2776 }
2777#endif
2778
2779 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2780 RTCCUINTREG uHostCR4 = ASMGetCR4();
2781 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2782 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2783
2784 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2785 {
2786 PVMCPU pVCpu = &pVM->aCpus[i];
2787 AssertPtr(pVCpu);
2788 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2789
2790 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2791 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2792
2793 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2794 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2795 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2796
2797 /* Set revision dword at the beginning of the VMCS structure. */
2798 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2799
2800 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2801 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2802 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2803 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2804
2805 /* Load this VMCS as the current VMCS. */
2806 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2807 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2808 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2809
2810 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2811 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2812 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2813
2814 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2815 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2816 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2817
2818 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2819 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2820 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2821
2822 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2823 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2824 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2825
2826 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2827 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2828 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2829
2830#if HC_ARCH_BITS == 32
2831 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2832 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2833 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2834#endif
2835
2836 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2837 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2838 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2839 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2840
2841 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2842
2843 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2844 }
2845
2846 return VINF_SUCCESS;
2847}
2848
2849
2850/**
2851 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2852 * the VMCS.
2853 *
2854 * @returns VBox status code.
2855 * @param pVM The cross context VM structure.
2856 * @param pVCpu The cross context virtual CPU structure.
2857 */
2858DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2859{
2860 NOREF(pVM); NOREF(pVCpu);
2861
2862 RTCCUINTREG uReg = ASMGetCR0();
2863 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2864 AssertRCReturn(rc, rc);
2865
2866 uReg = ASMGetCR3();
2867 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2868 AssertRCReturn(rc, rc);
2869
2870 uReg = ASMGetCR4();
2871 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2872 AssertRCReturn(rc, rc);
2873 return rc;
2874}
2875
2876
2877#if HC_ARCH_BITS == 64
2878/**
2879 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2880 * requirements. See hmR0VmxSaveHostSegmentRegs().
2881 */
2882# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2883 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2884 { \
2885 bool fValidSelector = true; \
2886 if ((selValue) & X86_SEL_LDT) \
2887 { \
2888 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2889 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2890 } \
2891 if (fValidSelector) \
2892 { \
2893 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2894 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2895 } \
2896 (selValue) = 0; \
2897 }
2898#endif
2899
2900
2901/**
2902 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2903 * the host-state area in the VMCS.
2904 *
2905 * @returns VBox status code.
2906 * @param pVM The cross context VM structure.
2907 * @param pVCpu The cross context virtual CPU structure.
2908 */
2909DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2910{
2911 int rc = VERR_INTERNAL_ERROR_5;
2912
2913#if HC_ARCH_BITS == 64
2914 /*
2915 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2916 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2917 *
2918 * This apparently can happen (most likely the FPU changes), deal with it rather than asserting.
2919 * Was observed booting Solaris10u10 32-bit guest.
2920 */
2921 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2922 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2923 {
2924 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2925 pVCpu->idCpu));
2926 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2927 }
2928 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2929#else
2930 RT_NOREF(pVCpu);
2931#endif
2932
2933 /*
2934 * Host DS, ES, FS and GS segment registers.
2935 */
2936#if HC_ARCH_BITS == 64
2937 RTSEL uSelDS = ASMGetDS();
2938 RTSEL uSelES = ASMGetES();
2939 RTSEL uSelFS = ASMGetFS();
2940 RTSEL uSelGS = ASMGetGS();
2941#else
2942 RTSEL uSelDS = 0;
2943 RTSEL uSelES = 0;
2944 RTSEL uSelFS = 0;
2945 RTSEL uSelGS = 0;
2946#endif
2947
2948 /*
2949 * Host CS and SS segment registers.
2950 */
2951 RTSEL uSelCS = ASMGetCS();
2952 RTSEL uSelSS = ASMGetSS();
2953
2954 /*
2955 * Host TR segment register.
2956 */
2957 RTSEL uSelTR = ASMGetTR();
2958
2959#if HC_ARCH_BITS == 64
2960 /*
2961 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2962 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2963 */
2964 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2965 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2966 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2967 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2968# undef VMXLOCAL_ADJUST_HOST_SEG
2969#endif
2970
2971 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2972 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2973 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2974 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2975 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2976 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2977 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2978 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2979 Assert(uSelCS);
2980 Assert(uSelTR);
2981
2982 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2983#if 0
2984 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2985 Assert(uSelSS != 0);
2986#endif
2987
2988 /* Write these host selector fields into the host-state area in the VMCS. */
2989 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2990 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2991#if HC_ARCH_BITS == 64
2992 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2993 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2994 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2995 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2996#else
2997 NOREF(uSelDS);
2998 NOREF(uSelES);
2999 NOREF(uSelFS);
3000 NOREF(uSelGS);
3001#endif
3002 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3003 AssertRCReturn(rc, rc);
3004
3005 /*
3006 * Host GDTR and IDTR.
3007 */
3008 RTGDTR Gdtr;
3009 RTIDTR Idtr;
3010 RT_ZERO(Gdtr);
3011 RT_ZERO(Idtr);
3012 ASMGetGDTR(&Gdtr);
3013 ASMGetIDTR(&Idtr);
3014 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3015 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3016 AssertRCReturn(rc, rc);
3017
3018#if HC_ARCH_BITS == 64
3019 /*
3020 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3021 * maximum limit (0xffff) on every VM-exit.
3022 */
3023 if (Gdtr.cbGdt != 0xffff)
3024 {
3025 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3026 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3027 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3028 }
3029
3030 /*
3031 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3032 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3033 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3034 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3035 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3036 * hosts where we are pretty sure it won't cause trouble.
3037 */
3038# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3039 if (Idtr.cbIdt < 0x0fff)
3040# else
3041 if (Idtr.cbIdt != 0xffff)
3042# endif
3043 {
3044 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3045 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3046 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3047 }
3048#endif
3049
3050 /*
3051 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3052 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3053 */
3054 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3055 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3056 VERR_VMX_INVALID_HOST_STATE);
3057
3058 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3059#if HC_ARCH_BITS == 64
3060 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3061
3062 /*
3063 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3064 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3065 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3066 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3067 *
3068 * [1] See Intel spec. 3.5 "System Descriptor Types".
3069 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3070 */
3071 Assert(pDesc->System.u4Type == 11);
3072 if ( pDesc->System.u16LimitLow != 0x67
3073 || pDesc->System.u4LimitHigh)
3074 {
3075 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3076 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3077 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3078 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3079 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3080
3081 /* Store the GDTR here as we need it while restoring TR. */
3082 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3083 }
3084#else
3085 NOREF(pVM);
3086 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3087#endif
3088 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3089 AssertRCReturn(rc, rc);
3090
3091 /*
3092 * Host FS base and GS base.
3093 */
3094#if HC_ARCH_BITS == 64
3095 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3096 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3097 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3098 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3099 AssertRCReturn(rc, rc);
3100
3101 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3102 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3103 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3104 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3105 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3106#endif
3107 return rc;
3108}
3109
3110
3111/**
3112 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3113 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3114 * the host after every successful VM-exit.
3115 *
3116 * @returns VBox status code.
3117 * @param pVM The cross context VM structure.
3118 * @param pVCpu The cross context virtual CPU structure.
3119 *
3120 * @remarks No-long-jump zone!!!
3121 */
3122DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3123{
3124 NOREF(pVM);
3125
3126 AssertPtr(pVCpu);
3127 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3128
3129 /*
3130 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3131 * rather than swapping them on every VM-entry.
3132 */
3133 hmR0VmxLazySaveHostMsrs(pVCpu);
3134
3135 /*
3136 * Host Sysenter MSRs.
3137 */
3138 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3139#if HC_ARCH_BITS == 32
3140 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3141 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3142#else
3143 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3144 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3145#endif
3146 AssertRCReturn(rc, rc);
3147
3148 /*
3149 * Host EFER MSR.
3150 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3151 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3152 */
3153 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3154 {
3155 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3156 AssertRCReturn(rc, rc);
3157 }
3158
3159 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3160 * hmR0VmxLoadGuestExitCtls() !! */
3161
3162 return rc;
3163}
3164
3165
3166/**
3167 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3168 *
3169 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3170 * these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3171 * hmR0VMxLoadGuestEntryCtls().
3172 *
3173 * @returns true if we need to load guest EFER, false otherwise.
3174 * @param pVCpu The cross context virtual CPU structure.
3175 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3176 * out-of-sync. Make sure to update the required fields
3177 * before using them.
3178 *
3179 * @remarks Requires EFER, CR4.
3180 * @remarks No-long-jump zone!!!
3181 */
3182static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3183{
3184#ifdef HMVMX_ALWAYS_SWAP_EFER
3185 return true;
3186#endif
3187
3188#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3189 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3190 if (CPUMIsGuestInLongMode(pVCpu))
3191 return false;
3192#endif
3193
3194 PVM pVM = pVCpu->CTX_SUFF(pVM);
3195 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3196 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3197
3198 /*
3199 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3200 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3201 */
3202 if ( CPUMIsGuestInLongMode(pVCpu)
3203 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3204 {
3205 return true;
3206 }
3207
3208 /*
3209 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3210 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3211 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3212 */
3213 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3214 && (pMixedCtx->cr0 & X86_CR0_PG)
3215 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3216 {
3217 /* Assert that host is PAE capable. */
3218 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3219 return true;
3220 }
3221
3222 /** @todo Check the latest Intel spec. for any other bits,
3223 * like SMEP/SMAP? */
3224 return false;
3225}
3226
3227
3228/**
3229 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3230 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3231 * controls".
3232 *
3233 * @returns VBox status code.
3234 * @param pVCpu The cross context virtual CPU structure.
3235 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3236 * out-of-sync. Make sure to update the required fields
3237 * before using them.
3238 *
3239 * @remarks Requires EFER.
3240 * @remarks No-long-jump zone!!!
3241 */
3242DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3243{
3244 int rc = VINF_SUCCESS;
3245 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3246 {
3247 PVM pVM = pVCpu->CTX_SUFF(pVM);
3248 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3249 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3250
3251 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3252 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3253
3254 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3255 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3256 {
3257 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3258 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3259 }
3260 else
3261 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3262
3263 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3264 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3265 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3266 {
3267 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3268 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3269 }
3270
3271 /*
3272 * The following should -not- be set (since we're not in SMM mode):
3273 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3274 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3275 */
3276
3277 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3278 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3279
3280 if ((val & zap) != val)
3281 {
3282 LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3283 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3284 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3285 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3286 }
3287
3288 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3289 AssertRCReturn(rc, rc);
3290
3291 pVCpu->hm.s.vmx.u32EntryCtls = val;
3292 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3293 }
3294 return rc;
3295}
3296
3297
3298/**
3299 * Sets up the VM-exit controls in the VMCS.
3300 *
3301 * @returns VBox status code.
3302 * @param pVCpu The cross context virtual CPU structure.
3303 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3304 * out-of-sync. Make sure to update the required fields
3305 * before using them.
3306 *
3307 * @remarks Requires EFER.
3308 */
3309DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3310{
3311 NOREF(pMixedCtx);
3312
3313 int rc = VINF_SUCCESS;
3314 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3315 {
3316 PVM pVM = pVCpu->CTX_SUFF(pVM);
3317 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3318 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3319
3320 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3321 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3322
3323 /*
3324 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3325 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3326 */
3327#if HC_ARCH_BITS == 64
3328 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3329 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3330#else
3331 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3332 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3333 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3334 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3335 {
3336 /* The switcher returns to long mode, EFER is managed by the switcher. */
3337 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3338 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3339 }
3340 else
3341 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3342#endif
3343
3344 /* If the newer VMCS fields for managing EFER exists, use it. */
3345 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3346 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3347 {
3348 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3349 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3350 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3351 }
3352
3353 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3354 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3355
3356 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3357 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3358 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3359
3360 if ( pVM->hm.s.vmx.fUsePreemptTimer
3361 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3362 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3363
3364 if ((val & zap) != val)
3365 {
3366 LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3367 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3368 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3369 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3370 }
3371
3372 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3373 AssertRCReturn(rc, rc);
3374
3375 pVCpu->hm.s.vmx.u32ExitCtls = val;
3376 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3377 }
3378 return rc;
3379}
3380
3381
3382/**
3383 * Sets the TPR threshold in the VMCS.
3384 *
3385 * @returns VBox status code.
3386 * @param pVCpu The cross context virtual CPU structure.
3387 * @param u32TprThreshold The TPR threshold (task-priority class only).
3388 */
3389DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3390{
3391 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3392 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3393 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3394}
3395
3396
3397/**
3398 * Loads the guest APIC and related state.
3399 *
3400 * @returns VBox status code.
3401 * @param pVCpu The cross context virtual CPU structure.
3402 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3403 * out-of-sync. Make sure to update the required fields
3404 * before using them.
3405 */
3406DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3407{
3408 NOREF(pMixedCtx);
3409
3410 int rc = VINF_SUCCESS;
3411 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3412 {
3413 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3414 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3415 {
3416 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3417
3418 bool fPendingIntr = false;
3419 uint8_t u8Tpr = 0;
3420 uint8_t u8PendingIntr = 0;
3421 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3422 AssertRCReturn(rc, rc);
3423
3424 /*
3425 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3426 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3427 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3428 * the interrupt when we VM-exit for other reasons.
3429 */
3430 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3431 uint32_t u32TprThreshold = 0;
3432 if (fPendingIntr)
3433 {
3434 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3435 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3436 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3437 if (u8PendingPriority <= u8TprPriority)
3438 u32TprThreshold = u8PendingPriority;
3439 else
3440 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3441 }
3442
3443 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3444 AssertRCReturn(rc, rc);
3445 }
3446
3447 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3448 }
3449 return rc;
3450}
3451
3452
3453/**
3454 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3455 *
3456 * @returns Guest's interruptibility-state.
3457 * @param pVCpu The cross context virtual CPU structure.
3458 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3459 * out-of-sync. Make sure to update the required fields
3460 * before using them.
3461 *
3462 * @remarks No-long-jump zone!!!
3463 */
3464DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3465{
3466 /*
3467 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3468 */
3469 uint32_t uIntrState = 0;
3470 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3471 {
3472 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3473 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3474 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3475 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3476 {
3477 if (pMixedCtx->eflags.Bits.u1IF)
3478 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3479 else
3480 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3481 }
3482 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3483 {
3484 /*
3485 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3486 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3487 */
3488 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3489 }
3490 }
3491
3492 /*
3493 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3494 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3495 * setting this would block host-NMIs and IRET will not clear the blocking.
3496 *
3497 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3498 */
3499 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3500 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3501 {
3502 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3503 }
3504
3505 return uIntrState;
3506}
3507
3508
3509/**
3510 * Loads the guest's interruptibility-state into the guest-state area in the
3511 * VMCS.
3512 *
3513 * @returns VBox status code.
3514 * @param pVCpu The cross context virtual CPU structure.
3515 * @param uIntrState The interruptibility-state to set.
3516 */
3517static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3518{
3519 NOREF(pVCpu);
3520 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3521 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3522 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3523 AssertRC(rc);
3524 return rc;
3525}
3526
3527
3528/**
3529 * Loads the exception intercepts required for guest execution in the VMCS.
3530 *
3531 * @returns VBox status code.
3532 * @param pVCpu The cross context virtual CPU structure.
3533 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3534 * out-of-sync. Make sure to update the required fields
3535 * before using them.
3536 */
3537static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3538{
3539 NOREF(pMixedCtx);
3540 int rc = VINF_SUCCESS;
3541 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3542 {
3543 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3544 if (pVCpu->hm.s.fGIMTrapXcptUD)
3545 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3546#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3547 else
3548 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3549#endif
3550
3551 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3552 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3553
3554 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3555 AssertRCReturn(rc, rc);
3556
3557 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3558 Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3559 pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3560 }
3561 return rc;
3562}
3563
3564
3565/**
3566 * Loads the guest's RIP into the guest-state area in the VMCS.
3567 *
3568 * @returns VBox status code.
3569 * @param pVCpu The cross context virtual CPU structure.
3570 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3571 * out-of-sync. Make sure to update the required fields
3572 * before using them.
3573 *
3574 * @remarks No-long-jump zone!!!
3575 */
3576static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3577{
3578 int rc = VINF_SUCCESS;
3579 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3580 {
3581 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3582 AssertRCReturn(rc, rc);
3583
3584 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3585 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3586 HMCPU_CF_VALUE(pVCpu)));
3587 }
3588 return rc;
3589}
3590
3591
3592/**
3593 * Loads the guest's RSP into the guest-state area in the VMCS.
3594 *
3595 * @returns VBox status code.
3596 * @param pVCpu The cross context virtual CPU structure.
3597 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3598 * out-of-sync. Make sure to update the required fields
3599 * before using them.
3600 *
3601 * @remarks No-long-jump zone!!!
3602 */
3603static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3604{
3605 int rc = VINF_SUCCESS;
3606 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3607 {
3608 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3609 AssertRCReturn(rc, rc);
3610
3611 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3612 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3613 }
3614 return rc;
3615}
3616
3617
3618/**
3619 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3620 *
3621 * @returns VBox status code.
3622 * @param pVCpu The cross context virtual CPU structure.
3623 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3624 * out-of-sync. Make sure to update the required fields
3625 * before using them.
3626 *
3627 * @remarks No-long-jump zone!!!
3628 */
3629static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3630{
3631 int rc = VINF_SUCCESS;
3632 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3633 {
3634 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3635 Let us assert it as such and use 32-bit VMWRITE. */
3636 Assert(!(pMixedCtx->rflags.u64 >> 32));
3637 X86EFLAGS Eflags = pMixedCtx->eflags;
3638 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3639 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3640 * These will never be cleared/set, unless some other part of the VMM
3641 * code is buggy - in which case we're better of finding and fixing
3642 * those bugs than hiding them. */
3643 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3644 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3645 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3646 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3647
3648 /*
3649 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3650 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3651 */
3652 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3653 {
3654 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3655 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3656 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3657 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3658 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3659 }
3660
3661 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3662 AssertRCReturn(rc, rc);
3663
3664 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3665 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3666 }
3667 return rc;
3668}
3669
3670
3671/**
3672 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3673 *
3674 * @returns VBox status code.
3675 * @param pVCpu The cross context virtual CPU structure.
3676 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3677 * out-of-sync. Make sure to update the required fields
3678 * before using them.
3679 *
3680 * @remarks No-long-jump zone!!!
3681 */
3682DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3683{
3684 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3685 rc |= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3686 rc |= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3687 AssertRCReturn(rc, rc);
3688 return rc;
3689}
3690
3691
3692/**
3693 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3694 * CR0 is partially shared with the host and we have to consider the FPU bits.
3695 *
3696 * @returns VBox status code.
3697 * @param pVCpu The cross context virtual CPU structure.
3698 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3699 * out-of-sync. Make sure to update the required fields
3700 * before using them.
3701 *
3702 * @remarks No-long-jump zone!!!
3703 */
3704static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3705{
3706 /*
3707 * Guest CR0.
3708 * Guest FPU.
3709 */
3710 int rc = VINF_SUCCESS;
3711 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3712 {
3713 Assert(!(pMixedCtx->cr0 >> 32));
3714 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3715 PVM pVM = pVCpu->CTX_SUFF(pVM);
3716
3717 /* The guest's view (read access) of its CR0 is unblemished. */
3718 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3719 AssertRCReturn(rc, rc);
3720 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3721
3722 /* Setup VT-x's view of the guest CR0. */
3723 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3724 if (pVM->hm.s.fNestedPaging)
3725 {
3726 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3727 {
3728 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3729 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3730 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3731 }
3732 else
3733 {
3734 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3735 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3736 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3737 }
3738
3739 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3740 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3741 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3742
3743 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3744 AssertRCReturn(rc, rc);
3745 }
3746 else
3747 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3748
3749 /*
3750 * Guest FPU bits.
3751 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3752 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3753 */
3754 u32GuestCR0 |= X86_CR0_NE;
3755 bool fInterceptNM = false;
3756 if (CPUMIsGuestFPUStateActive(pVCpu))
3757 {
3758 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3759 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3760 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3761 }
3762 else
3763 {
3764 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3765 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3766 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3767 }
3768
3769 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3770 bool fInterceptMF = false;
3771 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3772 fInterceptMF = true;
3773
3774 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3775 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3776 {
3777 Assert(PDMVmmDevHeapIsEnabled(pVM));
3778 Assert(pVM->hm.s.vmx.pRealModeTSS);
3779 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3780 fInterceptNM = true;
3781 fInterceptMF = true;
3782 }
3783 else
3784 {
3785 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3786 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3787 }
3788 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3789
3790 if (fInterceptNM)
3791 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3792 else
3793 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3794
3795 if (fInterceptMF)
3796 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3797 else
3798 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3799
3800 /* Additional intercepts for debugging, define these yourself explicitly. */
3801#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3802 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3803 | RT_BIT(X86_XCPT_BP)
3804 | RT_BIT(X86_XCPT_DE)
3805 | RT_BIT(X86_XCPT_NM)
3806 | RT_BIT(X86_XCPT_TS)
3807 | RT_BIT(X86_XCPT_UD)
3808 | RT_BIT(X86_XCPT_NP)
3809 | RT_BIT(X86_XCPT_SS)
3810 | RT_BIT(X86_XCPT_GP)
3811 | RT_BIT(X86_XCPT_PF)
3812 | RT_BIT(X86_XCPT_MF)
3813 ;
3814#elif defined(HMVMX_ALWAYS_TRAP_PF)
3815 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3816#endif
3817
3818 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3819
3820 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3821 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3822 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3823 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3824 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3825 else
3826 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3827
3828 u32GuestCR0 |= uSetCR0;
3829 u32GuestCR0 &= uZapCR0;
3830 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3831
3832 /* Write VT-x's view of the guest CR0 into the VMCS. */
3833 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3834 AssertRCReturn(rc, rc);
3835 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3836 uZapCR0));
3837
3838 /*
3839 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3840 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3841 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3842 */
3843 uint32_t u32CR0Mask = 0;
3844 u32CR0Mask = X86_CR0_PE
3845 | X86_CR0_NE
3846 | X86_CR0_WP
3847 | X86_CR0_PG
3848 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3849 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3850 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3851
3852 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3853 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3854 * and @bugref{6944}. */
3855#if 0
3856 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3857 u32CR0Mask &= ~X86_CR0_PE;
3858#endif
3859 if (pVM->hm.s.fNestedPaging)
3860 u32CR0Mask &= ~X86_CR0_WP;
3861
3862 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3863 if (fInterceptNM)
3864 {
3865 u32CR0Mask |= X86_CR0_TS
3866 | X86_CR0_MP;
3867 }
3868
3869 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3870 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3871 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3872 AssertRCReturn(rc, rc);
3873 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3874
3875 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3876 }
3877 return rc;
3878}
3879
3880
3881/**
3882 * Loads the guest control registers (CR3, CR4) into the guest-state area
3883 * in the VMCS.
3884 *
3885 * @returns VBox strict status code.
3886 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3887 * without unrestricted guest access and the VMMDev is not presently
3888 * mapped (e.g. EFI32).
3889 *
3890 * @param pVCpu The cross context virtual CPU structure.
3891 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3892 * out-of-sync. Make sure to update the required fields
3893 * before using them.
3894 *
3895 * @remarks No-long-jump zone!!!
3896 */
3897static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3898{
3899 int rc = VINF_SUCCESS;
3900 PVM pVM = pVCpu->CTX_SUFF(pVM);
3901
3902 /*
3903 * Guest CR2.
3904 * It's always loaded in the assembler code. Nothing to do here.
3905 */
3906
3907 /*
3908 * Guest CR3.
3909 */
3910 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3911 {
3912 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3913 if (pVM->hm.s.fNestedPaging)
3914 {
3915 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3916
3917 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3918 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3919 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3920 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3921
3922 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3923 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3924 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3925
3926 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3927 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3928 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3929 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3930 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3931 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3932 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3933
3934 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3935 AssertRCReturn(rc, rc);
3936 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3937
3938 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3939 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3940 {
3941 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3942 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3943 {
3944 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3945 AssertRCReturn(rc, rc);
3946 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3947 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3948 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3949 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3950 AssertRCReturn(rc, rc);
3951 }
3952
3953 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3954 have Unrestricted Execution to handle the guest when it's not using paging. */
3955 GCPhysGuestCR3 = pMixedCtx->cr3;
3956 }
3957 else
3958 {
3959 /*
3960 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3961 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3962 * EPT takes care of translating it to host-physical addresses.
3963 */
3964 RTGCPHYS GCPhys;
3965 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3966
3967 /* We obtain it here every time as the guest could have relocated this PCI region. */
3968 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3969 if (RT_SUCCESS(rc))
3970 { /* likely */ }
3971 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3972 {
3973 Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
3974 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3975 }
3976 else
3977 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3978
3979 GCPhysGuestCR3 = GCPhys;
3980 }
3981
3982 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3983 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3984 }
3985 else
3986 {
3987 /* Non-nested paging case, just use the hypervisor's CR3. */
3988 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3989
3990 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
3991 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3992 }
3993 AssertRCReturn(rc, rc);
3994
3995 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3996 }
3997
3998 /*
3999 * Guest CR4.
4000 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
4001 */
4002 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
4003 {
4004 Assert(!(pMixedCtx->cr4 >> 32));
4005 uint32_t u32GuestCR4 = pMixedCtx->cr4;
4006
4007 /* The guest's view of its CR4 is unblemished. */
4008 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4009 AssertRCReturn(rc, rc);
4010 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4011
4012 /* Setup VT-x's view of the guest CR4. */
4013 /*
4014 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4015 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4016 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4017 */
4018 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4019 {
4020 Assert(pVM->hm.s.vmx.pRealModeTSS);
4021 Assert(PDMVmmDevHeapIsEnabled(pVM));
4022 u32GuestCR4 &= ~X86_CR4_VME;
4023 }
4024
4025 if (pVM->hm.s.fNestedPaging)
4026 {
4027 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4028 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4029 {
4030 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4031 u32GuestCR4 |= X86_CR4_PSE;
4032 /* Our identity mapping is a 32-bit page directory. */
4033 u32GuestCR4 &= ~X86_CR4_PAE;
4034 }
4035 /* else use guest CR4.*/
4036 }
4037 else
4038 {
4039 /*
4040 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4041 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4042 */
4043 switch (pVCpu->hm.s.enmShadowMode)
4044 {
4045 case PGMMODE_REAL: /* Real-mode. */
4046 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4047 case PGMMODE_32_BIT: /* 32-bit paging. */
4048 {
4049 u32GuestCR4 &= ~X86_CR4_PAE;
4050 break;
4051 }
4052
4053 case PGMMODE_PAE: /* PAE paging. */
4054 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4055 {
4056 u32GuestCR4 |= X86_CR4_PAE;
4057 break;
4058 }
4059
4060 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4061 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4062#ifdef VBOX_ENABLE_64_BITS_GUESTS
4063 break;
4064#endif
4065 default:
4066 AssertFailed();
4067 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4068 }
4069 }
4070
4071 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4072 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4073 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4074 u32GuestCR4 |= uSetCR4;
4075 u32GuestCR4 &= uZapCR4;
4076
4077 /* Write VT-x's view of the guest CR4 into the VMCS. */
4078 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4079 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4080 AssertRCReturn(rc, rc);
4081
4082 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4083 uint32_t u32CR4Mask = X86_CR4_VME
4084 | X86_CR4_PAE
4085 | X86_CR4_PGE
4086 | X86_CR4_PSE
4087 | X86_CR4_VMXE;
4088 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4089 u32CR4Mask |= X86_CR4_OSXSAVE;
4090 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4091 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4092 AssertRCReturn(rc, rc);
4093
4094 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4095 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4096
4097 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4098 }
4099 return rc;
4100}
4101
4102
4103/**
4104 * Loads the guest debug registers into the guest-state area in the VMCS.
4105 *
4106 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4107 *
4108 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4109 *
4110 * @returns VBox status code.
4111 * @param pVCpu The cross context virtual CPU structure.
4112 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4113 * out-of-sync. Make sure to update the required fields
4114 * before using them.
4115 *
4116 * @remarks No-long-jump zone!!!
4117 */
4118static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4119{
4120 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4121 return VINF_SUCCESS;
4122
4123#ifdef VBOX_STRICT
4124 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4125 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4126 {
4127 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4128 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4129 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4130 }
4131#endif
4132
4133 int rc;
4134 PVM pVM = pVCpu->CTX_SUFF(pVM);
4135 bool fSteppingDB = false;
4136 bool fInterceptMovDRx = false;
4137 if (pVCpu->hm.s.fSingleInstruction)
4138 {
4139 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4140 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4141 {
4142 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4143 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4144 AssertRCReturn(rc, rc);
4145 Assert(fSteppingDB == false);
4146 }
4147 else
4148 {
4149 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4150 pVCpu->hm.s.fClearTrapFlag = true;
4151 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4152 fSteppingDB = true;
4153 }
4154 }
4155
4156 if ( fSteppingDB
4157 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4158 {
4159 /*
4160 * Use the combined guest and host DRx values found in the hypervisor
4161 * register set because the debugger has breakpoints active or someone
4162 * is single stepping on the host side without a monitor trap flag.
4163 *
4164 * Note! DBGF expects a clean DR6 state before executing guest code.
4165 */
4166#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4167 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4168 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4169 {
4170 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4171 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4172 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4173 }
4174 else
4175#endif
4176 if (!CPUMIsHyperDebugStateActive(pVCpu))
4177 {
4178 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4179 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4180 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4181 }
4182
4183 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4184 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4185 AssertRCReturn(rc, rc);
4186
4187 pVCpu->hm.s.fUsingHyperDR7 = true;
4188 fInterceptMovDRx = true;
4189 }
4190 else
4191 {
4192 /*
4193 * If the guest has enabled debug registers, we need to load them prior to
4194 * executing guest code so they'll trigger at the right time.
4195 */
4196 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4197 {
4198#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4199 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4200 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4201 {
4202 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4203 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4204 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4205 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4206 }
4207 else
4208#endif
4209 if (!CPUMIsGuestDebugStateActive(pVCpu))
4210 {
4211 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4212 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4213 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4214 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4215 }
4216 Assert(!fInterceptMovDRx);
4217 }
4218 /*
4219 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4220 * must intercept #DB in order to maintain a correct DR6 guest value, and
4221 * because we need to intercept it to prevent nested #DBs from hanging the
4222 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4223 */
4224#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4225 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4226 && !CPUMIsGuestDebugStateActive(pVCpu))
4227#else
4228 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4229#endif
4230 {
4231 fInterceptMovDRx = true;
4232 }
4233
4234 /* Update guest DR7. */
4235 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4236 AssertRCReturn(rc, rc);
4237
4238 pVCpu->hm.s.fUsingHyperDR7 = false;
4239 }
4240
4241 /*
4242 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4243 */
4244 if (fInterceptMovDRx)
4245 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4246 else
4247 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4248 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4249 AssertRCReturn(rc, rc);
4250
4251 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4252 return VINF_SUCCESS;
4253}
4254
4255
4256#ifdef VBOX_STRICT
4257/**
4258 * Strict function to validate segment registers.
4259 *
4260 * @remarks ASSUMES CR0 is up to date.
4261 */
4262static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4263{
4264 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4265 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4266 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4267 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4268 && ( !CPUMIsGuestInRealModeEx(pCtx)
4269 && !CPUMIsGuestInV86ModeEx(pCtx)))
4270 {
4271 /* Protected mode checks */
4272 /* CS */
4273 Assert(pCtx->cs.Attr.n.u1Present);
4274 Assert(!(pCtx->cs.Attr.u & 0xf00));
4275 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4276 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4277 || !(pCtx->cs.Attr.n.u1Granularity));
4278 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4279 || (pCtx->cs.Attr.n.u1Granularity));
4280 /* CS cannot be loaded with NULL in protected mode. */
4281 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4282 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4283 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4284 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4285 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4286 else
4287 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4288 /* SS */
4289 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4290 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4291 if ( !(pCtx->cr0 & X86_CR0_PE)
4292 || pCtx->cs.Attr.n.u4Type == 3)
4293 {
4294 Assert(!pCtx->ss.Attr.n.u2Dpl);
4295 }
4296 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4297 {
4298 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4299 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4300 Assert(pCtx->ss.Attr.n.u1Present);
4301 Assert(!(pCtx->ss.Attr.u & 0xf00));
4302 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4303 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4304 || !(pCtx->ss.Attr.n.u1Granularity));
4305 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4306 || (pCtx->ss.Attr.n.u1Granularity));
4307 }
4308 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4309 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4310 {
4311 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4312 Assert(pCtx->ds.Attr.n.u1Present);
4313 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4314 Assert(!(pCtx->ds.Attr.u & 0xf00));
4315 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4316 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4317 || !(pCtx->ds.Attr.n.u1Granularity));
4318 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4319 || (pCtx->ds.Attr.n.u1Granularity));
4320 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4321 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4322 }
4323 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4324 {
4325 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4326 Assert(pCtx->es.Attr.n.u1Present);
4327 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4328 Assert(!(pCtx->es.Attr.u & 0xf00));
4329 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4330 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4331 || !(pCtx->es.Attr.n.u1Granularity));
4332 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4333 || (pCtx->es.Attr.n.u1Granularity));
4334 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4335 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4336 }
4337 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4338 {
4339 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4340 Assert(pCtx->fs.Attr.n.u1Present);
4341 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4342 Assert(!(pCtx->fs.Attr.u & 0xf00));
4343 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4344 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4345 || !(pCtx->fs.Attr.n.u1Granularity));
4346 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4347 || (pCtx->fs.Attr.n.u1Granularity));
4348 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4349 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4350 }
4351 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4352 {
4353 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4354 Assert(pCtx->gs.Attr.n.u1Present);
4355 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4356 Assert(!(pCtx->gs.Attr.u & 0xf00));
4357 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4358 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4359 || !(pCtx->gs.Attr.n.u1Granularity));
4360 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4361 || (pCtx->gs.Attr.n.u1Granularity));
4362 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4363 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4364 }
4365 /* 64-bit capable CPUs. */
4366# if HC_ARCH_BITS == 64
4367 Assert(!(pCtx->cs.u64Base >> 32));
4368 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4369 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4370 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4371# endif
4372 }
4373 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4374 || ( CPUMIsGuestInRealModeEx(pCtx)
4375 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4376 {
4377 /* Real and v86 mode checks. */
4378 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4379 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4380 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4381 {
4382 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4383 }
4384 else
4385 {
4386 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4387 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4388 }
4389
4390 /* CS */
4391 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4392 Assert(pCtx->cs.u32Limit == 0xffff);
4393 Assert(u32CSAttr == 0xf3);
4394 /* SS */
4395 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4396 Assert(pCtx->ss.u32Limit == 0xffff);
4397 Assert(u32SSAttr == 0xf3);
4398 /* DS */
4399 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4400 Assert(pCtx->ds.u32Limit == 0xffff);
4401 Assert(u32DSAttr == 0xf3);
4402 /* ES */
4403 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4404 Assert(pCtx->es.u32Limit == 0xffff);
4405 Assert(u32ESAttr == 0xf3);
4406 /* FS */
4407 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4408 Assert(pCtx->fs.u32Limit == 0xffff);
4409 Assert(u32FSAttr == 0xf3);
4410 /* GS */
4411 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4412 Assert(pCtx->gs.u32Limit == 0xffff);
4413 Assert(u32GSAttr == 0xf3);
4414 /* 64-bit capable CPUs. */
4415# if HC_ARCH_BITS == 64
4416 Assert(!(pCtx->cs.u64Base >> 32));
4417 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4418 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4419 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4420# endif
4421 }
4422}
4423#endif /* VBOX_STRICT */
4424
4425
4426/**
4427 * Writes a guest segment register into the guest-state area in the VMCS.
4428 *
4429 * @returns VBox status code.
4430 * @param pVCpu The cross context virtual CPU structure.
4431 * @param idxSel Index of the selector in the VMCS.
4432 * @param idxLimit Index of the segment limit in the VMCS.
4433 * @param idxBase Index of the segment base in the VMCS.
4434 * @param idxAccess Index of the access rights of the segment in the VMCS.
4435 * @param pSelReg Pointer to the segment selector.
4436 *
4437 * @remarks No-long-jump zone!!!
4438 */
4439static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4440 uint32_t idxAccess, PCPUMSELREG pSelReg)
4441{
4442 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4443 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4444 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4445 AssertRCReturn(rc, rc);
4446
4447 uint32_t u32Access = pSelReg->Attr.u;
4448 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4449 {
4450 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4451 u32Access = 0xf3;
4452 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4453 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4454 }
4455 else
4456 {
4457 /*
4458 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4459 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4460 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4461 * loaded in protected-mode have their attribute as 0.
4462 */
4463 if (!u32Access)
4464 u32Access = X86DESCATTR_UNUSABLE;
4465 }
4466
4467 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4468 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4469 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4470
4471 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4472 AssertRCReturn(rc, rc);
4473 return rc;
4474}
4475
4476
4477/**
4478 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4479 * into the guest-state area in the VMCS.
4480 *
4481 * @returns VBox status code.
4482 * @param pVCpu The cross context virtual CPU structure.
4483 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4484 * out-of-sync. Make sure to update the required fields
4485 * before using them.
4486 *
4487 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4488 * @remarks No-long-jump zone!!!
4489 */
4490static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4491{
4492 int rc = VERR_INTERNAL_ERROR_5;
4493 PVM pVM = pVCpu->CTX_SUFF(pVM);
4494
4495 /*
4496 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4497 */
4498 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4499 {
4500 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4501 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4502 {
4503 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4504 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4505 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4506 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4507 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4508 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4509 }
4510
4511#ifdef VBOX_WITH_REM
4512 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4513 {
4514 Assert(pVM->hm.s.vmx.pRealModeTSS);
4515 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4516 if ( pVCpu->hm.s.vmx.fWasInRealMode
4517 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4518 {
4519 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4520 in real-mode (e.g. OpenBSD 4.0) */
4521 REMFlushTBs(pVM);
4522 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4523 pVCpu->hm.s.vmx.fWasInRealMode = false;
4524 }
4525 }
4526#endif
4527 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4528 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4529 AssertRCReturn(rc, rc);
4530 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4531 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4532 AssertRCReturn(rc, rc);
4533 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4534 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4535 AssertRCReturn(rc, rc);
4536 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4537 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4538 AssertRCReturn(rc, rc);
4539 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4540 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4541 AssertRCReturn(rc, rc);
4542 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4543 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4544 AssertRCReturn(rc, rc);
4545
4546#ifdef VBOX_STRICT
4547 /* Validate. */
4548 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4549#endif
4550
4551 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4552 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4553 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4554 }
4555
4556 /*
4557 * Guest TR.
4558 */
4559 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4560 {
4561 /*
4562 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4563 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4564 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4565 */
4566 uint16_t u16Sel = 0;
4567 uint32_t u32Limit = 0;
4568 uint64_t u64Base = 0;
4569 uint32_t u32AccessRights = 0;
4570
4571 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4572 {
4573 u16Sel = pMixedCtx->tr.Sel;
4574 u32Limit = pMixedCtx->tr.u32Limit;
4575 u64Base = pMixedCtx->tr.u64Base;
4576 u32AccessRights = pMixedCtx->tr.Attr.u;
4577 }
4578 else
4579 {
4580 Assert(pVM->hm.s.vmx.pRealModeTSS);
4581 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4582
4583 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4584 RTGCPHYS GCPhys;
4585 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4586 AssertRCReturn(rc, rc);
4587
4588 X86DESCATTR DescAttr;
4589 DescAttr.u = 0;
4590 DescAttr.n.u1Present = 1;
4591 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4592
4593 u16Sel = 0;
4594 u32Limit = HM_VTX_TSS_SIZE;
4595 u64Base = GCPhys; /* in real-mode phys = virt. */
4596 u32AccessRights = DescAttr.u;
4597 }
4598
4599 /* Validate. */
4600 Assert(!(u16Sel & RT_BIT(2)));
4601 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4602 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4603 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4604 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4605 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4606 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4607 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4608 Assert( (u32Limit & 0xfff) == 0xfff
4609 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4610 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4611 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4612
4613 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4614 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4615 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4616 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4617 AssertRCReturn(rc, rc);
4618
4619 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4620 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4621 }
4622
4623 /*
4624 * Guest GDTR.
4625 */
4626 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4627 {
4628 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4629 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4630 AssertRCReturn(rc, rc);
4631
4632 /* Validate. */
4633 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4634
4635 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4636 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4637 }
4638
4639 /*
4640 * Guest LDTR.
4641 */
4642 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4643 {
4644 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4645 uint32_t u32Access = 0;
4646 if (!pMixedCtx->ldtr.Attr.u)
4647 u32Access = X86DESCATTR_UNUSABLE;
4648 else
4649 u32Access = pMixedCtx->ldtr.Attr.u;
4650
4651 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4652 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4653 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4654 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4655 AssertRCReturn(rc, rc);
4656
4657 /* Validate. */
4658 if (!(u32Access & X86DESCATTR_UNUSABLE))
4659 {
4660 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4661 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4662 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4663 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4664 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4665 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4666 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4667 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4668 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4669 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4670 }
4671
4672 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4673 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4674 }
4675
4676 /*
4677 * Guest IDTR.
4678 */
4679 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4680 {
4681 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4682 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4683 AssertRCReturn(rc, rc);
4684
4685 /* Validate. */
4686 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4687
4688 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4689 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4690 }
4691
4692 return VINF_SUCCESS;
4693}
4694
4695
4696/**
4697 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4698 * areas.
4699 *
4700 * These MSRs will automatically be loaded to the host CPU on every successful
4701 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4702 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4703 * -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4704 *
4705 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4706 *
4707 * @returns VBox status code.
4708 * @param pVCpu The cross context virtual CPU structure.
4709 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4710 * out-of-sync. Make sure to update the required fields
4711 * before using them.
4712 *
4713 * @remarks No-long-jump zone!!!
4714 */
4715static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4716{
4717 AssertPtr(pVCpu);
4718 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4719
4720 /*
4721 * MSRs that we use the auto-load/store MSR area in the VMCS.
4722 */
4723 PVM pVM = pVCpu->CTX_SUFF(pVM);
4724 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4725 {
4726 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4727#if HC_ARCH_BITS == 32
4728 if (pVM->hm.s.fAllow64BitGuests)
4729 {
4730 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4731 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4732 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4733 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4734 AssertRCReturn(rc, rc);
4735# ifdef LOG_ENABLED
4736 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4737 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4738 {
4739 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4740 pMsr->u64Value));
4741 }
4742# endif
4743 }
4744#endif
4745 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4746 }
4747
4748 /*
4749 * Guest Sysenter MSRs.
4750 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4751 * VM-exits on WRMSRs for these MSRs.
4752 */
4753 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4754 {
4755 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4756 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4757 }
4758
4759 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4760 {
4761 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4762 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4763 }
4764
4765 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4766 {
4767 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4768 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4769 }
4770
4771 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4772 {
4773 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4774 {
4775 /*
4776 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4777 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4778 */
4779 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4780 {
4781 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4782 AssertRCReturn(rc,rc);
4783 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4784 }
4785 else
4786 {
4787 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4788 NULL /* pfAddedAndUpdated */);
4789 AssertRCReturn(rc, rc);
4790
4791 /* We need to intercept reads too, see @bugref{7386#c16}. */
4792 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4793 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4794 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4795 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4796 }
4797 }
4798 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4799 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4800 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4801 }
4802
4803 return VINF_SUCCESS;
4804}
4805
4806
4807/**
4808 * Loads the guest activity state into the guest-state area in the VMCS.
4809 *
4810 * @returns VBox status code.
4811 * @param pVCpu The cross context virtual CPU structure.
4812 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4813 * out-of-sync. Make sure to update the required fields
4814 * before using them.
4815 *
4816 * @remarks No-long-jump zone!!!
4817 */
4818static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4819{
4820 NOREF(pMixedCtx);
4821 /** @todo See if we can make use of other states, e.g.
4822 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4823 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4824 {
4825 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4826 AssertRCReturn(rc, rc);
4827
4828 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4829 }
4830 return VINF_SUCCESS;
4831}
4832
4833
4834#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4835/**
4836 * Check if guest state allows safe use of 32-bit switcher again.
4837 *
4838 * Segment bases and protected mode structures must be 32-bit addressable
4839 * because the 32-bit switcher will ignore high dword when writing these VMCS
4840 * fields. See @bugref{8432} for details.
4841 *
4842 * @returns true if safe, false if must continue to use the 64-bit switcher.
4843 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4844 * out-of-sync. Make sure to update the required fields
4845 * before using them.
4846 *
4847 * @remarks No-long-jump zone!!!
4848 */
4849static bool hmR0VmxIs32BitSwitcherSafe(PCPUMCTX pMixedCtx)
4850{
4851 if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000))
4852 return false;
4853 if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000))
4854 return false;
4855 if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000))
4856 return false;
4857 if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000))
4858 return false;
4859 if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000))
4860 return false;
4861 if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000))
4862 return false;
4863 if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000))
4864 return false;
4865 if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000))
4866 return false;
4867 if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000))
4868 return false;
4869 if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000))
4870 return false;
4871 /* All good, bases are 32-bit. */
4872 return true;
4873}
4874#endif
4875
4876
4877/**
4878 * Sets up the appropriate function to run guest code.
4879 *
4880 * @returns VBox status code.
4881 * @param pVCpu The cross context virtual CPU structure.
4882 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4883 * out-of-sync. Make sure to update the required fields
4884 * before using them.
4885 *
4886 * @remarks No-long-jump zone!!!
4887 */
4888static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4889{
4890 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4891 {
4892#ifndef VBOX_ENABLE_64_BITS_GUESTS
4893 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4894#endif
4895 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4896#if HC_ARCH_BITS == 32
4897 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4898 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4899 {
4900 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4901 {
4902 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4903 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4904 | HM_CHANGED_VMX_ENTRY_CTLS
4905 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4906 }
4907 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4908
4909 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4910 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4911 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4912 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 64-bit switcher\n", pVCpu->idCpu));
4913 }
4914#else
4915 /* 64-bit host. */
4916 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4917#endif
4918 }
4919 else
4920 {
4921 /* Guest is not in long mode, use the 32-bit handler. */
4922#if HC_ARCH_BITS == 32
4923 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4924 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4925 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4926 {
4927 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4928 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4929 | HM_CHANGED_VMX_ENTRY_CTLS
4930 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4931 }
4932# ifdef VBOX_ENABLE_64_BITS_GUESTS
4933 /*
4934 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel design, see @bugref{8432#c7}.
4935 * If real-on-v86 mode is active, clear the 64-bit switcher flag because now we know the guest is in a sane
4936 * state where it's safe to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4937 * the much faster 32-bit switcher again.
4938 */
4939 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4940 {
4941 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4942 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher\n", pVCpu->idCpu));
4943 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4944 }
4945 else
4946 {
4947 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4948 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4949 || hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
4950 {
4951 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4952 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4953 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR
4954 | HM_CHANGED_VMX_ENTRY_CTLS
4955 | HM_CHANGED_VMX_EXIT_CTLS
4956 | HM_CHANGED_HOST_CONTEXT);
4957 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher (safe)\n", pVCpu->idCpu));
4958 }
4959 }
4960# else
4961 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4962# endif
4963#else
4964 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4965#endif
4966 }
4967 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4968 return VINF_SUCCESS;
4969}
4970
4971
4972/**
4973 * Wrapper for running the guest code in VT-x.
4974 *
4975 * @returns VBox status code, no informational status codes.
4976 * @param pVM The cross context VM structure.
4977 * @param pVCpu The cross context virtual CPU structure.
4978 * @param pCtx Pointer to the guest-CPU context.
4979 *
4980 * @remarks No-long-jump zone!!!
4981 */
4982DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4983{
4984 /*
4985 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4986 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4987 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4988 */
4989 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4990 /** @todo Add stats for resume vs launch. */
4991#ifdef VBOX_WITH_KERNEL_USING_XMM
4992 int rc = HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4993#else
4994 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4995#endif
4996 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4997 return rc;
4998}
4999
5000
5001/**
5002 * Reports world-switch error and dumps some useful debug info.
5003 *
5004 * @param pVM The cross context VM structure.
5005 * @param pVCpu The cross context virtual CPU structure.
5006 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5007 * @param pCtx Pointer to the guest-CPU context.
5008 * @param pVmxTransient Pointer to the VMX transient structure (only
5009 * exitReason updated).
5010 */
5011static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
5012{
5013 Assert(pVM);
5014 Assert(pVCpu);
5015 Assert(pCtx);
5016 Assert(pVmxTransient);
5017 HMVMX_ASSERT_PREEMPT_SAFE();
5018
5019 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
5020 switch (rcVMRun)
5021 {
5022 case VERR_VMX_INVALID_VMXON_PTR:
5023 AssertFailed();
5024 break;
5025 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5026 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5027 {
5028 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5029 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5030 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5031 AssertRC(rc);
5032
5033 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5034 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5035 Cannot do it here as we may have been long preempted. */
5036
5037#ifdef VBOX_STRICT
5038 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5039 pVmxTransient->uExitReason));
5040 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5041 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5042 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5043 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5044 else
5045 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5046 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5047 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5048
5049 /* VMX control bits. */
5050 uint32_t u32Val;
5051 uint64_t u64Val;
5052 RTHCUINTREG uHCReg;
5053 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5054 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5055 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5056 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5057 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5058 {
5059 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5060 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5061 }
5062 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5063 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5064 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5065 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5066 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5067 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5068 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5069 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5070 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5071 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5072 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5073 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5074 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5075 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5076 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5077 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5078 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5079 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5080 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5081 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5082 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5083 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5084 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5085 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5086 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5087 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5088 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5089 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5090 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5091 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5092 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5093 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5094 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5095 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5096 if (pVM->hm.s.fNestedPaging)
5097 {
5098 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5099 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5100 }
5101
5102 /* Guest bits. */
5103 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5104 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5105 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5106 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5107 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5108 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5109 if (pVM->hm.s.vmx.fVpid)
5110 {
5111 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5112 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5113 }
5114
5115 /* Host bits. */
5116 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5117 Log4(("Host CR0 %#RHr\n", uHCReg));
5118 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5119 Log4(("Host CR3 %#RHr\n", uHCReg));
5120 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5121 Log4(("Host CR4 %#RHr\n", uHCReg));
5122
5123 RTGDTR HostGdtr;
5124 PCX86DESCHC pDesc;
5125 ASMGetGDTR(&HostGdtr);
5126 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5127 Log4(("Host CS %#08x\n", u32Val));
5128 if (u32Val < HostGdtr.cbGdt)
5129 {
5130 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5131 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5132 }
5133
5134 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5135 Log4(("Host DS %#08x\n", u32Val));
5136 if (u32Val < HostGdtr.cbGdt)
5137 {
5138 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5139 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5140 }
5141
5142 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5143 Log4(("Host ES %#08x\n", u32Val));
5144 if (u32Val < HostGdtr.cbGdt)
5145 {
5146 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5147 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5148 }
5149
5150 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5151 Log4(("Host FS %#08x\n", u32Val));
5152 if (u32Val < HostGdtr.cbGdt)
5153 {
5154 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5155 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5156 }
5157
5158 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5159 Log4(("Host GS %#08x\n", u32Val));
5160 if (u32Val < HostGdtr.cbGdt)
5161 {
5162 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5163 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5164 }
5165
5166 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5167 Log4(("Host SS %#08x\n", u32Val));
5168 if (u32Val < HostGdtr.cbGdt)
5169 {
5170 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5171 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5172 }
5173
5174 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5175 Log4(("Host TR %#08x\n", u32Val));
5176 if (u32Val < HostGdtr.cbGdt)
5177 {
5178 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5179 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5180 }
5181
5182 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5183 Log4(("Host TR Base %#RHv\n", uHCReg));
5184 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5185 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5186 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5187 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5188 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5189 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5190 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5191 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5192 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5193 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5194 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5195 Log4(("Host RSP %#RHv\n", uHCReg));
5196 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5197 Log4(("Host RIP %#RHv\n", uHCReg));
5198# if HC_ARCH_BITS == 64
5199 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5200 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5201 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5202 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5203 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5204 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5205# endif
5206#endif /* VBOX_STRICT */
5207 break;
5208 }
5209
5210 default:
5211 /* Impossible */
5212 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5213 break;
5214 }
5215 NOREF(pVM); NOREF(pCtx);
5216}
5217
5218
5219#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5220#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5221# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5222#endif
5223#ifdef VBOX_STRICT
5224static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5225{
5226 switch (idxField)
5227 {
5228 case VMX_VMCS_GUEST_RIP:
5229 case VMX_VMCS_GUEST_RSP:
5230 case VMX_VMCS_GUEST_SYSENTER_EIP:
5231 case VMX_VMCS_GUEST_SYSENTER_ESP:
5232 case VMX_VMCS_GUEST_GDTR_BASE:
5233 case VMX_VMCS_GUEST_IDTR_BASE:
5234 case VMX_VMCS_GUEST_CS_BASE:
5235 case VMX_VMCS_GUEST_DS_BASE:
5236 case VMX_VMCS_GUEST_ES_BASE:
5237 case VMX_VMCS_GUEST_FS_BASE:
5238 case VMX_VMCS_GUEST_GS_BASE:
5239 case VMX_VMCS_GUEST_SS_BASE:
5240 case VMX_VMCS_GUEST_LDTR_BASE:
5241 case VMX_VMCS_GUEST_TR_BASE:
5242 case VMX_VMCS_GUEST_CR3:
5243 return true;
5244 }
5245 return false;
5246}
5247
5248static bool hmR0VmxIsValidReadField(uint32_t idxField)
5249{
5250 switch (idxField)
5251 {
5252 /* Read-only fields. */
5253 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5254 return true;
5255 }
5256 /* Remaining readable fields should also be writable. */
5257 return hmR0VmxIsValidWriteField(idxField);
5258}
5259#endif /* VBOX_STRICT */
5260
5261
5262/**
5263 * Executes the specified handler in 64-bit mode.
5264 *
5265 * @returns VBox status code (no informational status codes).
5266 * @param pVM The cross context VM structure.
5267 * @param pVCpu The cross context virtual CPU structure.
5268 * @param pCtx Pointer to the guest CPU context.
5269 * @param enmOp The operation to perform.
5270 * @param cParams Number of parameters.
5271 * @param paParam Array of 32-bit parameters.
5272 */
5273VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5274 uint32_t cParams, uint32_t *paParam)
5275{
5276 NOREF(pCtx);
5277
5278 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5279 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5280 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5281 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5282
5283#ifdef VBOX_STRICT
5284 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5285 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5286
5287 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5288 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5289#endif
5290
5291 /* Disable interrupts. */
5292 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5293
5294#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5295 RTCPUID idHostCpu = RTMpCpuId();
5296 CPUMR0SetLApic(pVCpu, idHostCpu);
5297#endif
5298
5299 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5300 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5301
5302 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5303 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5304 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5305
5306 /* Leave VMX Root Mode. */
5307 VMXDisable();
5308
5309 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5310
5311 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5312 CPUMSetHyperEIP(pVCpu, enmOp);
5313 for (int i = (int)cParams - 1; i >= 0; i--)
5314 CPUMPushHyper(pVCpu, paParam[i]);
5315
5316 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5317
5318 /* Call the switcher. */
5319 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5320 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5321
5322 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5323 /* Make sure the VMX instructions don't cause #UD faults. */
5324 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5325
5326 /* Re-enter VMX Root Mode */
5327 int rc2 = VMXEnable(HCPhysCpuPage);
5328 if (RT_FAILURE(rc2))
5329 {
5330 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5331 ASMSetFlags(fOldEFlags);
5332 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5333 return rc2;
5334 }
5335
5336 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5337 AssertRC(rc2);
5338 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5339 Assert(!(ASMGetFlags() & X86_EFL_IF));
5340 ASMSetFlags(fOldEFlags);
5341 return rc;
5342}
5343
5344
5345/**
5346 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5347 * supporting 64-bit guests.
5348 *
5349 * @returns VBox status code.
5350 * @param fResume Whether to VMLAUNCH or VMRESUME.
5351 * @param pCtx Pointer to the guest-CPU context.
5352 * @param pCache Pointer to the VMCS cache.
5353 * @param pVM The cross context VM structure.
5354 * @param pVCpu The cross context virtual CPU structure.
5355 */
5356DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5357{
5358 NOREF(fResume);
5359
5360 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5361 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5362
5363#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5364 pCache->uPos = 1;
5365 pCache->interPD = PGMGetInterPaeCR3(pVM);
5366 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5367#endif
5368
5369#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5370 pCache->TestIn.HCPhysCpuPage = 0;
5371 pCache->TestIn.HCPhysVmcs = 0;
5372 pCache->TestIn.pCache = 0;
5373 pCache->TestOut.HCPhysVmcs = 0;
5374 pCache->TestOut.pCache = 0;
5375 pCache->TestOut.pCtx = 0;
5376 pCache->TestOut.eflags = 0;
5377#else
5378 NOREF(pCache);
5379#endif
5380
5381 uint32_t aParam[10];
5382 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5383 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5384 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5385 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5386 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5387 aParam[5] = 0;
5388 aParam[6] = VM_RC_ADDR(pVM, pVM);
5389 aParam[7] = 0;
5390 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5391 aParam[9] = 0;
5392
5393#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5394 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5395 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5396#endif
5397 int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5398
5399#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5400 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5401 Assert(pCtx->dr[4] == 10);
5402 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5403#endif
5404
5405#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5406 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5407 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5408 pVCpu->hm.s.vmx.HCPhysVmcs));
5409 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5410 pCache->TestOut.HCPhysVmcs));
5411 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5412 pCache->TestOut.pCache));
5413 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5414 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5415 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5416 pCache->TestOut.pCtx));
5417 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5418#endif
5419 return rc;
5420}
5421
5422
5423/**
5424 * Initialize the VMCS-Read cache.
5425 *
5426 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5427 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5428 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5429 * (those that have a 32-bit FULL & HIGH part).
5430 *
5431 * @returns VBox status code.
5432 * @param pVM The cross context VM structure.
5433 * @param pVCpu The cross context virtual CPU structure.
5434 */
5435static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5436{
5437#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5438{ \
5439 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5440 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5441 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5442 ++cReadFields; \
5443}
5444
5445 AssertPtr(pVM);
5446 AssertPtr(pVCpu);
5447 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5448 uint32_t cReadFields = 0;
5449
5450 /*
5451 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5452 * and serve to indicate exceptions to the rules.
5453 */
5454
5455 /* Guest-natural selector base fields. */
5456#if 0
5457 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5458 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5459 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5460#endif
5461 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5462 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5463 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5464 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5465 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5466 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5467 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5468 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5469 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5470 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5471 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5472 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5473#if 0
5474 /* Unused natural width guest-state fields. */
5475 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5476 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5477#endif
5478 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5479 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5480
5481 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5482#if 0
5483 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5484 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5485 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5486 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5487 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5488 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5489 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5490 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5491 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5492#endif
5493
5494 /* Natural width guest-state fields. */
5495 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5496#if 0
5497 /* Currently unused field. */
5498 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5499#endif
5500
5501 if (pVM->hm.s.fNestedPaging)
5502 {
5503 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5504 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5505 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5506 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5507 }
5508 else
5509 {
5510 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5511 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5512 }
5513
5514#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5515 return VINF_SUCCESS;
5516}
5517
5518
5519/**
5520 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5521 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5522 * darwin, running 64-bit guests).
5523 *
5524 * @returns VBox status code.
5525 * @param pVCpu The cross context virtual CPU structure.
5526 * @param idxField The VMCS field encoding.
5527 * @param u64Val 16, 32 or 64-bit value.
5528 */
5529VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5530{
5531 int rc;
5532 switch (idxField)
5533 {
5534 /*
5535 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5536 */
5537 /* 64-bit Control fields. */
5538 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5539 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5540 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5541 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5542 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5543 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5544 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5545 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5546 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5547 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5548 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5549 case VMX_VMCS64_CTRL_EPTP_FULL:
5550 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5551 /* 64-bit Guest-state fields. */
5552 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5553 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5554 case VMX_VMCS64_GUEST_PAT_FULL:
5555 case VMX_VMCS64_GUEST_EFER_FULL:
5556 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5557 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5558 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5559 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5560 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5561 /* 64-bit Host-state fields. */
5562 case VMX_VMCS64_HOST_PAT_FULL:
5563 case VMX_VMCS64_HOST_EFER_FULL:
5564 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5565 {
5566 rc = VMXWriteVmcs32(idxField, u64Val);
5567 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5568 break;
5569 }
5570
5571 /*
5572 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5573 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5574 */
5575 /* Natural-width Guest-state fields. */
5576 case VMX_VMCS_GUEST_CR3:
5577 case VMX_VMCS_GUEST_ES_BASE:
5578 case VMX_VMCS_GUEST_CS_BASE:
5579 case VMX_VMCS_GUEST_SS_BASE:
5580 case VMX_VMCS_GUEST_DS_BASE:
5581 case VMX_VMCS_GUEST_FS_BASE:
5582 case VMX_VMCS_GUEST_GS_BASE:
5583 case VMX_VMCS_GUEST_LDTR_BASE:
5584 case VMX_VMCS_GUEST_TR_BASE:
5585 case VMX_VMCS_GUEST_GDTR_BASE:
5586 case VMX_VMCS_GUEST_IDTR_BASE:
5587 case VMX_VMCS_GUEST_RSP:
5588 case VMX_VMCS_GUEST_RIP:
5589 case VMX_VMCS_GUEST_SYSENTER_ESP:
5590 case VMX_VMCS_GUEST_SYSENTER_EIP:
5591 {
5592 if (!(u64Val >> 32))
5593 {
5594 /* If this field is 64-bit, VT-x will zero out the top bits. */
5595 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5596 }
5597 else
5598 {
5599 /* Assert that only the 32->64 switcher case should ever come here. */
5600 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5601 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5602 }
5603 break;
5604 }
5605
5606 default:
5607 {
5608 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5609 rc = VERR_INVALID_PARAMETER;
5610 break;
5611 }
5612 }
5613 AssertRCReturn(rc, rc);
5614 return rc;
5615}
5616
5617
5618/**
5619 * Queue up a VMWRITE by using the VMCS write cache.
5620 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5621 *
5622 * @param pVCpu The cross context virtual CPU structure.
5623 * @param idxField The VMCS field encoding.
5624 * @param u64Val 16, 32 or 64-bit value.
5625 */
5626VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5627{
5628 AssertPtr(pVCpu);
5629 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5630
5631 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5632 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5633
5634 /* Make sure there are no duplicates. */
5635 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5636 {
5637 if (pCache->Write.aField[i] == idxField)
5638 {
5639 pCache->Write.aFieldVal[i] = u64Val;
5640 return VINF_SUCCESS;
5641 }
5642 }
5643
5644 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5645 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5646 pCache->Write.cValidEntries++;
5647 return VINF_SUCCESS;
5648}
5649#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5650
5651
5652/**
5653 * Sets up the usage of TSC-offsetting and updates the VMCS.
5654 *
5655 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5656 * VMX preemption timer.
5657 *
5658 * @returns VBox status code.
5659 * @param pVM The cross context VM structure.
5660 * @param pVCpu The cross context virtual CPU structure.
5661 *
5662 * @remarks No-long-jump zone!!!
5663 */
5664static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5665{
5666 int rc;
5667 bool fOffsettedTsc;
5668 bool fParavirtTsc;
5669 if (pVM->hm.s.vmx.fUsePreemptTimer)
5670 {
5671 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5672 &fOffsettedTsc, &fParavirtTsc);
5673
5674 /* Make sure the returned values have sane upper and lower boundaries. */
5675 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5676 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5677 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5678 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5679
5680 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5681 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5682 }
5683 else
5684 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5685
5686 /** @todo later optimize this to be done elsewhere and not before every
5687 * VM-entry. */
5688 if (fParavirtTsc)
5689 {
5690 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5691 information before every VM-entry, hence disable it for performance sake. */
5692#if 0
5693 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5694 AssertRC(rc);
5695#endif
5696 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5697 }
5698
5699 if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5700 {
5701 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5702 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5703
5704 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5705 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5706 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5707 }
5708 else
5709 {
5710 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5711 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5712 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5713 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5714 }
5715}
5716
5717
5718/**
5719 * Determines if an exception is a contributory exception.
5720 *
5721 * Contributory exceptions are ones which can cause double-faults unless the
5722 * original exception was a benign exception. Page-fault is intentionally not
5723 * included here as it's a conditional contributory exception.
5724 *
5725 * @returns true if the exception is contributory, false otherwise.
5726 * @param uVector The exception vector.
5727 */
5728DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5729{
5730 switch (uVector)
5731 {
5732 case X86_XCPT_GP:
5733 case X86_XCPT_SS:
5734 case X86_XCPT_NP:
5735 case X86_XCPT_TS:
5736 case X86_XCPT_DE:
5737 return true;
5738 default:
5739 break;
5740 }
5741 return false;
5742}
5743
5744
5745/**
5746 * Sets an event as a pending event to be injected into the guest.
5747 *
5748 * @param pVCpu The cross context virtual CPU structure.
5749 * @param u32IntInfo The VM-entry interruption-information field.
5750 * @param cbInstr The VM-entry instruction length in bytes (for software
5751 * interrupts, exceptions and privileged software
5752 * exceptions).
5753 * @param u32ErrCode The VM-entry exception error code.
5754 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5755 * page-fault.
5756 *
5757 * @remarks Statistics counter assumes this is a guest event being injected or
5758 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5759 * always incremented.
5760 */
5761DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5762 RTGCUINTPTR GCPtrFaultAddress)
5763{
5764 Assert(!pVCpu->hm.s.Event.fPending);
5765 pVCpu->hm.s.Event.fPending = true;
5766 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5767 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5768 pVCpu->hm.s.Event.cbInstr = cbInstr;
5769 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5770}
5771
5772
5773/**
5774 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5775 *
5776 * @param pVCpu The cross context virtual CPU structure.
5777 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5778 * out-of-sync. Make sure to update the required fields
5779 * before using them.
5780 */
5781DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5782{
5783 NOREF(pMixedCtx);
5784 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5785 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5786 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5787 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5788}
5789
5790
5791/**
5792 * Handle a condition that occurred while delivering an event through the guest
5793 * IDT.
5794 *
5795 * @returns Strict VBox status code (i.e. informational status codes too).
5796 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5797 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5798 * to continue execution of the guest which will delivery the \#DF.
5799 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5800 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5801 *
5802 * @param pVCpu The cross context virtual CPU structure.
5803 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5804 * out-of-sync. Make sure to update the required fields
5805 * before using them.
5806 * @param pVmxTransient Pointer to the VMX transient structure.
5807 *
5808 * @remarks No-long-jump zone!!!
5809 */
5810static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5811{
5812 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5813
5814 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5815 rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5816
5817 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5818 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5819 {
5820 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5821 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5822
5823 typedef enum
5824 {
5825 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5826 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5827 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5828 VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
5829 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5830 } VMXREFLECTXCPT;
5831
5832 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5833 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5834 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5835 {
5836 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5837 {
5838 enmReflect = VMXREFLECTXCPT_XCPT;
5839#ifdef VBOX_STRICT
5840 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5841 && uExitVector == X86_XCPT_PF)
5842 {
5843 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5844 }
5845#endif
5846 if ( uExitVector == X86_XCPT_PF
5847 && uIdtVector == X86_XCPT_PF)
5848 {
5849 pVmxTransient->fVectoringDoublePF = true;
5850 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5851 }
5852 else if ( uExitVector == X86_XCPT_AC
5853 && uIdtVector == X86_XCPT_AC)
5854 {
5855 enmReflect = VMXREFLECTXCPT_HANG;
5856 Log4(("IDT: Nested #AC - Bad guest\n"));
5857 }
5858 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5859 && hmR0VmxIsContributoryXcpt(uExitVector)
5860 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5861 || uIdtVector == X86_XCPT_PF))
5862 {
5863 enmReflect = VMXREFLECTXCPT_DF;
5864 }
5865 else if (uIdtVector == X86_XCPT_DF)
5866 enmReflect = VMXREFLECTXCPT_TF;
5867 }
5868 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5869 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5870 {
5871 /*
5872 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5873 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5874 */
5875 enmReflect = VMXREFLECTXCPT_XCPT;
5876
5877 if (uExitVector == X86_XCPT_PF)
5878 {
5879 pVmxTransient->fVectoringPF = true;
5880 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5881 }
5882 }
5883 }
5884 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5885 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5886 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5887 {
5888 /*
5889 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5890 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5891 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5892 */
5893 enmReflect = VMXREFLECTXCPT_XCPT;
5894 }
5895
5896 /*
5897 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5898 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5899 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5900 *
5901 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5902 */
5903 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5904 && enmReflect == VMXREFLECTXCPT_XCPT
5905 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5906 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5907 {
5908 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5909 }
5910
5911 switch (enmReflect)
5912 {
5913 case VMXREFLECTXCPT_XCPT:
5914 {
5915 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5916 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5917 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5918
5919 uint32_t u32ErrCode = 0;
5920 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5921 {
5922 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5923 AssertRCReturn(rc2, rc2);
5924 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5925 }
5926
5927 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5928 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5929 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5930 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5931 rcStrict = VINF_SUCCESS;
5932 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5933 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5934
5935 break;
5936 }
5937
5938 case VMXREFLECTXCPT_DF:
5939 {
5940 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5941 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5942 rcStrict = VINF_HM_DOUBLE_FAULT;
5943 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5944 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5945
5946 break;
5947 }
5948
5949 case VMXREFLECTXCPT_TF:
5950 {
5951 rcStrict = VINF_EM_RESET;
5952 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5953 uExitVector));
5954 break;
5955 }
5956
5957 case VMXREFLECTXCPT_HANG:
5958 {
5959 rcStrict = VERR_EM_GUEST_CPU_HANG;
5960 break;
5961 }
5962
5963 default:
5964 Assert(rcStrict == VINF_SUCCESS);
5965 break;
5966 }
5967 }
5968 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5969 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5970 && uExitVector != X86_XCPT_DF
5971 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5972 {
5973 /*
5974 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5975 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5976 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5977 */
5978 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5979 {
5980 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
5981 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5982 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5983 }
5984 }
5985
5986 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
5987 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
5988 return rcStrict;
5989}
5990
5991
5992/**
5993 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5994 *
5995 * @returns VBox status code.
5996 * @param pVCpu The cross context virtual CPU structure.
5997 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5998 * out-of-sync. Make sure to update the required fields
5999 * before using them.
6000 *
6001 * @remarks No-long-jump zone!!!
6002 */
6003static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6004{
6005 NOREF(pMixedCtx);
6006
6007 /*
6008 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
6009 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
6010 */
6011 VMMRZCallRing3Disable(pVCpu);
6012 HM_DISABLE_PREEMPT();
6013
6014 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
6015 {
6016 uint32_t uVal = 0;
6017 uint32_t uShadow = 0;
6018 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
6019 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
6020 AssertRCReturn(rc, rc);
6021
6022 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
6023 CPUMSetGuestCR0(pVCpu, uVal);
6024 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
6025 }
6026
6027 HM_RESTORE_PREEMPT();
6028 VMMRZCallRing3Enable(pVCpu);
6029 return VINF_SUCCESS;
6030}
6031
6032
6033/**
6034 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
6035 *
6036 * @returns VBox status code.
6037 * @param pVCpu The cross context virtual CPU structure.
6038 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6039 * out-of-sync. Make sure to update the required fields
6040 * before using them.
6041 *
6042 * @remarks No-long-jump zone!!!
6043 */
6044static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6045{
6046 NOREF(pMixedCtx);
6047
6048 int rc = VINF_SUCCESS;
6049 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
6050 {
6051 uint32_t uVal = 0;
6052 uint32_t uShadow = 0;
6053 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
6054 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
6055 AssertRCReturn(rc, rc);
6056
6057 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
6058 CPUMSetGuestCR4(pVCpu, uVal);
6059 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6060 }
6061 return rc;
6062}
6063
6064
6065/**
6066 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
6067 *
6068 * @returns VBox status code.
6069 * @param pVCpu The cross context virtual CPU structure.
6070 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6071 * out-of-sync. Make sure to update the required fields
6072 * before using them.
6073 *
6074 * @remarks No-long-jump zone!!!
6075 */
6076static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6077{
6078 int rc = VINF_SUCCESS;
6079 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6080 {
6081 uint64_t u64Val = 0;
6082 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6083 AssertRCReturn(rc, rc);
6084
6085 pMixedCtx->rip = u64Val;
6086 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6087 }
6088 return rc;
6089}
6090
6091
6092/**
6093 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6094 *
6095 * @returns VBox status code.
6096 * @param pVCpu The cross context virtual CPU structure.
6097 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6098 * out-of-sync. Make sure to update the required fields
6099 * before using them.
6100 *
6101 * @remarks No-long-jump zone!!!
6102 */
6103static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6104{
6105 int rc = VINF_SUCCESS;
6106 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6107 {
6108 uint64_t u64Val = 0;
6109 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6110 AssertRCReturn(rc, rc);
6111
6112 pMixedCtx->rsp = u64Val;
6113 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6114 }
6115 return rc;
6116}
6117
6118
6119/**
6120 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6121 *
6122 * @returns VBox status code.
6123 * @param pVCpu The cross context virtual CPU structure.
6124 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6125 * out-of-sync. Make sure to update the required fields
6126 * before using them.
6127 *
6128 * @remarks No-long-jump zone!!!
6129 */
6130static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6131{
6132 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6133 {
6134 uint32_t uVal = 0;
6135 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6136 AssertRCReturn(rc, rc);
6137
6138 pMixedCtx->eflags.u32 = uVal;
6139 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6140 {
6141 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6142 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6143
6144 pMixedCtx->eflags.Bits.u1VM = 0;
6145 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6146 }
6147
6148 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6149 }
6150 return VINF_SUCCESS;
6151}
6152
6153
6154/**
6155 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6156 * guest-CPU context.
6157 */
6158DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6159{
6160 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6161 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6162 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6163 return rc;
6164}
6165
6166
6167/**
6168 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6169 * from the guest-state area in the VMCS.
6170 *
6171 * @param pVCpu The cross context virtual CPU structure.
6172 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6173 * out-of-sync. Make sure to update the required fields
6174 * before using them.
6175 *
6176 * @remarks No-long-jump zone!!!
6177 */
6178static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6179{
6180 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6181 {
6182 uint32_t uIntrState = 0;
6183 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6184 AssertRC(rc);
6185
6186 if (!uIntrState)
6187 {
6188 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6189 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6190
6191 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6192 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6193 }
6194 else
6195 {
6196 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6197 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6198 {
6199 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6200 AssertRC(rc);
6201 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6202 AssertRC(rc);
6203
6204 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6205 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6206 }
6207 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6208 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6209
6210 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6211 {
6212 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6213 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6214 }
6215 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6216 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6217 }
6218
6219 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6220 }
6221}
6222
6223
6224/**
6225 * Saves the guest's activity state.
6226 *
6227 * @returns VBox status code.
6228 * @param pVCpu The cross context virtual CPU structure.
6229 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6230 * out-of-sync. Make sure to update the required fields
6231 * before using them.
6232 *
6233 * @remarks No-long-jump zone!!!
6234 */
6235static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6236{
6237 NOREF(pMixedCtx);
6238 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6239 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6240 return VINF_SUCCESS;
6241}
6242
6243
6244/**
6245 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6246 * the current VMCS into the guest-CPU context.
6247 *
6248 * @returns VBox status code.
6249 * @param pVCpu The cross context virtual CPU structure.
6250 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6251 * out-of-sync. Make sure to update the required fields
6252 * before using them.
6253 *
6254 * @remarks No-long-jump zone!!!
6255 */
6256static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6257{
6258 int rc = VINF_SUCCESS;
6259 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6260 {
6261 uint32_t u32Val = 0;
6262 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6263 pMixedCtx->SysEnter.cs = u32Val;
6264 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6265 }
6266
6267 uint64_t u64Val = 0;
6268 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6269 {
6270 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6271 pMixedCtx->SysEnter.eip = u64Val;
6272 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6273 }
6274 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6275 {
6276 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6277 pMixedCtx->SysEnter.esp = u64Val;
6278 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6279 }
6280 return rc;
6281}
6282
6283
6284/**
6285 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6286 * the CPU back into the guest-CPU context.
6287 *
6288 * @returns VBox status code.
6289 * @param pVCpu The cross context virtual CPU structure.
6290 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6291 * out-of-sync. Make sure to update the required fields
6292 * before using them.
6293 *
6294 * @remarks No-long-jump zone!!!
6295 */
6296static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6297{
6298 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6299 VMMRZCallRing3Disable(pVCpu);
6300 HM_DISABLE_PREEMPT();
6301
6302 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6303 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6304 {
6305 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6306 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6307 }
6308
6309 HM_RESTORE_PREEMPT();
6310 VMMRZCallRing3Enable(pVCpu);
6311
6312 return VINF_SUCCESS;
6313}
6314
6315
6316/**
6317 * Saves the auto load/store'd guest MSRs from the current VMCS into
6318 * the guest-CPU context.
6319 *
6320 * @returns VBox status code.
6321 * @param pVCpu The cross context virtual CPU structure.
6322 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6323 * out-of-sync. Make sure to update the required fields
6324 * before using them.
6325 *
6326 * @remarks No-long-jump zone!!!
6327 */
6328static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6329{
6330 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6331 return VINF_SUCCESS;
6332
6333 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6334 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6335 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6336 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6337 {
6338 switch (pMsr->u32Msr)
6339 {
6340 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6341 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6342 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6343 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6344 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6345 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6346 break;
6347
6348 default:
6349 {
6350 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6351 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6352 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6353 }
6354 }
6355 }
6356
6357 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6358 return VINF_SUCCESS;
6359}
6360
6361
6362/**
6363 * Saves the guest control registers from the current VMCS into the guest-CPU
6364 * context.
6365 *
6366 * @returns VBox status code.
6367 * @param pVCpu The cross context virtual CPU structure.
6368 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6369 * out-of-sync. Make sure to update the required fields
6370 * before using them.
6371 *
6372 * @remarks No-long-jump zone!!!
6373 */
6374static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6375{
6376 /* Guest CR0. Guest FPU. */
6377 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6378 AssertRCReturn(rc, rc);
6379
6380 /* Guest CR4. */
6381 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6382 AssertRCReturn(rc, rc);
6383
6384 /* Guest CR2 - updated always during the world-switch or in #PF. */
6385 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6386 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6387 {
6388 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6389 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6390
6391 PVM pVM = pVCpu->CTX_SUFF(pVM);
6392 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6393 || ( pVM->hm.s.fNestedPaging
6394 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6395 {
6396 uint64_t u64Val = 0;
6397 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6398 if (pMixedCtx->cr3 != u64Val)
6399 {
6400 CPUMSetGuestCR3(pVCpu, u64Val);
6401 if (VMMRZCallRing3IsEnabled(pVCpu))
6402 {
6403 PGMUpdateCR3(pVCpu, u64Val);
6404 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6405 }
6406 else
6407 {
6408 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6409 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6410 }
6411 }
6412
6413 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6414 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6415 {
6416 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6417 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6418 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6419 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6420 AssertRCReturn(rc, rc);
6421
6422 if (VMMRZCallRing3IsEnabled(pVCpu))
6423 {
6424 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6425 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6426 }
6427 else
6428 {
6429 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6430 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6431 }
6432 }
6433 }
6434
6435 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6436 }
6437
6438 /*
6439 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6440 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6441 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6442 *
6443 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6444 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6445 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6446 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6447 *
6448 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6449 */
6450 if (VMMRZCallRing3IsEnabled(pVCpu))
6451 {
6452 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6453 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6454
6455 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6456 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6457
6458 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6459 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6460 }
6461
6462 return rc;
6463}
6464
6465
6466/**
6467 * Reads a guest segment register from the current VMCS into the guest-CPU
6468 * context.
6469 *
6470 * @returns VBox status code.
6471 * @param pVCpu The cross context virtual CPU structure.
6472 * @param idxSel Index of the selector in the VMCS.
6473 * @param idxLimit Index of the segment limit in the VMCS.
6474 * @param idxBase Index of the segment base in the VMCS.
6475 * @param idxAccess Index of the access rights of the segment in the VMCS.
6476 * @param pSelReg Pointer to the segment selector.
6477 *
6478 * @remarks No-long-jump zone!!!
6479 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6480 * macro as that takes care of whether to read from the VMCS cache or
6481 * not.
6482 */
6483DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6484 PCPUMSELREG pSelReg)
6485{
6486 NOREF(pVCpu);
6487
6488 uint32_t u32Val = 0;
6489 int rc = VMXReadVmcs32(idxSel, &u32Val);
6490 AssertRCReturn(rc, rc);
6491 pSelReg->Sel = (uint16_t)u32Val;
6492 pSelReg->ValidSel = (uint16_t)u32Val;
6493 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6494
6495 rc = VMXReadVmcs32(idxLimit, &u32Val);
6496 AssertRCReturn(rc, rc);
6497 pSelReg->u32Limit = u32Val;
6498
6499 uint64_t u64Val = 0;
6500 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6501 AssertRCReturn(rc, rc);
6502 pSelReg->u64Base = u64Val;
6503
6504 rc = VMXReadVmcs32(idxAccess, &u32Val);
6505 AssertRCReturn(rc, rc);
6506 pSelReg->Attr.u = u32Val;
6507
6508 /*
6509 * If VT-x marks the segment as unusable, most other bits remain undefined:
6510 * - For CS the L, D and G bits have meaning.
6511 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6512 * - For the remaining data segments no bits are defined.
6513 *
6514 * The present bit and the unusable bit has been observed to be set at the
6515 * same time (the selector was supposed to be invalid as we started executing
6516 * a V8086 interrupt in ring-0).
6517 *
6518 * What should be important for the rest of the VBox code, is that the P bit is
6519 * cleared. Some of the other VBox code recognizes the unusable bit, but
6520 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6521 * safe side here, we'll strip off P and other bits we don't care about. If
6522 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6523 *
6524 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6525 */
6526 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6527 {
6528 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6529
6530 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6531 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6532 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6533
6534 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6535#ifdef DEBUG_bird
6536 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6537 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6538 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6539#endif
6540 }
6541 return VINF_SUCCESS;
6542}
6543
6544
6545#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6546# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6547 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6548 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6549#else
6550# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6551 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6552 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6553#endif
6554
6555
6556/**
6557 * Saves the guest segment registers from the current VMCS into the guest-CPU
6558 * context.
6559 *
6560 * @returns VBox status code.
6561 * @param pVCpu The cross context virtual CPU structure.
6562 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6563 * out-of-sync. Make sure to update the required fields
6564 * before using them.
6565 *
6566 * @remarks No-long-jump zone!!!
6567 */
6568static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6569{
6570 /* Guest segment registers. */
6571 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6572 {
6573 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6574 AssertRCReturn(rc, rc);
6575
6576 rc = VMXLOCAL_READ_SEG(CS, cs);
6577 rc |= VMXLOCAL_READ_SEG(SS, ss);
6578 rc |= VMXLOCAL_READ_SEG(DS, ds);
6579 rc |= VMXLOCAL_READ_SEG(ES, es);
6580 rc |= VMXLOCAL_READ_SEG(FS, fs);
6581 rc |= VMXLOCAL_READ_SEG(GS, gs);
6582 AssertRCReturn(rc, rc);
6583
6584 /* Restore segment attributes for real-on-v86 mode hack. */
6585 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6586 {
6587 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6588 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6589 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6590 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6591 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6592 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6593 }
6594 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6595 }
6596
6597 return VINF_SUCCESS;
6598}
6599
6600
6601/**
6602 * Saves the guest descriptor table registers and task register from the current
6603 * VMCS into the guest-CPU context.
6604 *
6605 * @returns VBox status code.
6606 * @param pVCpu The cross context virtual CPU structure.
6607 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6608 * out-of-sync. Make sure to update the required fields
6609 * before using them.
6610 *
6611 * @remarks No-long-jump zone!!!
6612 */
6613static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6614{
6615 int rc = VINF_SUCCESS;
6616
6617 /* Guest LDTR. */
6618 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6619 {
6620 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6621 AssertRCReturn(rc, rc);
6622 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6623 }
6624
6625 /* Guest GDTR. */
6626 uint64_t u64Val = 0;
6627 uint32_t u32Val = 0;
6628 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6629 {
6630 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6631 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6632 pMixedCtx->gdtr.pGdt = u64Val;
6633 pMixedCtx->gdtr.cbGdt = u32Val;
6634 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6635 }
6636
6637 /* Guest IDTR. */
6638 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6639 {
6640 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6641 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6642 pMixedCtx->idtr.pIdt = u64Val;
6643 pMixedCtx->idtr.cbIdt = u32Val;
6644 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6645 }
6646
6647 /* Guest TR. */
6648 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6649 {
6650 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6651 AssertRCReturn(rc, rc);
6652
6653 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6654 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6655 {
6656 rc = VMXLOCAL_READ_SEG(TR, tr);
6657 AssertRCReturn(rc, rc);
6658 }
6659 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6660 }
6661 return rc;
6662}
6663
6664#undef VMXLOCAL_READ_SEG
6665
6666
6667/**
6668 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6669 * context.
6670 *
6671 * @returns VBox status code.
6672 * @param pVCpu The cross context virtual CPU structure.
6673 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6674 * out-of-sync. Make sure to update the required fields
6675 * before using them.
6676 *
6677 * @remarks No-long-jump zone!!!
6678 */
6679static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6680{
6681 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6682 {
6683 if (!pVCpu->hm.s.fUsingHyperDR7)
6684 {
6685 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6686 uint32_t u32Val;
6687 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6688 pMixedCtx->dr[7] = u32Val;
6689 }
6690
6691 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6692 }
6693 return VINF_SUCCESS;
6694}
6695
6696
6697/**
6698 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6699 *
6700 * @returns VBox status code.
6701 * @param pVCpu The cross context virtual CPU structure.
6702 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6703 * out-of-sync. Make sure to update the required fields
6704 * before using them.
6705 *
6706 * @remarks No-long-jump zone!!!
6707 */
6708static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6709{
6710 NOREF(pMixedCtx);
6711
6712 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6713 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6714 return VINF_SUCCESS;
6715}
6716
6717
6718/**
6719 * Saves the entire guest state from the currently active VMCS into the
6720 * guest-CPU context.
6721 *
6722 * This essentially VMREADs all guest-data.
6723 *
6724 * @returns VBox status code.
6725 * @param pVCpu The cross context virtual CPU structure.
6726 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6727 * out-of-sync. Make sure to update the required fields
6728 * before using them.
6729 */
6730static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6731{
6732 Assert(pVCpu);
6733 Assert(pMixedCtx);
6734
6735 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6736 return VINF_SUCCESS;
6737
6738 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6739 again on the ring-3 callback path, there is no real need to. */
6740 if (VMMRZCallRing3IsEnabled(pVCpu))
6741 VMMR0LogFlushDisable(pVCpu);
6742 else
6743 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6744 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6745
6746 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6747 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6748
6749 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6750 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6751
6752 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6753 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6754
6755 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6756 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6757
6758 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6759 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6760
6761 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6762 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6763
6764 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6765 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6766
6767 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6768 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6769
6770 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6771 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6772
6773 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6774 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6775
6776 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6777 ("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
6778 HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
6779
6780 if (VMMRZCallRing3IsEnabled(pVCpu))
6781 VMMR0LogFlushEnable(pVCpu);
6782
6783 return VINF_SUCCESS;
6784}
6785
6786
6787/**
6788 * Saves basic guest registers needed for IEM instruction execution.
6789 *
6790 * @returns VBox status code (OR-able).
6791 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6792 * @param pMixedCtx Pointer to the CPU context of the guest.
6793 * @param fMemory Whether the instruction being executed operates on
6794 * memory or not. Only CR0 is synced up if clear.
6795 * @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
6796 */
6797static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
6798{
6799 /*
6800 * We assume all general purpose registers other than RSP are available.
6801 *
6802 * RIP is a must, as it will be incremented or otherwise changed.
6803 *
6804 * RFLAGS are always required to figure the CPL.
6805 *
6806 * RSP isn't always required, however it's a GPR, so frequently required.
6807 *
6808 * SS and CS are the only segment register needed if IEM doesn't do memory
6809 * access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
6810 *
6811 * CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
6812 * be required for memory accesses.
6813 *
6814 * Note! Before IEM dispatches an exception, it will call us to sync in everything.
6815 */
6816 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6817 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6818 if (fNeedRsp)
6819 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6820 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6821 if (!fMemory)
6822 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6823 else
6824 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6825 AssertRCReturn(rc, rc);
6826 return rc;
6827}
6828
6829
6830/**
6831 * Ensures that we've got a complete basic guest-context.
6832 *
6833 * This excludes the FPU, SSE, AVX, and similar extended state. The interface
6834 * is for the interpreter.
6835 *
6836 * @returns VBox status code.
6837 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6838 * @param pMixedCtx Pointer to the guest-CPU context which may have data
6839 * needing to be synced in.
6840 * @thread EMT(pVCpu)
6841 */
6842VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6843{
6844 /* Note! Since this is only applicable to VT-x, the implementation is placed
6845 in the VT-x part of the sources instead of the generic stuff. */
6846 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
6847 {
6848 /* For now, imply that the caller might change everything too. */
6849 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6850 return hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6851 }
6852 return VINF_SUCCESS;
6853}
6854
6855
6856/**
6857 * Check per-VM and per-VCPU force flag actions that require us to go back to
6858 * ring-3 for one reason or another.
6859 *
6860 * @returns Strict VBox status code (i.e. informational status codes too)
6861 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6862 * ring-3.
6863 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6864 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6865 * interrupts)
6866 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6867 * all EMTs to be in ring-3.
6868 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6869 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6870 * to the EM loop.
6871 *
6872 * @param pVM The cross context VM structure.
6873 * @param pVCpu The cross context virtual CPU structure.
6874 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6875 * out-of-sync. Make sure to update the required fields
6876 * before using them.
6877 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6878 */
6879static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6880{
6881 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6882
6883 /*
6884 * Anything pending? Should be more likely than not if we're doing a good job.
6885 */
6886 if ( !fStepping
6887 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6888 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6889 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6890 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6891 return VINF_SUCCESS;
6892
6893 /* We need the control registers now, make sure the guest-CPU context is updated. */
6894 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6895 AssertRCReturn(rc3, rc3);
6896
6897 /* Pending HM CR3 sync. */
6898 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6899 {
6900 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6901 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6902 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6903 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6904 }
6905
6906 /* Pending HM PAE PDPEs. */
6907 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6908 {
6909 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6910 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6911 }
6912
6913 /* Pending PGM C3 sync. */
6914 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6915 {
6916 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6917 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6918 if (rcStrict2 != VINF_SUCCESS)
6919 {
6920 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6921 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6922 return rcStrict2;
6923 }
6924 }
6925
6926 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6927 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6928 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6929 {
6930 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6931 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6932 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6933 return rc2;
6934 }
6935
6936 /* Pending VM request packets, such as hardware interrupts. */
6937 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6938 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6939 {
6940 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6941 return VINF_EM_PENDING_REQUEST;
6942 }
6943
6944 /* Pending PGM pool flushes. */
6945 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6946 {
6947 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6948 return VINF_PGM_POOL_FLUSH_PENDING;
6949 }
6950
6951 /* Pending DMA requests. */
6952 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6953 {
6954 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6955 return VINF_EM_RAW_TO_R3;
6956 }
6957
6958 return VINF_SUCCESS;
6959}
6960
6961
6962/**
6963 * Converts any TRPM trap into a pending HM event. This is typically used when
6964 * entering from ring-3 (not longjmp returns).
6965 *
6966 * @param pVCpu The cross context virtual CPU structure.
6967 */
6968static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6969{
6970 Assert(TRPMHasTrap(pVCpu));
6971 Assert(!pVCpu->hm.s.Event.fPending);
6972
6973 uint8_t uVector;
6974 TRPMEVENT enmTrpmEvent;
6975 RTGCUINT uErrCode;
6976 RTGCUINTPTR GCPtrFaultAddress;
6977 uint8_t cbInstr;
6978
6979 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6980 AssertRC(rc);
6981
6982 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6983 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6984 if (enmTrpmEvent == TRPM_TRAP)
6985 {
6986 switch (uVector)
6987 {
6988 case X86_XCPT_NMI:
6989 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6990 break;
6991
6992 case X86_XCPT_BP:
6993 case X86_XCPT_OF:
6994 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6995 break;
6996
6997 case X86_XCPT_PF:
6998 case X86_XCPT_DF:
6999 case X86_XCPT_TS:
7000 case X86_XCPT_NP:
7001 case X86_XCPT_SS:
7002 case X86_XCPT_GP:
7003 case X86_XCPT_AC:
7004 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7005 /* no break! */
7006 default:
7007 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7008 break;
7009 }
7010 }
7011 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7012 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7013 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7014 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7015 else
7016 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7017
7018 rc = TRPMResetTrap(pVCpu);
7019 AssertRC(rc);
7020 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7021 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7022
7023 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7024}
7025
7026
7027/**
7028 * Converts the pending HM event into a TRPM trap.
7029 *
7030 * @param pVCpu The cross context virtual CPU structure.
7031 */
7032static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7033{
7034 Assert(pVCpu->hm.s.Event.fPending);
7035
7036 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7037 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7038 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
7039 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7040
7041 /* If a trap was already pending, we did something wrong! */
7042 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7043
7044 TRPMEVENT enmTrapType;
7045 switch (uVectorType)
7046 {
7047 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7048 enmTrapType = TRPM_HARDWARE_INT;
7049 break;
7050
7051 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7052 enmTrapType = TRPM_SOFTWARE_INT;
7053 break;
7054
7055 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7056 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7057 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7058 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7059 enmTrapType = TRPM_TRAP;
7060 break;
7061
7062 default:
7063 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7064 enmTrapType = TRPM_32BIT_HACK;
7065 break;
7066 }
7067
7068 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7069
7070 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7071 AssertRC(rc);
7072
7073 if (fErrorCodeValid)
7074 TRPMSetErrorCode(pVCpu, uErrorCode);
7075
7076 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7077 && uVector == X86_XCPT_PF)
7078 {
7079 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7080 }
7081 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7082 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7083 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7084 {
7085 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7086 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
7087 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7088 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7089 }
7090
7091 /* Clear any pending events from the VMCS. */
7092 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
7093 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
7094
7095 /* We're now done converting the pending event. */
7096 pVCpu->hm.s.Event.fPending = false;
7097}
7098
7099
7100/**
7101 * Does the necessary state syncing before returning to ring-3 for any reason
7102 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7103 *
7104 * @returns VBox status code.
7105 * @param pVCpu The cross context virtual CPU structure.
7106 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7107 * be out-of-sync. Make sure to update the required
7108 * fields before using them.
7109 * @param fSaveGuestState Whether to save the guest state or not.
7110 *
7111 * @remarks No-long-jmp zone!!!
7112 */
7113static int hmR0VmxLeave(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7114{
7115 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7116 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7117
7118 RTCPUID idCpu = RTMpCpuId();
7119 Log4Func(("HostCpuId=%u\n", idCpu));
7120
7121 /*
7122 * !!! IMPORTANT !!!
7123 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7124 */
7125
7126 /* Save the guest state if necessary. */
7127 if ( fSaveGuestState
7128 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7129 {
7130 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7131 AssertRCReturn(rc, rc);
7132 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7133 }
7134
7135 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
7136 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7137 {
7138 if (fSaveGuestState)
7139 {
7140 /* We shouldn't reload CR0 without saving it first. */
7141 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7142 AssertRCReturn(rc, rc);
7143 }
7144 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7145 }
7146
7147 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7148#ifdef VBOX_STRICT
7149 if (CPUMIsHyperDebugStateActive(pVCpu))
7150 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7151#endif
7152 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7153 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7154 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7155 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7156
7157#if HC_ARCH_BITS == 64
7158 /* Restore host-state bits that VT-x only restores partially. */
7159 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7160 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7161 {
7162 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7163 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7164 }
7165 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7166#endif
7167
7168 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7169 if (pVCpu->hm.s.vmx.fLazyMsrs)
7170 {
7171 /* We shouldn't reload the guest MSRs without saving it first. */
7172 if (!fSaveGuestState)
7173 {
7174 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7175 AssertRCReturn(rc, rc);
7176 }
7177 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7178 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7179 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7180 }
7181
7182 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7183 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7184
7185 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7186 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7187 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7188 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7189 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7190 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7191 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7192 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7193
7194 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7195
7196 /** @todo This partially defeats the purpose of having preemption hooks.
7197 * The problem is, deregistering the hooks should be moved to a place that
7198 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7199 * context.
7200 */
7201 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7202 {
7203 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7204 AssertRCReturn(rc, rc);
7205
7206 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7207 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7208 }
7209 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7210 NOREF(idCpu);
7211
7212 return VINF_SUCCESS;
7213}
7214
7215
7216/**
7217 * Leaves the VT-x session.
7218 *
7219 * @returns VBox status code.
7220 * @param pVCpu The cross context virtual CPU structure.
7221 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7222 * out-of-sync. Make sure to update the required fields
7223 * before using them.
7224 *
7225 * @remarks No-long-jmp zone!!!
7226 */
7227DECLINLINE(int) hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7228{
7229 HM_DISABLE_PREEMPT();
7230 HMVMX_ASSERT_CPU_SAFE();
7231 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7232 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7233
7234 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7235 and done this from the VMXR0ThreadCtxCallback(). */
7236 if (!pVCpu->hm.s.fLeaveDone)
7237 {
7238 int rc2 = hmR0VmxLeave(pVCpu, pMixedCtx, true /* fSaveGuestState */);
7239 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7240 pVCpu->hm.s.fLeaveDone = true;
7241 }
7242 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7243
7244 /*
7245 * !!! IMPORTANT !!!
7246 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7247 */
7248
7249 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7250 /** @todo Deregistering here means we need to VMCLEAR always
7251 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7252 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7253 VMMR0ThreadCtxHookDisable(pVCpu);
7254
7255 /* Leave HM context. This takes care of local init (term). */
7256 int rc = HMR0LeaveCpu(pVCpu);
7257
7258 HM_RESTORE_PREEMPT();
7259 return rc;
7260}
7261
7262
7263/**
7264 * Does the necessary state syncing before doing a longjmp to ring-3.
7265 *
7266 * @returns VBox status code.
7267 * @param pVCpu The cross context virtual CPU structure.
7268 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7269 * out-of-sync. Make sure to update the required fields
7270 * before using them.
7271 *
7272 * @remarks No-long-jmp zone!!!
7273 */
7274DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7275{
7276 return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7277}
7278
7279
7280/**
7281 * Take necessary actions before going back to ring-3.
7282 *
7283 * An action requires us to go back to ring-3. This function does the necessary
7284 * steps before we can safely return to ring-3. This is not the same as longjmps
7285 * to ring-3, this is voluntary and prepares the guest so it may continue
7286 * executing outside HM (recompiler/IEM).
7287 *
7288 * @returns VBox status code.
7289 * @param pVM The cross context VM structure.
7290 * @param pVCpu The cross context virtual CPU structure.
7291 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7292 * out-of-sync. Make sure to update the required fields
7293 * before using them.
7294 * @param rcExit The reason for exiting to ring-3. Can be
7295 * VINF_VMM_UNKNOWN_RING3_CALL.
7296 */
7297static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7298{
7299 Assert(pVM);
7300 Assert(pVCpu);
7301 Assert(pMixedCtx);
7302 HMVMX_ASSERT_PREEMPT_SAFE();
7303
7304 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7305 {
7306 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7307 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7308 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7309 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7310 }
7311
7312 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7313 VMMRZCallRing3Disable(pVCpu);
7314 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7315
7316 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7317 if (pVCpu->hm.s.Event.fPending)
7318 {
7319 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7320 Assert(!pVCpu->hm.s.Event.fPending);
7321 }
7322
7323 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7324 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7325
7326 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7327 and if we're injecting an event we should have a TRPM trap pending. */
7328 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7329#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7330 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7331#endif
7332
7333 /* Save guest state and restore host state bits. */
7334 int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7335 AssertRCReturn(rc, rc);
7336 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7337 /* Thread-context hooks are unregistered at this point!!! */
7338
7339 /* Sync recompiler state. */
7340 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7341 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7342 | CPUM_CHANGED_LDTR
7343 | CPUM_CHANGED_GDTR
7344 | CPUM_CHANGED_IDTR
7345 | CPUM_CHANGED_TR
7346 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7347 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7348 if ( pVM->hm.s.fNestedPaging
7349 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7350 {
7351 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7352 }
7353
7354 Assert(!pVCpu->hm.s.fClearTrapFlag);
7355
7356 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7357 if (rcExit != VINF_EM_RAW_INTERRUPT)
7358 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7359
7360 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7361
7362 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7363 VMMRZCallRing3RemoveNotification(pVCpu);
7364 VMMRZCallRing3Enable(pVCpu);
7365
7366 return rc;
7367}
7368
7369
7370/**
7371 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7372 * longjump to ring-3 and possibly get preempted.
7373 *
7374 * @returns VBox status code.
7375 * @param pVCpu The cross context virtual CPU structure.
7376 * @param enmOperation The operation causing the ring-3 longjump.
7377 * @param pvUser Opaque pointer to the guest-CPU context. The data
7378 * may be out-of-sync. Make sure to update the required
7379 * fields before using them.
7380 */
7381static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7382{
7383 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7384 {
7385 /*
7386 * !!! IMPORTANT !!!
7387 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7388 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7389 */
7390 VMMRZCallRing3RemoveNotification(pVCpu);
7391 VMMRZCallRing3Disable(pVCpu);
7392 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7393 RTThreadPreemptDisable(&PreemptState);
7394
7395 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7396 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7397
7398#if HC_ARCH_BITS == 64
7399 /* Restore host-state bits that VT-x only restores partially. */
7400 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7401 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7402 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7403 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7404#endif
7405 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7406 if (pVCpu->hm.s.vmx.fLazyMsrs)
7407 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7408
7409 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7410 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7411 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7412 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7413 {
7414 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7415 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7416 }
7417
7418 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7419 VMMR0ThreadCtxHookDisable(pVCpu);
7420 HMR0LeaveCpu(pVCpu);
7421 RTThreadPreemptRestore(&PreemptState);
7422 return VINF_SUCCESS;
7423 }
7424
7425 Assert(pVCpu);
7426 Assert(pvUser);
7427 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7428 HMVMX_ASSERT_PREEMPT_SAFE();
7429
7430 VMMRZCallRing3Disable(pVCpu);
7431 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7432
7433 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7434 enmOperation));
7435
7436 int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7437 AssertRCReturn(rc, rc);
7438
7439 VMMRZCallRing3Enable(pVCpu);
7440 return VINF_SUCCESS;
7441}
7442
7443
7444/**
7445 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7446 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7447 *
7448 * @param pVCpu The cross context virtual CPU structure.
7449 */
7450DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7451{
7452 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7453 {
7454 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7455 {
7456 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7457 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7458 AssertRC(rc);
7459 Log4(("Setup interrupt-window exiting\n"));
7460 }
7461 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7462}
7463
7464
7465/**
7466 * Clears the interrupt-window exiting control in the VMCS.
7467 *
7468 * @param pVCpu The cross context virtual CPU structure.
7469 */
7470DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7471{
7472 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7473 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7474 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7475 AssertRC(rc);
7476 Log4(("Cleared interrupt-window exiting\n"));
7477}
7478
7479
7480/**
7481 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7482 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7483 *
7484 * @param pVCpu The cross context virtual CPU structure.
7485 */
7486DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7487{
7488 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7489 {
7490 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7491 {
7492 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7493 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7494 AssertRC(rc);
7495 Log4(("Setup NMI-window exiting\n"));
7496 }
7497 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7498}
7499
7500
7501/**
7502 * Clears the NMI-window exiting control in the VMCS.
7503 *
7504 * @param pVCpu The cross context virtual CPU structure.
7505 */
7506DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7507{
7508 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7509 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7510 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7511 AssertRC(rc);
7512 Log4(("Cleared NMI-window exiting\n"));
7513}
7514
7515
7516/**
7517 * Evaluates the event to be delivered to the guest and sets it as the pending
7518 * event.
7519 *
7520 * @returns The VT-x guest-interruptibility state.
7521 * @param pVCpu The cross context virtual CPU structure.
7522 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7523 * out-of-sync. Make sure to update the required fields
7524 * before using them.
7525 */
7526static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7527{
7528 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7529 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7530 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7531 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7532 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7533
7534 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7535 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7536 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7537 Assert(!TRPMHasTrap(pVCpu));
7538
7539#ifdef VBOX_WITH_NEW_APIC
7540 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7541 APICUpdatePendingInterrupts(pVCpu);
7542#endif
7543
7544 /*
7545 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7546 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7547 */
7548 /** @todo SMI. SMIs take priority over NMIs. */
7549 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7550 {
7551 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7552 if ( !pVCpu->hm.s.Event.fPending
7553 && !fBlockNmi
7554 && !fBlockSti
7555 && !fBlockMovSS)
7556 {
7557 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7558 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7559 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7560
7561 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7562 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7563 }
7564 else
7565 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7566 }
7567 /*
7568 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7569 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7570 */
7571 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7572 && !pVCpu->hm.s.fSingleInstruction)
7573 {
7574 Assert(!DBGFIsStepping(pVCpu));
7575 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7576 AssertRC(rc);
7577 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7578 if ( !pVCpu->hm.s.Event.fPending
7579 && !fBlockInt
7580 && !fBlockSti
7581 && !fBlockMovSS)
7582 {
7583 uint8_t u8Interrupt;
7584 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7585 if (RT_SUCCESS(rc))
7586 {
7587 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7588 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7589 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7590
7591 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7592 }
7593 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7594 {
7595 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7596 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7597 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7598 }
7599 else
7600 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7601 }
7602 else
7603 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7604 }
7605
7606 return uIntrState;
7607}
7608
7609
7610/**
7611 * Sets a pending-debug exception to be delivered to the guest if the guest is
7612 * single-stepping in the VMCS.
7613 *
7614 * @param pVCpu The cross context virtual CPU structure.
7615 */
7616DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7617{
7618 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7619 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7620 AssertRC(rc);
7621}
7622
7623
7624/**
7625 * Injects any pending events into the guest if the guest is in a state to
7626 * receive them.
7627 *
7628 * @returns Strict VBox status code (i.e. informational status codes too).
7629 * @param pVCpu The cross context virtual CPU structure.
7630 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7631 * out-of-sync. Make sure to update the required fields
7632 * before using them.
7633 * @param uIntrState The VT-x guest-interruptibility state.
7634 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7635 * return VINF_EM_DBG_STEPPED if the event was
7636 * dispatched directly.
7637 */
7638static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7639{
7640 HMVMX_ASSERT_PREEMPT_SAFE();
7641 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7642
7643 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7644 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7645
7646 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7647 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7648 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7649 Assert(!TRPMHasTrap(pVCpu));
7650
7651 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7652 if (pVCpu->hm.s.Event.fPending)
7653 {
7654 /*
7655 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7656 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7657 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7658 *
7659 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7660 */
7661 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7662#ifdef VBOX_STRICT
7663 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7664 {
7665 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7666 Assert(!fBlockInt);
7667 Assert(!fBlockSti);
7668 Assert(!fBlockMovSS);
7669 }
7670 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7671 {
7672 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7673 Assert(!fBlockSti);
7674 Assert(!fBlockMovSS);
7675 Assert(!fBlockNmi);
7676 }
7677#endif
7678 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7679 (uint8_t)uIntType));
7680 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7681 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7682 fStepping, &uIntrState);
7683 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7684
7685 /* Update the interruptibility-state as it could have been changed by
7686 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7687 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7688 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7689
7690 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7691 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7692 else
7693 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7694 }
7695
7696 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7697 if ( fBlockSti
7698 || fBlockMovSS)
7699 {
7700 if (!pVCpu->hm.s.fSingleInstruction)
7701 {
7702 /*
7703 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7704 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7705 * See Intel spec. 27.3.4 "Saving Non-Register State".
7706 */
7707 Assert(!DBGFIsStepping(pVCpu));
7708 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7709 AssertRCReturn(rc2, rc2);
7710 if (pMixedCtx->eflags.Bits.u1TF)
7711 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7712 }
7713 else if (pMixedCtx->eflags.Bits.u1TF)
7714 {
7715 /*
7716 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7717 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7718 */
7719 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7720 uIntrState = 0;
7721 }
7722 }
7723
7724 /*
7725 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7726 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7727 */
7728 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7729 AssertRC(rc2);
7730
7731 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7732 NOREF(fBlockMovSS); NOREF(fBlockSti);
7733 return rcStrict;
7734}
7735
7736
7737/**
7738 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7739 *
7740 * @param pVCpu The cross context virtual CPU structure.
7741 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7742 * out-of-sync. Make sure to update the required fields
7743 * before using them.
7744 */
7745DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7746{
7747 NOREF(pMixedCtx);
7748 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7749 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7750}
7751
7752
7753/**
7754 * Injects a double-fault (\#DF) exception into the VM.
7755 *
7756 * @returns Strict VBox status code (i.e. informational status codes too).
7757 * @param pVCpu The cross context virtual CPU structure.
7758 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7759 * out-of-sync. Make sure to update the required fields
7760 * before using them.
7761 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7762 * and should return VINF_EM_DBG_STEPPED if the event
7763 * is injected directly (register modified by us, not
7764 * by hardware on VM-entry).
7765 * @param puIntrState Pointer to the current guest interruptibility-state.
7766 * This interruptibility-state will be updated if
7767 * necessary. This cannot not be NULL.
7768 */
7769DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7770{
7771 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7772 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7773 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7774 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7775 fStepping, puIntrState);
7776}
7777
7778
7779/**
7780 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7781 *
7782 * @param pVCpu The cross context virtual CPU structure.
7783 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7784 * out-of-sync. Make sure to update the required fields
7785 * before using them.
7786 */
7787DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7788{
7789 NOREF(pMixedCtx);
7790 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7791 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7792 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7793}
7794
7795
7796/**
7797 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7798 *
7799 * @param pVCpu The cross context virtual CPU structure.
7800 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7801 * out-of-sync. Make sure to update the required fields
7802 * before using them.
7803 * @param cbInstr The value of RIP that is to be pushed on the guest
7804 * stack.
7805 */
7806DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7807{
7808 NOREF(pMixedCtx);
7809 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7810 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7811 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7812}
7813
7814
7815/**
7816 * Injects a general-protection (\#GP) fault into the VM.
7817 *
7818 * @returns Strict VBox status code (i.e. informational status codes too).
7819 * @param pVCpu The cross context virtual CPU structure.
7820 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7821 * out-of-sync. Make sure to update the required fields
7822 * before using them.
7823 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7824 * mode, i.e. in real-mode it's not valid).
7825 * @param u32ErrorCode The error code associated with the \#GP.
7826 * @param fStepping Whether we're running in
7827 * hmR0VmxRunGuestCodeStep() and should return
7828 * VINF_EM_DBG_STEPPED if the event is injected
7829 * directly (register modified by us, not by
7830 * hardware on VM-entry).
7831 * @param puIntrState Pointer to the current guest interruptibility-state.
7832 * This interruptibility-state will be updated if
7833 * necessary. This cannot not be NULL.
7834 */
7835DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7836 bool fStepping, uint32_t *puIntrState)
7837{
7838 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7839 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7840 if (fErrorCodeValid)
7841 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7842 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7843 fStepping, puIntrState);
7844}
7845
7846
7847/**
7848 * Sets a general-protection (\#GP) exception as pending-for-injection into the
7849 * VM.
7850 *
7851 * @param pVCpu The cross context virtual CPU structure.
7852 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7853 * out-of-sync. Make sure to update the required fields
7854 * before using them.
7855 * @param u32ErrorCode The error code associated with the \#GP.
7856 */
7857DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7858{
7859 NOREF(pMixedCtx);
7860 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7861 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7862 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7863 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7864}
7865
7866
7867/**
7868 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7869 *
7870 * @param pVCpu The cross context virtual CPU structure.
7871 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7872 * out-of-sync. Make sure to update the required fields
7873 * before using them.
7874 * @param uVector The software interrupt vector number.
7875 * @param cbInstr The value of RIP that is to be pushed on the guest
7876 * stack.
7877 */
7878DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7879{
7880 NOREF(pMixedCtx);
7881 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7882 if ( uVector == X86_XCPT_BP
7883 || uVector == X86_XCPT_OF)
7884 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7885 else
7886 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7887 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7888}
7889
7890
7891/**
7892 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7893 * stack.
7894 *
7895 * @returns Strict VBox status code (i.e. informational status codes too).
7896 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7897 * @param pVM The cross context VM structure.
7898 * @param pMixedCtx Pointer to the guest-CPU context.
7899 * @param uValue The value to push to the guest stack.
7900 */
7901DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7902{
7903 /*
7904 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7905 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7906 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7907 */
7908 if (pMixedCtx->sp == 1)
7909 return VINF_EM_RESET;
7910 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7911 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7912 AssertRC(rc);
7913 return rc;
7914}
7915
7916
7917/**
7918 * Injects an event into the guest upon VM-entry by updating the relevant fields
7919 * in the VM-entry area in the VMCS.
7920 *
7921 * @returns Strict VBox status code (i.e. informational status codes too).
7922 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7923 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7924 *
7925 * @param pVCpu The cross context virtual CPU structure.
7926 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7927 * be out-of-sync. Make sure to update the required
7928 * fields before using them.
7929 * @param u64IntInfo The VM-entry interruption-information field.
7930 * @param cbInstr The VM-entry instruction length in bytes (for
7931 * software interrupts, exceptions and privileged
7932 * software exceptions).
7933 * @param u32ErrCode The VM-entry exception error code.
7934 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7935 * @param puIntrState Pointer to the current guest interruptibility-state.
7936 * This interruptibility-state will be updated if
7937 * necessary. This cannot not be NULL.
7938 * @param fStepping Whether we're running in
7939 * hmR0VmxRunGuestCodeStep() and should return
7940 * VINF_EM_DBG_STEPPED if the event is injected
7941 * directly (register modified by us, not by
7942 * hardware on VM-entry).
7943 *
7944 * @remarks Requires CR0!
7945 * @remarks No-long-jump zone!!!
7946 */
7947static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7948 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
7949 uint32_t *puIntrState)
7950{
7951 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7952 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7953 Assert(puIntrState);
7954 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7955
7956 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7957 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7958
7959#ifdef VBOX_STRICT
7960 /* Validate the error-code-valid bit for hardware exceptions. */
7961 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7962 {
7963 switch (uVector)
7964 {
7965 case X86_XCPT_PF:
7966 case X86_XCPT_DF:
7967 case X86_XCPT_TS:
7968 case X86_XCPT_NP:
7969 case X86_XCPT_SS:
7970 case X86_XCPT_GP:
7971 case X86_XCPT_AC:
7972 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7973 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7974 /* fallthru */
7975 default:
7976 break;
7977 }
7978 }
7979#endif
7980
7981 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7982 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7983 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7984
7985 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7986
7987 /* We require CR0 to check if the guest is in real-mode. */
7988 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7989 AssertRCReturn(rc, rc);
7990
7991 /*
7992 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7993 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7994 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7995 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7996 */
7997 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7998 {
7999 PVM pVM = pVCpu->CTX_SUFF(pVM);
8000 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
8001 {
8002 Assert(PDMVmmDevHeapIsEnabled(pVM));
8003 Assert(pVM->hm.s.vmx.pRealModeTSS);
8004
8005 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
8006 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8007 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
8008 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8009 AssertRCReturn(rc, rc);
8010 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
8011
8012 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8013 size_t const cbIdtEntry = sizeof(X86IDTR16);
8014 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
8015 {
8016 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8017 if (uVector == X86_XCPT_DF)
8018 return VINF_EM_RESET;
8019
8020 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
8021 if (uVector == X86_XCPT_GP)
8022 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
8023
8024 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
8025 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
8026 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
8027 fStepping, puIntrState);
8028 }
8029
8030 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8031 uint16_t uGuestIp = pMixedCtx->ip;
8032 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
8033 {
8034 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
8035 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
8036 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8037 }
8038 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
8039 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8040
8041 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8042 X86IDTR16 IdtEntry;
8043 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
8044 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8045 AssertRCReturn(rc, rc);
8046
8047 /* Construct the stack frame for the interrupt/exception handler. */
8048 VBOXSTRICTRC rcStrict;
8049 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
8050 if (rcStrict == VINF_SUCCESS)
8051 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
8052 if (rcStrict == VINF_SUCCESS)
8053 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
8054
8055 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
8056 if (rcStrict == VINF_SUCCESS)
8057 {
8058 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
8059 pMixedCtx->rip = IdtEntry.offSel;
8060 pMixedCtx->cs.Sel = IdtEntry.uSel;
8061 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
8062 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8063 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8064 && uVector == X86_XCPT_PF)
8065 pMixedCtx->cr2 = GCPtrFaultAddress;
8066
8067 /* If any other guest-state bits are changed here, make sure to update
8068 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
8069 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
8070 | HM_CHANGED_GUEST_RIP
8071 | HM_CHANGED_GUEST_RFLAGS
8072 | HM_CHANGED_GUEST_RSP);
8073
8074 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8075 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8076 {
8077 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8078 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8079 Log4(("Clearing inhibition due to STI.\n"));
8080 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
8081 }
8082 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8083 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
8084
8085 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8086 it, if we are returning to ring-3 before executing guest code. */
8087 pVCpu->hm.s.Event.fPending = false;
8088
8089 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8090 if (fStepping)
8091 rcStrict = VINF_EM_DBG_STEPPED;
8092 }
8093 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8094 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8095 return rcStrict;
8096 }
8097
8098 /*
8099 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8100 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8101 */
8102 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8103 }
8104
8105 /* Validate. */
8106 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8107 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8108 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8109
8110 /* Inject. */
8111 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8112 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8113 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8114 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8115
8116 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8117 && uVector == X86_XCPT_PF)
8118 pMixedCtx->cr2 = GCPtrFaultAddress;
8119
8120 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8121 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8122
8123 AssertRCReturn(rc, rc);
8124 return VINF_SUCCESS;
8125}
8126
8127
8128/**
8129 * Clears the interrupt-window exiting control in the VMCS and if necessary
8130 * clears the current event in the VMCS as well.
8131 *
8132 * @returns VBox status code.
8133 * @param pVCpu The cross context virtual CPU structure.
8134 *
8135 * @remarks Use this function only to clear events that have not yet been
8136 * delivered to the guest but are injected in the VMCS!
8137 * @remarks No-long-jump zone!!!
8138 */
8139static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8140{
8141 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8142
8143 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8144 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8145
8146 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8147 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8148}
8149
8150
8151/**
8152 * Enters the VT-x session.
8153 *
8154 * @returns VBox status code.
8155 * @param pVM The cross context VM structure.
8156 * @param pVCpu The cross context virtual CPU structure.
8157 * @param pCpu Pointer to the CPU info struct.
8158 */
8159VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8160{
8161 AssertPtr(pVM);
8162 AssertPtr(pVCpu);
8163 Assert(pVM->hm.s.vmx.fSupported);
8164 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8165 NOREF(pCpu); NOREF(pVM);
8166
8167 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8168 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8169
8170#ifdef VBOX_STRICT
8171 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8172 RTCCUINTREG uHostCR4 = ASMGetCR4();
8173 if (!(uHostCR4 & X86_CR4_VMXE))
8174 {
8175 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8176 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8177 }
8178#endif
8179
8180 /*
8181 * Load the VCPU's VMCS as the current (and active) one.
8182 */
8183 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8184 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8185 if (RT_FAILURE(rc))
8186 return rc;
8187
8188 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8189 pVCpu->hm.s.fLeaveDone = false;
8190 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8191
8192 return VINF_SUCCESS;
8193}
8194
8195
8196/**
8197 * The thread-context callback (only on platforms which support it).
8198 *
8199 * @param enmEvent The thread-context event.
8200 * @param pVCpu The cross context virtual CPU structure.
8201 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8202 * @thread EMT(pVCpu)
8203 */
8204VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8205{
8206 NOREF(fGlobalInit);
8207
8208 switch (enmEvent)
8209 {
8210 case RTTHREADCTXEVENT_OUT:
8211 {
8212 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8213 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8214 VMCPU_ASSERT_EMT(pVCpu);
8215
8216 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8217
8218 /* No longjmps (logger flushes, locks) in this fragile context. */
8219 VMMRZCallRing3Disable(pVCpu);
8220 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8221
8222 /*
8223 * Restore host-state (FPU, debug etc.)
8224 */
8225 if (!pVCpu->hm.s.fLeaveDone)
8226 {
8227 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8228 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8229 hmR0VmxLeave(pVCpu, pMixedCtx, false /* fSaveGuestState */);
8230 pVCpu->hm.s.fLeaveDone = true;
8231 }
8232
8233 /* Leave HM context, takes care of local init (term). */
8234 int rc = HMR0LeaveCpu(pVCpu);
8235 AssertRC(rc); NOREF(rc);
8236
8237 /* Restore longjmp state. */
8238 VMMRZCallRing3Enable(pVCpu);
8239 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8240 break;
8241 }
8242
8243 case RTTHREADCTXEVENT_IN:
8244 {
8245 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8246 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8247 VMCPU_ASSERT_EMT(pVCpu);
8248
8249 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8250 VMMRZCallRing3Disable(pVCpu);
8251 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8252
8253 /* Initialize the bare minimum state required for HM. This takes care of
8254 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8255 int rc = HMR0EnterCpu(pVCpu);
8256 AssertRC(rc);
8257 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8258
8259 /* Load the active VMCS as the current one. */
8260 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8261 {
8262 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8263 AssertRC(rc); NOREF(rc);
8264 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8265 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8266 }
8267 pVCpu->hm.s.fLeaveDone = false;
8268
8269 /* Restore longjmp state. */
8270 VMMRZCallRing3Enable(pVCpu);
8271 break;
8272 }
8273
8274 default:
8275 break;
8276 }
8277}
8278
8279
8280/**
8281 * Saves the host state in the VMCS host-state.
8282 * Sets up the VM-exit MSR-load area.
8283 *
8284 * The CPU state will be loaded from these fields on every successful VM-exit.
8285 *
8286 * @returns VBox status code.
8287 * @param pVM The cross context VM structure.
8288 * @param pVCpu The cross context virtual CPU structure.
8289 *
8290 * @remarks No-long-jump zone!!!
8291 */
8292static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8293{
8294 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8295
8296 int rc = VINF_SUCCESS;
8297 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8298 {
8299 rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8300 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8301
8302 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8303 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8304
8305 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8306 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8307
8308 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8309 }
8310 return rc;
8311}
8312
8313
8314/**
8315 * Saves the host state in the VMCS host-state.
8316 *
8317 * @returns VBox status code.
8318 * @param pVM The cross context VM structure.
8319 * @param pVCpu The cross context virtual CPU structure.
8320 *
8321 * @remarks No-long-jump zone!!!
8322 */
8323VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8324{
8325 AssertPtr(pVM);
8326 AssertPtr(pVCpu);
8327
8328 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8329
8330 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8331 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8332 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8333 return hmR0VmxSaveHostState(pVM, pVCpu);
8334}
8335
8336
8337/**
8338 * Loads the guest state into the VMCS guest-state area.
8339 *
8340 * The will typically be done before VM-entry when the guest-CPU state and the
8341 * VMCS state may potentially be out of sync.
8342 *
8343 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8344 * VM-entry controls.
8345 * Sets up the appropriate VMX non-root function to execute guest code based on
8346 * the guest CPU mode.
8347 *
8348 * @returns VBox strict status code.
8349 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8350 * without unrestricted guest access and the VMMDev is not presently
8351 * mapped (e.g. EFI32).
8352 *
8353 * @param pVM The cross context VM structure.
8354 * @param pVCpu The cross context virtual CPU structure.
8355 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8356 * out-of-sync. Make sure to update the required fields
8357 * before using them.
8358 *
8359 * @remarks No-long-jump zone!!! (Disables and enables long jmps for itself,
8360 * caller disables then again on successfull return. Confusing.)
8361 */
8362static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8363{
8364 AssertPtr(pVM);
8365 AssertPtr(pVCpu);
8366 AssertPtr(pMixedCtx);
8367 HMVMX_ASSERT_PREEMPT_SAFE();
8368
8369 VMMRZCallRing3Disable(pVCpu);
8370 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8371
8372 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8373
8374 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8375
8376 /* Determine real-on-v86 mode. */
8377 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8378 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8379 && CPUMIsGuestInRealModeEx(pMixedCtx))
8380 {
8381 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8382 }
8383
8384 /*
8385 * Load the guest-state into the VMCS.
8386 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8387 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8388 */
8389 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8390 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8391
8392 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8393 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8394 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8395
8396 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8397 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8398 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8399
8400 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8401 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8402
8403 VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8404 if (rcStrict == VINF_SUCCESS)
8405 { /* likely */ }
8406 else
8407 {
8408 VMMRZCallRing3Enable(pVCpu);
8409 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8410 return rcStrict;
8411 }
8412
8413 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8414 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8415 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8416
8417 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8418 determine we don't have to swap EFER after all. */
8419 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8420 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8421
8422 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8423 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8424
8425 rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8426 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8427
8428 /*
8429 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8430 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8431 */
8432 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8433 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8434
8435 /* Clear any unused and reserved bits. */
8436 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8437
8438 VMMRZCallRing3Enable(pVCpu);
8439
8440 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8441 return rc;
8442}
8443
8444
8445/**
8446 * Loads the state shared between the host and guest into the VMCS.
8447 *
8448 * @param pVM The cross context VM structure.
8449 * @param pVCpu The cross context virtual CPU structure.
8450 * @param pCtx Pointer to the guest-CPU context.
8451 *
8452 * @remarks No-long-jump zone!!!
8453 */
8454static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8455{
8456 NOREF(pVM);
8457
8458 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8459 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8460
8461 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8462 {
8463 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8464 AssertRC(rc);
8465 }
8466
8467 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8468 {
8469 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8470 AssertRC(rc);
8471
8472 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8473 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8474 {
8475 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8476 AssertRC(rc);
8477 }
8478 }
8479
8480 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8481 {
8482 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8483 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8484 }
8485
8486 /* Loading CR0, debug state might have changed intercepts, update VMCS. */
8487 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8488 {
8489 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8490 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8491 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8492 AssertRC(rc);
8493 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8494 }
8495
8496 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8497 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8498}
8499
8500
8501/**
8502 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8503 *
8504 * @returns Strict VBox status code (i.e. informational status codes too).
8505 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8506 * without unrestricted guest access and the VMMDev is not presently
8507 * mapped (e.g. EFI32).
8508 *
8509 * @param pVM The cross context VM structure.
8510 * @param pVCpu The cross context virtual CPU structure.
8511 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8512 * out-of-sync. Make sure to update the required fields
8513 * before using them.
8514 */
8515static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8516{
8517 HMVMX_ASSERT_PREEMPT_SAFE();
8518
8519 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8520#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8521 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8522#endif
8523
8524 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8525 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8526 {
8527 rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8528 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8529 { /* likely */}
8530 else
8531 {
8532 AssertMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8533 VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8534 }
8535 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8536 }
8537 else if (HMCPU_CF_VALUE(pVCpu))
8538 {
8539 rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8540 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8541 { /* likely */}
8542 else
8543 {
8544 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8545 ("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8546 return rcStrict;
8547 }
8548 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8549 }
8550
8551 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8552 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8553 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8554 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8555 return rcStrict;
8556}
8557
8558
8559/**
8560 * Does the preparations before executing guest code in VT-x.
8561 *
8562 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8563 * recompiler/IEM. We must be cautious what we do here regarding committing
8564 * guest-state information into the VMCS assuming we assuredly execute the
8565 * guest in VT-x mode.
8566 *
8567 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8568 * the common-state (TRPM/forceflags), we must undo those changes so that the
8569 * recompiler/IEM can (and should) use them when it resumes guest execution.
8570 * Otherwise such operations must be done when we can no longer exit to ring-3.
8571 *
8572 * @returns Strict VBox status code (i.e. informational status codes too).
8573 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8574 * have been disabled.
8575 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8576 * double-fault into the guest.
8577 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8578 * dispatched directly.
8579 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8580 *
8581 * @param pVM The cross context VM structure.
8582 * @param pVCpu The cross context virtual CPU structure.
8583 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8584 * out-of-sync. Make sure to update the required fields
8585 * before using them.
8586 * @param pVmxTransient Pointer to the VMX transient structure.
8587 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8588 * us ignore some of the reasons for returning to
8589 * ring-3, and return VINF_EM_DBG_STEPPED if event
8590 * dispatching took place.
8591 */
8592static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8593{
8594 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8595
8596#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8597 PGMRZDynMapFlushAutoSet(pVCpu);
8598#endif
8599
8600 /* Check force flag actions that might require us to go back to ring-3. */
8601 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8602 if (rcStrict == VINF_SUCCESS)
8603 { /* FFs doesn't get set all the time. */ }
8604 else
8605 return rcStrict;
8606
8607#ifndef IEM_VERIFICATION_MODE_FULL
8608 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8609 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8610 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8611 {
8612 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8613 RTGCPHYS GCPhysApicBase;
8614 GCPhysApicBase = pMixedCtx->msrApicBase;
8615 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8616
8617 /* Unalias any existing mapping. */
8618 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8619 AssertRCReturn(rc, rc);
8620
8621 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8622 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGp\n", GCPhysApicBase));
8623 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8624 AssertRCReturn(rc, rc);
8625
8626 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8627 }
8628#endif /* !IEM_VERIFICATION_MODE_FULL */
8629
8630 if (TRPMHasTrap(pVCpu))
8631 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8632 uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8633
8634 /*
8635 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8636 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8637 */
8638 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
8639 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8640 { /* likely */ }
8641 else
8642 {
8643 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8644 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8645 return rcStrict;
8646 }
8647
8648 /*
8649 * Load the guest state bits, we can handle longjmps/getting preempted here.
8650 *
8651 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8652 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8653 * Hence, this needs to be done -after- injection of events.
8654 */
8655 rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8656 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8657 { /* likely */ }
8658 else
8659 return rcStrict;
8660
8661 /*
8662 * No longjmps to ring-3 from this point on!!!
8663 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8664 * This also disables flushing of the R0-logger instance (if any).
8665 */
8666 VMMRZCallRing3Disable(pVCpu);
8667
8668 /*
8669 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8670 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8671 *
8672 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8673 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8674 *
8675 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8676 * executing guest code.
8677 */
8678 pVmxTransient->fEFlags = ASMIntDisableFlags();
8679
8680 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8681 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8682 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8683 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8684 {
8685 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8686 {
8687 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8688 pVCpu->hm.s.Event.fPending = false;
8689
8690 return VINF_SUCCESS;
8691 }
8692
8693 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8694 rcStrict = VINF_EM_RAW_INTERRUPT;
8695 }
8696 else
8697 {
8698 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8699 rcStrict = VINF_EM_RAW_TO_R3;
8700 }
8701
8702 ASMSetFlags(pVmxTransient->fEFlags);
8703 VMMRZCallRing3Enable(pVCpu);
8704
8705 return rcStrict;
8706}
8707
8708
8709/**
8710 * Prepares to run guest code in VT-x and we've committed to doing so. This
8711 * means there is no backing out to ring-3 or anywhere else at this
8712 * point.
8713 *
8714 * @param pVM The cross context VM structure.
8715 * @param pVCpu The cross context virtual CPU structure.
8716 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8717 * out-of-sync. Make sure to update the required fields
8718 * before using them.
8719 * @param pVmxTransient Pointer to the VMX transient structure.
8720 *
8721 * @remarks Called with preemption disabled.
8722 * @remarks No-long-jump zone!!!
8723 */
8724static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8725{
8726 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8727 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8728 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8729
8730 /*
8731 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8732 */
8733 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8734 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8735
8736#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8737 if (!CPUMIsGuestFPUStateActive(pVCpu))
8738 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8739 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8740 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8741#endif
8742
8743 if ( pVCpu->hm.s.fPreloadGuestFpu
8744 && !CPUMIsGuestFPUStateActive(pVCpu))
8745 {
8746 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8747 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8748 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8749 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8750 }
8751
8752 /*
8753 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8754 */
8755 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8756 && pVCpu->hm.s.vmx.cMsrs > 0)
8757 {
8758 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8759 }
8760
8761 /*
8762 * Load the host state bits as we may've been preempted (only happens when
8763 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8764 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8765 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8766 * See @bugref{8432}.
8767 */
8768 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8769 {
8770 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8771 AssertRC(rc);
8772 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
8773 }
8774 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8775
8776 /*
8777 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8778 */
8779 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8780 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8781 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8782
8783 /* Store status of the shared guest-host state at the time of VM-entry. */
8784#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8785 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8786 {
8787 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8788 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8789 }
8790 else
8791#endif
8792 {
8793 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8794 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8795 }
8796 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8797
8798 /*
8799 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8800 */
8801 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8802 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8803
8804 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8805 RTCPUID idCurrentCpu = pCpu->idCpu;
8806 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8807 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8808 {
8809 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8810 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8811 }
8812
8813 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8814 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8815 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8816 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8817
8818 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8819
8820 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8821 to start executing. */
8822
8823 /*
8824 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8825 */
8826 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8827 {
8828 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8829 {
8830 bool fMsrUpdated;
8831 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8832 AssertRC(rc2);
8833 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8834
8835 rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8836 &fMsrUpdated);
8837 AssertRC(rc2);
8838 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8839
8840 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8841 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8842 }
8843 else
8844 {
8845 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8846 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8847 }
8848 }
8849
8850#ifdef VBOX_STRICT
8851 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8852 hmR0VmxCheckHostEferMsr(pVCpu);
8853 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8854#endif
8855#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8856 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8857 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8858 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8859#endif
8860}
8861
8862
8863/**
8864 * Performs some essential restoration of state after running guest code in
8865 * VT-x.
8866 *
8867 * @param pVM The cross context VM structure.
8868 * @param pVCpu The cross context virtual CPU structure.
8869 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8870 * out-of-sync. Make sure to update the required fields
8871 * before using them.
8872 * @param pVmxTransient Pointer to the VMX transient structure.
8873 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8874 *
8875 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8876 *
8877 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8878 * unconditionally when it is safe to do so.
8879 */
8880static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8881{
8882 NOREF(pVM);
8883
8884 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8885
8886 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8887 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8888 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8889 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8890 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8891 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8892
8893 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8894 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
8895
8896 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8897 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8898 Assert(!ASMIntAreEnabled());
8899 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8900
8901#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8902 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVM, pVCpu))
8903 {
8904 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8905 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8906 }
8907#endif
8908
8909#if HC_ARCH_BITS == 64
8910 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8911#endif
8912#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8913 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8914 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8915 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8916#else
8917 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8918#endif
8919#ifdef VBOX_STRICT
8920 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8921#endif
8922 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8923 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8924
8925 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8926 uint32_t uExitReason;
8927 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8928 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8929 AssertRC(rc);
8930 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8931 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8932
8933 /* Update the VM-exit history array. */
8934 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8935
8936 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8937 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8938 {
8939 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8940 pVmxTransient->fVMEntryFailed));
8941 return;
8942 }
8943
8944 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8945 {
8946 /** @todo We can optimize this by only syncing with our force-flags when
8947 * really needed and keeping the VMCS state as it is for most
8948 * VM-exits. */
8949 /* Update the guest interruptibility-state from the VMCS. */
8950 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8951
8952#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8953 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8954 AssertRC(rc);
8955#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8956 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8957 AssertRC(rc);
8958#endif
8959
8960 /*
8961 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8962 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8963 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8964 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8965 */
8966 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8967 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8968 {
8969 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8970 AssertRC(rc);
8971 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8972 }
8973 }
8974}
8975
8976
8977/**
8978 * Runs the guest code using VT-x the normal way.
8979 *
8980 * @returns VBox status code.
8981 * @param pVM The cross context VM structure.
8982 * @param pVCpu The cross context virtual CPU structure.
8983 * @param pCtx Pointer to the guest-CPU context.
8984 *
8985 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8986 */
8987static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8988{
8989 VMXTRANSIENT VmxTransient;
8990 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8991 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8992 uint32_t cLoops = 0;
8993
8994 for (;; cLoops++)
8995 {
8996 Assert(!HMR0SuspendPending());
8997 HMVMX_ASSERT_CPU_SAFE();
8998
8999 /* Preparatory work for running guest code, this may force us to return
9000 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
9001 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9002 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
9003 if (rcStrict != VINF_SUCCESS)
9004 break;
9005
9006 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9007 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9008 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9009
9010 /* Restore any residual host-state and save any bits shared between host
9011 and guest into the guest-CPU state. Re-enables interrupts! */
9012 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
9013
9014 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9015 if (RT_SUCCESS(rcRun))
9016 { /* very likely */ }
9017 else
9018 {
9019 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9020 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9021 return rcRun;
9022 }
9023
9024 /* Profile the VM-exit. */
9025 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9026 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9027 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9028 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9029 HMVMX_START_EXIT_DISPATCH_PROF();
9030
9031 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9032
9033 /* Handle the VM-exit. */
9034#ifdef HMVMX_USE_FUNCTION_TABLE
9035 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
9036#else
9037 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
9038#endif
9039 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9040 if (rcStrict == VINF_SUCCESS)
9041 {
9042 if (cLoops <= pVM->hm.s.cMaxResumeLoops)
9043 continue; /* likely */
9044 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9045 rcStrict = VINF_EM_RAW_INTERRUPT;
9046 }
9047 break;
9048 }
9049
9050 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9051 return rcStrict;
9052}
9053
9054
9055
9056/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
9057 * probes.
9058 *
9059 * The following few functions and associated structure contains the bloat
9060 * necessary for providing detailed debug events and dtrace probes as well as
9061 * reliable host side single stepping. This works on the principle of
9062 * "subclassing" the normal execution loop and workers. We replace the loop
9063 * method completely and override selected helpers to add necessary adjustments
9064 * to their core operation.
9065 *
9066 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
9067 * any performance for debug and analysis features.
9068 *
9069 * @{
9070 */
9071
9072typedef struct VMXRUNDBGSTATE
9073{
9074 /** The RIP we started executing at. This is for detecting that we stepped. */
9075 uint64_t uRipStart;
9076 /** The CS we started executing with. */
9077 uint16_t uCsStart;
9078
9079 /** Whether we've actually modified the 1st execution control field. */
9080 bool fModifiedProcCtls : 1;
9081 /** Whether we've actually modified the 2nd execution control field. */
9082 bool fModifiedProcCtls2 : 1;
9083 /** Whether we've actually modified the exception bitmap. */
9084 bool fModifiedXcptBitmap : 1;
9085
9086 /** We desire the modified the CR0 mask to be cleared. */
9087 bool fClearCr0Mask : 1;
9088 /** We desire the modified the CR4 mask to be cleared. */
9089 bool fClearCr4Mask : 1;
9090 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9091 uint32_t fCpe1Extra;
9092 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9093 uint32_t fCpe1Unwanted;
9094 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9095 uint32_t fCpe2Extra;
9096 /** Extra stuff we need in */
9097 uint32_t bmXcptExtra;
9098 /** The sequence number of the Dtrace provider settings the state was
9099 * configured against. */
9100 uint32_t uDtraceSettingsSeqNo;
9101 /** Exits to check (one bit per exit). */
9102 uint32_t bmExitsToCheck[3];
9103
9104 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9105 uint32_t fProcCtlsInitial;
9106 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9107 uint32_t fProcCtls2Initial;
9108 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9109 uint32_t bmXcptInitial;
9110} VMXRUNDBGSTATE;
9111AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9112typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9113
9114
9115/**
9116 * Initializes the VMXRUNDBGSTATE structure.
9117 *
9118 * @param pVCpu The cross context virtual CPU structure of the
9119 * calling EMT.
9120 * @param pCtx The CPU register context to go with @a pVCpu.
9121 * @param pDbgState The structure to initialize.
9122 */
9123DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9124{
9125 pDbgState->uRipStart = pCtx->rip;
9126 pDbgState->uCsStart = pCtx->cs.Sel;
9127
9128 pDbgState->fModifiedProcCtls = false;
9129 pDbgState->fModifiedProcCtls2 = false;
9130 pDbgState->fModifiedXcptBitmap = false;
9131 pDbgState->fClearCr0Mask = false;
9132 pDbgState->fClearCr4Mask = false;
9133 pDbgState->fCpe1Extra = 0;
9134 pDbgState->fCpe1Unwanted = 0;
9135 pDbgState->fCpe2Extra = 0;
9136 pDbgState->bmXcptExtra = 0;
9137 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9138 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9139 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9140}
9141
9142
9143/**
9144 * Updates the VMSC fields with changes requested by @a pDbgState.
9145 *
9146 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9147 * immediately before executing guest code, i.e. when interrupts are disabled.
9148 * We don't check status codes here as we cannot easily assert or return in the
9149 * latter case.
9150 *
9151 * @param pVCpu The cross context virtual CPU structure.
9152 * @param pDbgState The debug state.
9153 */
9154DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9155{
9156 /*
9157 * Ensure desired flags in VMCS control fields are set.
9158 * (Ignoring write failure here, as we're committed and it's just debug extras.)
9159 *
9160 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9161 * there should be no stale data in pCtx at this point.
9162 */
9163 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9164 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9165 {
9166 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
9167 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9168 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9169 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9170 pDbgState->fModifiedProcCtls = true;
9171 }
9172
9173 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9174 {
9175 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
9176 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9177 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9178 pDbgState->fModifiedProcCtls2 = true;
9179 }
9180
9181 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9182 {
9183 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
9184 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9185 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9186 pDbgState->fModifiedXcptBitmap = true;
9187 }
9188
9189 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9190 {
9191 pVCpu->hm.s.vmx.u32CR0Mask = 0;
9192 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9193 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9194 }
9195
9196 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9197 {
9198 pVCpu->hm.s.vmx.u32CR4Mask = 0;
9199 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9200 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9201 }
9202}
9203
9204
9205DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9206{
9207 /*
9208 * Restore exit control settings as we may not reenter this function the
9209 * next time around.
9210 */
9211 /* We reload the initial value, trigger what we can of recalculations the
9212 next time around. From the looks of things, that's all that's required atm. */
9213 if (pDbgState->fModifiedProcCtls)
9214 {
9215 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9216 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9217 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9218 AssertRCReturn(rc2, rc2);
9219 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9220 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0 | HM_CHANGED_GUEST_DEBUG);
9221 }
9222
9223 /* We're currently the only ones messing with this one, so just restore the
9224 cached value and reload the field. */
9225 if ( pDbgState->fModifiedProcCtls2
9226 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9227 {
9228 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9229 AssertRCReturn(rc2, rc2);
9230 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9231 }
9232
9233 /* If we've modified the exception bitmap, we restore it and trigger
9234 reloading and partial recalculation the next time around. */
9235 if (pDbgState->fModifiedXcptBitmap)
9236 {
9237 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9238 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS | HM_CHANGED_GUEST_CR0);
9239 }
9240
9241 /* We assume hmR0VmxLoadSharedCR0 will recalculate and load the CR0 mask. */
9242 if (pDbgState->fClearCr0Mask)
9243 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9244
9245 /* We assume hmR0VmxLoadGuestCR3AndCR4 will recalculate and load the CR4 mask. */
9246 if (pDbgState->fClearCr4Mask)
9247 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9248
9249 return rcStrict;
9250}
9251
9252
9253/**
9254 * Configures VM-exit controls for current DBGF and DTrace settings.
9255 *
9256 * This updates @a pDbgState and the VMCS execution control fields to reflect
9257 * the necessary exits demanded by DBGF and DTrace.
9258 *
9259 * @param pVM The cross context VM structure.
9260 * @param pVCpu The cross context virtual CPU structure.
9261 * @param pCtx Pointer to the guest-CPU context.
9262 * @param pDbgState The debug state.
9263 * @param pVmxTransient Pointer to the VMX transient structure. May update
9264 * fUpdateTscOffsettingAndPreemptTimer.
9265 */
9266static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9267 PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9268{
9269 /*
9270 * Take down the dtrace serial number so we can spot changes.
9271 */
9272 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9273 ASMCompilerBarrier();
9274
9275 /*
9276 * We'll rebuild most of the middle block of data members (holding the
9277 * current settings) as we go along here, so start by clearing it all.
9278 */
9279 pDbgState->bmXcptExtra = 0;
9280 pDbgState->fCpe1Extra = 0;
9281 pDbgState->fCpe1Unwanted = 0;
9282 pDbgState->fCpe2Extra = 0;
9283 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9284 pDbgState->bmExitsToCheck[i] = 0;
9285
9286 /*
9287 * Software interrupts (INT XXh) - no idea how to trigger these...
9288 */
9289 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9290 || VBOXVMM_INT_SOFTWARE_ENABLED())
9291 {
9292 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9293 }
9294
9295 /*
9296 * Exception bitmap and XCPT events+probes.
9297 */
9298 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9299 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9300 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9301
9302 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9303 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9304 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9305 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9306 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9307 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9308 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9309 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9310 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9311 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9312 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9313 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9314 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9315 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9316 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9317 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9318 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9319 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9320
9321 if (pDbgState->bmXcptExtra)
9322 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9323
9324 /*
9325 * Process events and probes for VM exits, making sure we get the wanted exits.
9326 *
9327 * Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9328 * So, when adding/changing/removing please don't forget to update it.
9329 *
9330 * Some of the macros are picking up local variables to save horizontal space,
9331 * (being able to see it in a table is the lesser evil here).
9332 */
9333#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9334 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9335 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9336#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9337 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9338 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9339 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9340 } else do { } while (0)
9341#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9342 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9343 { \
9344 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9345 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9346 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9347 } else do { } while (0)
9348#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9349 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9350 { \
9351 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9352 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9353 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9354 } else do { } while (0)
9355#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9356 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9357 { \
9358 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9359 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9360 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9361 } else do { } while (0)
9362
9363 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9364 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9365 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9366 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9367 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9368
9369 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9370 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9371 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9372 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9373 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9374 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9375 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9376 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9377 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9378 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9379 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9380 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9381 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9382 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9383 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9384 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9385 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9386 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9387 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9388 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9389 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9390 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9391 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9392 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9393 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9394 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9395 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9396 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9397 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9398 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9399 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9400 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9401 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9402 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9403 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9404 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9405
9406 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9407 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9408 {
9409 int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9410 rc2 |= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9411 rc2 |= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9412 AssertRC(rc2);
9413
9414#if 0 /** @todo fix me */
9415 pDbgState->fClearCr0Mask = true;
9416 pDbgState->fClearCr4Mask = true;
9417#endif
9418 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9419 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9420 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9421 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9422 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9423 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9424 require clearing here and in the loop if we start using it. */
9425 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9426 }
9427 else
9428 {
9429 if (pDbgState->fClearCr0Mask)
9430 {
9431 pDbgState->fClearCr0Mask = false;
9432 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9433 }
9434 if (pDbgState->fClearCr4Mask)
9435 {
9436 pDbgState->fClearCr4Mask = false;
9437 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9438 }
9439 }
9440 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9441 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9442
9443 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9444 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9445 {
9446 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9447 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9448 }
9449 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9450 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9451
9452 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9453 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9454 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9455 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9456 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9457 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9458 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9459 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9460#if 0 /** @todo too slow, fix handler. */
9461 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9462#endif
9463 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9464
9465 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9466 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9467 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9468 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9469 {
9470 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9471 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9472 }
9473 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9474 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9475 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9476 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9477
9478 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9479 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9480 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9481 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9482 {
9483 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9484 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9485 }
9486 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9487 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9488 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9489 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9490
9491 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9492 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9493 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9494 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9495 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9496 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9497 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9498 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9499 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9500 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9501 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9502 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9503 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9504 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9505 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9506 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9507 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9508 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9509 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9510 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9511 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9512 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9513
9514#undef IS_EITHER_ENABLED
9515#undef SET_ONLY_XBM_IF_EITHER_EN
9516#undef SET_CPE1_XBM_IF_EITHER_EN
9517#undef SET_CPEU_XBM_IF_EITHER_EN
9518#undef SET_CPE2_XBM_IF_EITHER_EN
9519
9520 /*
9521 * Sanitize the control stuff.
9522 */
9523 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9524 if (pDbgState->fCpe2Extra)
9525 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9526 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9527 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9528 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9529 {
9530 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9531 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9532 }
9533
9534 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9535 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9536 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9537 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9538}
9539
9540
9541/**
9542 * Fires off DBGF events and dtrace probes for an exit, when it's appropriate.
9543 *
9544 * The caller has checked exit against the VMXRUNDBGSTATE::bmExitsToCheck
9545 * bitmap. The caller has checked for NMIs already, so we don't have to do that
9546 * either.
9547 *
9548 * @returns Strict VBox status code (i.e. informational status codes too).
9549 * @param pVM The cross context VM structure.
9550 * @param pVCpu The cross context virtual CPU structure.
9551 * @param pMixedCtx Pointer to the guest-CPU context.
9552 * @param pVmxTransient Pointer to the VMX-transient structure.
9553 * @param uExitReason The VM-exit reason.
9554 *
9555 * @remarks The name of this function is displayed by dtrace, so keep it short
9556 * and to the point. No longer than 33 chars long, please.
9557 */
9558static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9559 PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9560{
9561 /*
9562 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9563 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9564 *
9565 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9566 * does. Must add/change/remove both places. Same ordering, please.
9567 *
9568 * Added/removed events must also be reflected in the next section
9569 * where we dispatch dtrace events.
9570 */
9571 bool fDtrace1 = false;
9572 bool fDtrace2 = false;
9573 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9574 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9575 uint32_t uEventArg = 0;
9576#define SET_EXIT(a_EventSubName) \
9577 do { \
9578 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9579 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9580 } while (0)
9581#define SET_BOTH(a_EventSubName) \
9582 do { \
9583 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9584 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9585 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9586 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9587 } while (0)
9588 switch (uExitReason)
9589 {
9590 case VMX_EXIT_MTF:
9591 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9592
9593 case VMX_EXIT_XCPT_OR_NMI:
9594 {
9595 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9596 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9597 {
9598 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9599 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9600 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9601 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9602 {
9603 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9604 {
9605 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9606 uEventArg = pVmxTransient->uExitIntErrorCode;
9607 }
9608 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9609 switch (enmEvent1)
9610 {
9611 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9612 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9613 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9614 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9615 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9616 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9617 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9618 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9619 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9620 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9621 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9622 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9623 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9624 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9625 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9626 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9627 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9628 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9629 default: break;
9630 }
9631 }
9632 else
9633 AssertFailed();
9634 break;
9635
9636 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9637 uEventArg = idxVector;
9638 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9639 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9640 break;
9641 }
9642 break;
9643 }
9644
9645 case VMX_EXIT_TRIPLE_FAULT:
9646 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9647 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9648 break;
9649 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9650 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9651 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9652 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9653 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9654
9655 /* Instruction specific VM-exits: */
9656 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9657 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9658 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9659 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9660 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9661 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9662 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9663 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9664 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9665 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9666 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9667 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9668 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9669 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9670 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9671 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9672 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9673 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9674 case VMX_EXIT_MOV_CRX:
9675 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9676/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
9677* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
9678 if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
9679 == VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
9680 SET_BOTH(CRX_READ);
9681 else
9682 SET_BOTH(CRX_WRITE);
9683 uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
9684 break;
9685 case VMX_EXIT_MOV_DRX:
9686 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9687 if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
9688 == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
9689 SET_BOTH(DRX_READ);
9690 else
9691 SET_BOTH(DRX_WRITE);
9692 uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
9693 break;
9694 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9695 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9696 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9697 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9698 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9699 case VMX_EXIT_XDTR_ACCESS:
9700 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9701 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9702 {
9703 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9704 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9705 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9706 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9707 }
9708 break;
9709
9710 case VMX_EXIT_TR_ACCESS:
9711 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9712 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9713 {
9714 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9715 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9716 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9717 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9718 }
9719 break;
9720
9721 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9722 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9723 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9724 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9725 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9726 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9727 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9728 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9729 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9730 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9731 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9732
9733 /* Events that aren't relevant at this point. */
9734 case VMX_EXIT_EXT_INT:
9735 case VMX_EXIT_INT_WINDOW:
9736 case VMX_EXIT_NMI_WINDOW:
9737 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9738 case VMX_EXIT_PREEMPT_TIMER:
9739 case VMX_EXIT_IO_INSTR:
9740 break;
9741
9742 /* Errors and unexpected events. */
9743 case VMX_EXIT_INIT_SIGNAL:
9744 case VMX_EXIT_SIPI:
9745 case VMX_EXIT_IO_SMI:
9746 case VMX_EXIT_SMI:
9747 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9748 case VMX_EXIT_ERR_MSR_LOAD:
9749 case VMX_EXIT_ERR_MACHINE_CHECK:
9750 break;
9751
9752 default:
9753 AssertMsgFailed(("Unexpected exit=%#x\n", uExitReason));
9754 break;
9755 }
9756#undef SET_BOTH
9757#undef SET_EXIT
9758
9759 /*
9760 * Dtrace tracepoints go first. We do them here at once so we don't
9761 * have to copy the guest state saving and stuff a few dozen times.
9762 * Down side is that we've got to repeat the switch, though this time
9763 * we use enmEvent since the probes are a subset of what DBGF does.
9764 */
9765 if (fDtrace1 || fDtrace2)
9766 {
9767 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9768 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9769 switch (enmEvent1)
9770 {
9771 /** @todo consider which extra parameters would be helpful for each probe. */
9772 case DBGFEVENT_END: break;
9773 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9774 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9775 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9776 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9777 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9778 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9779 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9780 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9781 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9782 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9783 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9784 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9785 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9786 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9787 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9788 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9789 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9790 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9791 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9792 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9793 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9794 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9795 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9796 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9797 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9798 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9799 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9800 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9801 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9802 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9803 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9804 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9805 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9806 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9807 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9808 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9809 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9810 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9811 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9812 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9813 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9814 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9815 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9816 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9817 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9818 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9819 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9820 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9821 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9822 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9823 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9824 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9825 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9826 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9827 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9828 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9829 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9830 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9831 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9832 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9833 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9834 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9835 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9836 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9837 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9838 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9839 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9840 }
9841 switch (enmEvent2)
9842 {
9843 /** @todo consider which extra parameters would be helpful for each probe. */
9844 case DBGFEVENT_END: break;
9845 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9846 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9847 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9848 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9849 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9850 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9851 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9852 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9853 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9854 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9855 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9856 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9857 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9858 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9859 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9860 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9861 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9862 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9863 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9864 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9865 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9866 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9867 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9868 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9869 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9870 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9871 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9872 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9873 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9874 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9875 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9876 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9877 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9878 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9879 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9880 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9881 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9882 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9883 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9884 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9885 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9886 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9887 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9888 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9889 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9890 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9891 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9892 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9893 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9894 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9895 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9896 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9897 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9898 }
9899 }
9900
9901 /*
9902 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9903 * the DBGF call will do a full check).
9904 *
9905 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9906 * Note! If we have to events, we prioritize the first, i.e. the instruction
9907 * one, in order to avoid event nesting.
9908 */
9909 if ( enmEvent1 != DBGFEVENT_END
9910 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9911 {
9912 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9913 if (rcStrict != VINF_SUCCESS)
9914 return rcStrict;
9915 }
9916 else if ( enmEvent2 != DBGFEVENT_END
9917 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9918 {
9919 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9920 if (rcStrict != VINF_SUCCESS)
9921 return rcStrict;
9922 }
9923
9924 return VINF_SUCCESS;
9925}
9926
9927
9928/**
9929 * Single-stepping VM-exit filtering.
9930 *
9931 * This is preprocessing the exits and deciding whether we've gotten far enough
9932 * to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit handling is
9933 * performed.
9934 *
9935 * @returns Strict VBox status code (i.e. informational status codes too).
9936 * @param pVM The cross context VM structure.
9937 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9938 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9939 * out-of-sync. Make sure to update the required
9940 * fields before using them.
9941 * @param pVmxTransient Pointer to the VMX-transient structure.
9942 * @param uExitReason The VM-exit reason.
9943 * @param pDbgState The debug state.
9944 */
9945DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9946 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9947{
9948 /*
9949 * Expensive (saves context) generic dtrace exit probe.
9950 */
9951 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9952 { /* more likely */ }
9953 else
9954 {
9955 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9956 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9957 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9958 }
9959
9960 /*
9961 * Check for host NMI, just to get that out of the way.
9962 */
9963 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9964 { /* normally likely */ }
9965 else
9966 {
9967 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9968 AssertRCReturn(rc2, rc2);
9969 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9970 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9971 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9972 }
9973
9974 /*
9975 * Check for single stepping event if we're stepping.
9976 */
9977 if (pVCpu->hm.s.fSingleInstruction)
9978 {
9979 switch (uExitReason)
9980 {
9981 case VMX_EXIT_MTF:
9982 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9983
9984 /* Various events: */
9985 case VMX_EXIT_XCPT_OR_NMI:
9986 case VMX_EXIT_EXT_INT:
9987 case VMX_EXIT_TRIPLE_FAULT:
9988 case VMX_EXIT_INT_WINDOW:
9989 case VMX_EXIT_NMI_WINDOW:
9990 case VMX_EXIT_TASK_SWITCH:
9991 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9992 case VMX_EXIT_APIC_ACCESS:
9993 case VMX_EXIT_EPT_VIOLATION:
9994 case VMX_EXIT_EPT_MISCONFIG:
9995 case VMX_EXIT_PREEMPT_TIMER:
9996
9997 /* Instruction specific VM-exits: */
9998 case VMX_EXIT_CPUID:
9999 case VMX_EXIT_GETSEC:
10000 case VMX_EXIT_HLT:
10001 case VMX_EXIT_INVD:
10002 case VMX_EXIT_INVLPG:
10003 case VMX_EXIT_RDPMC:
10004 case VMX_EXIT_RDTSC:
10005 case VMX_EXIT_RSM:
10006 case VMX_EXIT_VMCALL:
10007 case VMX_EXIT_VMCLEAR:
10008 case VMX_EXIT_VMLAUNCH:
10009 case VMX_EXIT_VMPTRLD:
10010 case VMX_EXIT_VMPTRST:
10011 case VMX_EXIT_VMREAD:
10012 case VMX_EXIT_VMRESUME:
10013 case VMX_EXIT_VMWRITE:
10014 case VMX_EXIT_VMXOFF:
10015 case VMX_EXIT_VMXON:
10016 case VMX_EXIT_MOV_CRX:
10017 case VMX_EXIT_MOV_DRX:
10018 case VMX_EXIT_IO_INSTR:
10019 case VMX_EXIT_RDMSR:
10020 case VMX_EXIT_WRMSR:
10021 case VMX_EXIT_MWAIT:
10022 case VMX_EXIT_MONITOR:
10023 case VMX_EXIT_PAUSE:
10024 case VMX_EXIT_XDTR_ACCESS:
10025 case VMX_EXIT_TR_ACCESS:
10026 case VMX_EXIT_INVEPT:
10027 case VMX_EXIT_RDTSCP:
10028 case VMX_EXIT_INVVPID:
10029 case VMX_EXIT_WBINVD:
10030 case VMX_EXIT_XSETBV:
10031 case VMX_EXIT_RDRAND:
10032 case VMX_EXIT_INVPCID:
10033 case VMX_EXIT_VMFUNC:
10034 case VMX_EXIT_RDSEED:
10035 case VMX_EXIT_XSAVES:
10036 case VMX_EXIT_XRSTORS:
10037 {
10038 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10039 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10040 AssertRCReturn(rc2, rc2);
10041 if ( pMixedCtx->rip != pDbgState->uRipStart
10042 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
10043 return VINF_EM_DBG_STEPPED;
10044 break;
10045 }
10046
10047 /* Errors and unexpected events: */
10048 case VMX_EXIT_INIT_SIGNAL:
10049 case VMX_EXIT_SIPI:
10050 case VMX_EXIT_IO_SMI:
10051 case VMX_EXIT_SMI:
10052 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10053 case VMX_EXIT_ERR_MSR_LOAD:
10054 case VMX_EXIT_ERR_MACHINE_CHECK:
10055 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
10056 break;
10057
10058 default:
10059 AssertMsgFailed(("Unexpected exit=%#x\n", uExitReason));
10060 break;
10061 }
10062 }
10063
10064 /*
10065 * Check for debugger event breakpoints and dtrace probes.
10066 */
10067 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
10068 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
10069 {
10070 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10071 if (rcStrict != VINF_SUCCESS)
10072 return rcStrict;
10073 }
10074
10075 /*
10076 * Normal processing.
10077 */
10078#ifdef HMVMX_USE_FUNCTION_TABLE
10079 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
10080#else
10081 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10082#endif
10083}
10084
10085
10086/**
10087 * Single steps guest code using VT-x.
10088 *
10089 * @returns Strict VBox status code (i.e. informational status codes too).
10090 * @param pVM The cross context VM structure.
10091 * @param pVCpu The cross context virtual CPU structure.
10092 * @param pCtx Pointer to the guest-CPU context.
10093 *
10094 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10095 */
10096static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10097{
10098 VMXTRANSIENT VmxTransient;
10099 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10100
10101 /* Set HMCPU indicators. */
10102 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10103 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
10104 pVCpu->hm.s.fDebugWantRdTscExit = false;
10105 pVCpu->hm.s.fUsingDebugLoop = true;
10106
10107 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10108 VMXRUNDBGSTATE DbgState;
10109 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10110 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10111
10112 /*
10113 * The loop.
10114 */
10115 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10116 for (uint32_t cLoops = 0; ; cLoops++)
10117 {
10118 Assert(!HMR0SuspendPending());
10119 HMVMX_ASSERT_CPU_SAFE();
10120 bool fStepping = pVCpu->hm.s.fSingleInstruction;
10121
10122 /*
10123 * Preparatory work for running guest code, this may force us to return
10124 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10125 */
10126 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10127 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10128 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10129 if (rcStrict != VINF_SUCCESS)
10130 break;
10131
10132 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10133 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10134
10135 /*
10136 * Now we can run the guest code.
10137 */
10138 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10139
10140 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10141
10142 /*
10143 * Restore any residual host-state and save any bits shared between host
10144 * and guest into the guest-CPU state. Re-enables interrupts!
10145 */
10146 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
10147
10148 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10149 if (RT_SUCCESS(rcRun))
10150 { /* very likely */ }
10151 else
10152 {
10153 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10154 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10155 return rcRun;
10156 }
10157
10158 /* Profile the VM-exit. */
10159 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10160 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10161 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10162 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10163 HMVMX_START_EXIT_DISPATCH_PROF();
10164
10165 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10166
10167 /*
10168 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10169 */
10170 rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10171 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10172 if (rcStrict != VINF_SUCCESS)
10173 break;
10174 if (cLoops > pVM->hm.s.cMaxResumeLoops)
10175 {
10176 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10177 rcStrict = VINF_EM_RAW_INTERRUPT;
10178 break;
10179 }
10180
10181 /*
10182 * Stepping: Did the RIP change, if so, consider it a single step.
10183 * Otherwise, make sure one of the TFs gets set.
10184 */
10185 if (fStepping)
10186 {
10187 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10188 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10189 AssertRCReturn(rc2, rc2);
10190 if ( pCtx->rip != DbgState.uRipStart
10191 || pCtx->cs.Sel != DbgState.uCsStart)
10192 {
10193 rcStrict = VINF_EM_DBG_STEPPED;
10194 break;
10195 }
10196 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10197 }
10198
10199 /*
10200 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
10201 */
10202 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10203 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10204 }
10205
10206 /*
10207 * Clear the X86_EFL_TF if necessary.
10208 */
10209 if (pVCpu->hm.s.fClearTrapFlag)
10210 {
10211 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10212 AssertRCReturn(rc2, rc2);
10213 pVCpu->hm.s.fClearTrapFlag = false;
10214 pCtx->eflags.Bits.u1TF = 0;
10215 }
10216 /** @todo there seems to be issues with the resume flag when the monitor trap
10217 * flag is pending without being used. Seen early in bios init when
10218 * accessing APIC page in protected mode. */
10219
10220 /*
10221 * Restore VM-exit control settings as we may not reenter this function the
10222 * next time around.
10223 */
10224 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10225
10226 /* Restore HMCPU indicators. */
10227 pVCpu->hm.s.fUsingDebugLoop = false;
10228 pVCpu->hm.s.fDebugWantRdTscExit = false;
10229 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10230
10231 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10232 return rcStrict;
10233}
10234
10235
10236/** @} */
10237
10238
10239/**
10240 * Checks if any expensive dtrace probes are enabled and we should go to the
10241 * debug loop.
10242 *
10243 * @returns true if we should use debug loop, false if not.
10244 */
10245static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10246{
10247 /* It's probably faster to OR the raw 32-bit counter variables together.
10248 Since the variables are in an array and the probes are next to one
10249 another (more or less), we have good locality. So, better read
10250 eight-nine cache lines ever time and only have one conditional, than
10251 128+ conditionals, right? */
10252 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10253 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10254 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10255 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10256 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10257 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10258 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10259 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10260 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10261 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10262 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10263 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10264 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10265 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10266 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10267 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10268 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10269 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10270 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10271 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10272 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10273 ) != 0
10274 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10275 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10276 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10277 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10278 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10279 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10280 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10281 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10282 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10283 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10284 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10285 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10286 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10287 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10288 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10289 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10290 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10291 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10292 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10293 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10294 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10295 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10296 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10297 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10298 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10299 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10300 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10301 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10302 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10303 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10304 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10305 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10306 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10307 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10308 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10309 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10310 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10311 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10312 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10313 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10314 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10315 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10316 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10317 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10318 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10319 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10320 ) != 0
10321 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10322 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10323 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10324 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10325 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10326 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10327 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10328 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10329 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10330 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10331 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10332 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10333 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10334 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10335 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10336 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10337 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10338 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10339 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10340 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10341 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10342 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10343 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10344 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10345 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10346 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10347 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10348 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10349 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10350 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10351 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10352 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10353 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10354 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10355 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10356 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10357 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10358 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10359 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10360 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10361 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10362 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10363 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10364 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10365 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10366 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10367 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10368 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10369 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10370 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10371 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10372 ) != 0;
10373}
10374
10375
10376/**
10377 * Runs the guest code using VT-x.
10378 *
10379 * @returns Strict VBox status code (i.e. informational status codes too).
10380 * @param pVM The cross context VM structure.
10381 * @param pVCpu The cross context virtual CPU structure.
10382 * @param pCtx Pointer to the guest-CPU context.
10383 */
10384VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10385{
10386 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10387 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10388 HMVMX_ASSERT_PREEMPT_SAFE();
10389
10390 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10391
10392 VBOXSTRICTRC rcStrict;
10393 if ( !pVCpu->hm.s.fUseDebugLoop
10394 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10395 && !DBGFIsStepping(pVCpu) )
10396 rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10397 else
10398 rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10399
10400 if (rcStrict == VERR_EM_INTERPRETER)
10401 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10402 else if (rcStrict == VINF_EM_RESET)
10403 rcStrict = VINF_EM_TRIPLE_FAULT;
10404
10405 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10406 if (RT_FAILURE(rc2))
10407 {
10408 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10409 rcStrict = rc2;
10410 }
10411 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10412 return rcStrict;
10413}
10414
10415
10416#ifndef HMVMX_USE_FUNCTION_TABLE
10417DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10418{
10419# ifdef DEBUG_ramshankar
10420# define RETURN_EXIT_CALL(a_CallExpr) \
10421 do { \
10422 int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10423 VBOXSTRICTRC rcStrict = a_CallExpr; \
10424 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10425 return rcStrict; \
10426 } while (0)
10427# else
10428# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10429# endif
10430 switch (rcReason)
10431 {
10432 case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10433 case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10434 case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10435 case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10436 case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10437 case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10438 case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10439 case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10440 case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10441 case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10442 case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10443 case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10444 case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10445 case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10446 case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10447 case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10448 case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10449 case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10450 case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10451 case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10452 case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10453 case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10454 case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10455 case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10456 case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10457 case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10458 case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10459 case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10460 case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10461 case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10462 case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10463 case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10464 case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10465 case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10466
10467 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10468 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10469 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10470 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10471 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10472 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10473 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10474 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10475 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10476
10477 case VMX_EXIT_VMCLEAR:
10478 case VMX_EXIT_VMLAUNCH:
10479 case VMX_EXIT_VMPTRLD:
10480 case VMX_EXIT_VMPTRST:
10481 case VMX_EXIT_VMREAD:
10482 case VMX_EXIT_VMRESUME:
10483 case VMX_EXIT_VMWRITE:
10484 case VMX_EXIT_VMXOFF:
10485 case VMX_EXIT_VMXON:
10486 case VMX_EXIT_INVEPT:
10487 case VMX_EXIT_INVVPID:
10488 case VMX_EXIT_VMFUNC:
10489 case VMX_EXIT_XSAVES:
10490 case VMX_EXIT_XRSTORS:
10491 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10492 case VMX_EXIT_RESERVED_60:
10493 case VMX_EXIT_RDSEED: /* only spurious exits, so undefined */
10494 case VMX_EXIT_RESERVED_62:
10495 default:
10496 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10497 }
10498#undef RETURN_EXIT_CALL
10499}
10500#endif /* !HMVMX_USE_FUNCTION_TABLE */
10501
10502
10503#ifdef VBOX_STRICT
10504/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10505# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10506 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10507
10508# define HMVMX_ASSERT_PREEMPT_CPUID() \
10509 do { \
10510 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10511 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10512 } while (0)
10513
10514# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10515 do { \
10516 AssertPtr(pVCpu); \
10517 AssertPtr(pMixedCtx); \
10518 AssertPtr(pVmxTransient); \
10519 Assert(pVmxTransient->fVMEntryFailed == false); \
10520 Assert(ASMIntAreEnabled()); \
10521 HMVMX_ASSERT_PREEMPT_SAFE(); \
10522 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10523 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10524 HMVMX_ASSERT_PREEMPT_SAFE(); \
10525 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10526 HMVMX_ASSERT_PREEMPT_CPUID(); \
10527 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10528 } while (0)
10529
10530# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10531 do { \
10532 Log4Func(("\n")); \
10533 } while (0)
10534#else /* nonstrict builds: */
10535# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10536 do { \
10537 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10538 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10539 } while (0)
10540# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10541#endif
10542
10543
10544/**
10545 * Advances the guest RIP by the specified number of bytes.
10546 *
10547 * @param pVCpu The cross context virtual CPU structure.
10548 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10549 * out-of-sync. Make sure to update the required fields
10550 * before using them.
10551 * @param cbInstr Number of bytes to advance the RIP by.
10552 *
10553 * @remarks No-long-jump zone!!!
10554 */
10555DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10556{
10557 /* Advance the RIP. */
10558 pMixedCtx->rip += cbInstr;
10559 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10560
10561 /* Update interrupt inhibition. */
10562 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10563 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10564 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10565}
10566
10567
10568/**
10569 * Advances the guest RIP after reading it from the VMCS.
10570 *
10571 * @returns VBox status code, no informational status codes.
10572 * @param pVCpu The cross context virtual CPU structure.
10573 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10574 * out-of-sync. Make sure to update the required fields
10575 * before using them.
10576 * @param pVmxTransient Pointer to the VMX transient structure.
10577 *
10578 * @remarks No-long-jump zone!!!
10579 */
10580static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10581{
10582 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10583 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10584 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10585 AssertRCReturn(rc, rc);
10586
10587 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10588
10589 /*
10590 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10591 * pending debug exception field as it takes care of priority of events.
10592 *
10593 * See Intel spec. 32.2.1 "Debug Exceptions".
10594 */
10595 if ( !pVCpu->hm.s.fSingleInstruction
10596 && pMixedCtx->eflags.Bits.u1TF)
10597 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10598
10599 return VINF_SUCCESS;
10600}
10601
10602
10603/**
10604 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10605 * and update error record fields accordingly.
10606 *
10607 * @return VMX_IGS_* return codes.
10608 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10609 * wrong with the guest state.
10610 *
10611 * @param pVM The cross context VM structure.
10612 * @param pVCpu The cross context virtual CPU structure.
10613 * @param pCtx Pointer to the guest-CPU state.
10614 *
10615 * @remarks This function assumes our cache of the VMCS controls
10616 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10617 */
10618static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10619{
10620#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10621#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10622 uError = (err); \
10623 break; \
10624 } else do { } while (0)
10625
10626 int rc;
10627 uint32_t uError = VMX_IGS_ERROR;
10628 uint32_t u32Val;
10629 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10630
10631 do
10632 {
10633 /*
10634 * CR0.
10635 */
10636 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10637 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10638 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10639 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10640 if (fUnrestrictedGuest)
10641 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
10642
10643 uint32_t u32GuestCR0;
10644 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
10645 AssertRCBreak(rc);
10646 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
10647 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
10648 if ( !fUnrestrictedGuest
10649 && (u32GuestCR0 & X86_CR0_PG)
10650 && !(u32GuestCR0 & X86_CR0_PE))
10651 {
10652 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10653 }
10654
10655 /*
10656 * CR4.
10657 */
10658 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10659 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10660
10661 uint32_t u32GuestCR4;
10662 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
10663 AssertRCBreak(rc);
10664 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
10665 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
10666
10667 /*
10668 * IA32_DEBUGCTL MSR.
10669 */
10670 uint64_t u64Val;
10671 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10672 AssertRCBreak(rc);
10673 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10674 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10675 {
10676 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10677 }
10678 uint64_t u64DebugCtlMsr = u64Val;
10679
10680#ifdef VBOX_STRICT
10681 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10682 AssertRCBreak(rc);
10683 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10684#endif
10685 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10686
10687 /*
10688 * RIP and RFLAGS.
10689 */
10690 uint32_t u32Eflags;
10691#if HC_ARCH_BITS == 64
10692 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10693 AssertRCBreak(rc);
10694 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10695 if ( !fLongModeGuest
10696 || !pCtx->cs.Attr.n.u1Long)
10697 {
10698 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10699 }
10700 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10701 * must be identical if the "IA-32e mode guest" VM-entry
10702 * control is 1 and CS.L is 1. No check applies if the
10703 * CPU supports 64 linear-address bits. */
10704
10705 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10706 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10707 AssertRCBreak(rc);
10708 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10709 VMX_IGS_RFLAGS_RESERVED);
10710 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10711 u32Eflags = u64Val;
10712#else
10713 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10714 AssertRCBreak(rc);
10715 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10716 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10717#endif
10718
10719 if ( fLongModeGuest
10720 || ( fUnrestrictedGuest
10721 && !(u32GuestCR0 & X86_CR0_PE)))
10722 {
10723 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10724 }
10725
10726 uint32_t u32EntryInfo;
10727 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10728 AssertRCBreak(rc);
10729 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10730 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10731 {
10732 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10733 }
10734
10735 /*
10736 * 64-bit checks.
10737 */
10738#if HC_ARCH_BITS == 64
10739 if (fLongModeGuest)
10740 {
10741 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10742 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10743 }
10744
10745 if ( !fLongModeGuest
10746 && (u32GuestCR4 & X86_CR4_PCIDE))
10747 {
10748 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10749 }
10750
10751 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10752 * 51:32 beyond the processor's physical-address width are 0. */
10753
10754 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10755 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10756 {
10757 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10758 }
10759
10760 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10761 AssertRCBreak(rc);
10762 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10763
10764 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10765 AssertRCBreak(rc);
10766 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10767#endif
10768
10769 /*
10770 * PERF_GLOBAL MSR.
10771 */
10772 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10773 {
10774 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10775 AssertRCBreak(rc);
10776 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10777 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10778 }
10779
10780 /*
10781 * PAT MSR.
10782 */
10783 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10784 {
10785 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10786 AssertRCBreak(rc);
10787 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10788 for (unsigned i = 0; i < 8; i++)
10789 {
10790 uint8_t u8Val = (u64Val & 0xff);
10791 if ( u8Val != 0 /* UC */
10792 && u8Val != 1 /* WC */
10793 && u8Val != 4 /* WT */
10794 && u8Val != 5 /* WP */
10795 && u8Val != 6 /* WB */
10796 && u8Val != 7 /* UC- */)
10797 {
10798 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10799 }
10800 u64Val >>= 8;
10801 }
10802 }
10803
10804 /*
10805 * EFER MSR.
10806 */
10807 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10808 {
10809 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10810 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10811 AssertRCBreak(rc);
10812 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10813 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10814 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10815 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10816 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10817 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10818 || !(u32GuestCR0 & X86_CR0_PG)
10819 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10820 VMX_IGS_EFER_LMA_LME_MISMATCH);
10821 }
10822
10823 /*
10824 * Segment registers.
10825 */
10826 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10827 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10828 if (!(u32Eflags & X86_EFL_VM))
10829 {
10830 /* CS */
10831 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10832 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10833 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10834 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10835 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10836 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10837 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10838 /* CS cannot be loaded with NULL in protected mode. */
10839 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10840 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10841 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10842 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10843 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10844 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10845 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10846 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10847 else
10848 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10849
10850 /* SS */
10851 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10852 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10853 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10854 if ( !(pCtx->cr0 & X86_CR0_PE)
10855 || pCtx->cs.Attr.n.u4Type == 3)
10856 {
10857 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10858 }
10859 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10860 {
10861 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10862 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10863 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10864 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10865 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10866 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10867 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10868 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10869 }
10870
10871 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
10872 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10873 {
10874 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10875 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10876 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10877 || pCtx->ds.Attr.n.u4Type > 11
10878 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10879 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10880 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10881 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10882 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10883 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10884 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10885 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10886 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10887 }
10888 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10889 {
10890 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10891 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10892 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10893 || pCtx->es.Attr.n.u4Type > 11
10894 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10895 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10896 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10897 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10898 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10899 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10900 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10901 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10902 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10903 }
10904 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10905 {
10906 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10907 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10908 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10909 || pCtx->fs.Attr.n.u4Type > 11
10910 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10911 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10912 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10913 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10914 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10915 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10916 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10917 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10918 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10919 }
10920 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10921 {
10922 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10923 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10924 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10925 || pCtx->gs.Attr.n.u4Type > 11
10926 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10927 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10928 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10929 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10930 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10931 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10932 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10933 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10934 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10935 }
10936 /* 64-bit capable CPUs. */
10937#if HC_ARCH_BITS == 64
10938 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10939 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10940 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10941 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10942 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10943 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10944 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10945 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10946 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10947 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10948 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10949#endif
10950 }
10951 else
10952 {
10953 /* V86 mode checks. */
10954 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10955 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10956 {
10957 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10958 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10959 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10960 }
10961 else
10962 {
10963 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10964 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10965 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10966 }
10967
10968 /* CS */
10969 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10970 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10971 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10972 /* SS */
10973 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10974 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10975 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10976 /* DS */
10977 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10978 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10979 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10980 /* ES */
10981 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10982 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10983 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10984 /* FS */
10985 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10986 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10987 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10988 /* GS */
10989 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10990 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10991 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10992 /* 64-bit capable CPUs. */
10993#if HC_ARCH_BITS == 64
10994 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10995 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10996 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10997 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10998 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10999 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
11000 VMX_IGS_LONGMODE_SS_BASE_INVALID);
11001 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
11002 VMX_IGS_LONGMODE_DS_BASE_INVALID);
11003 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
11004 VMX_IGS_LONGMODE_ES_BASE_INVALID);
11005#endif
11006 }
11007
11008 /*
11009 * TR.
11010 */
11011 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
11012 /* 64-bit capable CPUs. */
11013#if HC_ARCH_BITS == 64
11014 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
11015#endif
11016 if (fLongModeGuest)
11017 {
11018 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
11019 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
11020 }
11021 else
11022 {
11023 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
11024 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
11025 VMX_IGS_TR_ATTR_TYPE_INVALID);
11026 }
11027 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
11028 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
11029 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
11030 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
11031 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11032 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
11033 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11034 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
11035
11036 /*
11037 * GDTR and IDTR.
11038 */
11039#if HC_ARCH_BITS == 64
11040 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
11041 AssertRCBreak(rc);
11042 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
11043
11044 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
11045 AssertRCBreak(rc);
11046 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
11047#endif
11048
11049 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
11050 AssertRCBreak(rc);
11051 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11052
11053 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
11054 AssertRCBreak(rc);
11055 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11056
11057 /*
11058 * Guest Non-Register State.
11059 */
11060 /* Activity State. */
11061 uint32_t u32ActivityState;
11062 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
11063 AssertRCBreak(rc);
11064 HMVMX_CHECK_BREAK( !u32ActivityState
11065 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
11066 VMX_IGS_ACTIVITY_STATE_INVALID);
11067 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
11068 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
11069 uint32_t u32IntrState;
11070 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
11071 AssertRCBreak(rc);
11072 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
11073 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11074 {
11075 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11076 }
11077
11078 /** @todo Activity state and injecting interrupts. Left as a todo since we
11079 * currently don't use activity states but ACTIVE. */
11080
11081 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11082 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11083
11084 /* Guest interruptibility-state. */
11085 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11086 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11087 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
11088 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11089 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11090 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11091 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11092 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11093 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11094 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
11095 {
11096 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11097 {
11098 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11099 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11100 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11101 }
11102 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11103 {
11104 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11105 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11106 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11107 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11108 }
11109 }
11110 /** @todo Assumes the processor is not in SMM. */
11111 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11112 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11113 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11114 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11115 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11116 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11117 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11118 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11119 {
11120 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11121 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11122 }
11123
11124 /* Pending debug exceptions. */
11125#if HC_ARCH_BITS == 64
11126 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11127 AssertRCBreak(rc);
11128 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11129 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11130 u32Val = u64Val; /* For pending debug exceptions checks below. */
11131#else
11132 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11133 AssertRCBreak(rc);
11134 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11135 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11136#endif
11137
11138 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11139 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11140 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11141 {
11142 if ( (u32Eflags & X86_EFL_TF)
11143 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11144 {
11145 /* Bit 14 is PendingDebug.BS. */
11146 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11147 }
11148 if ( !(u32Eflags & X86_EFL_TF)
11149 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11150 {
11151 /* Bit 14 is PendingDebug.BS. */
11152 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11153 }
11154 }
11155
11156 /* VMCS link pointer. */
11157 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11158 AssertRCBreak(rc);
11159 if (u64Val != UINT64_C(0xffffffffffffffff))
11160 {
11161 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11162 /** @todo Bits beyond the processor's physical-address width MBZ. */
11163 /** @todo 32-bit located in memory referenced by value of this field (as a
11164 * physical address) must contain the processor's VMCS revision ID. */
11165 /** @todo SMM checks. */
11166 }
11167
11168 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11169 * not using Nested Paging? */
11170 if ( pVM->hm.s.fNestedPaging
11171 && !fLongModeGuest
11172 && CPUMIsGuestInPAEModeEx(pCtx))
11173 {
11174 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11175 AssertRCBreak(rc);
11176 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11177
11178 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11179 AssertRCBreak(rc);
11180 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11181
11182 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11183 AssertRCBreak(rc);
11184 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11185
11186 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11187 AssertRCBreak(rc);
11188 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11189 }
11190
11191 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11192 if (uError == VMX_IGS_ERROR)
11193 uError = VMX_IGS_REASON_NOT_FOUND;
11194 } while (0);
11195
11196 pVCpu->hm.s.u32HMError = uError;
11197 return uError;
11198
11199#undef HMVMX_ERROR_BREAK
11200#undef HMVMX_CHECK_BREAK
11201}
11202
11203/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11204/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11205/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11206
11207/** @name VM-exit handlers.
11208 * @{
11209 */
11210
11211/**
11212 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11213 */
11214HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11215{
11216 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11217 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11218 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11219 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11220 return VINF_SUCCESS;
11221 return VINF_EM_RAW_INTERRUPT;
11222}
11223
11224
11225/**
11226 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11227 */
11228HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11229{
11230 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11231 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11232
11233 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11234 AssertRCReturn(rc, rc);
11235
11236 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11237 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11238 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11239 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11240
11241 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11242 {
11243 /*
11244 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11245 * anything we inject is not going to cause a VM-exit directly for the event being injected.
11246 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11247 *
11248 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11249 */
11250 VMXDispatchHostNmi();
11251 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11252 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11253 return VINF_SUCCESS;
11254 }
11255
11256 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11257 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11258 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11259 { /* likely */ }
11260 else
11261 {
11262 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11263 rcStrictRc1 = VINF_SUCCESS;
11264 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11265 return rcStrictRc1;
11266 }
11267
11268 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11269 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11270 switch (uIntType)
11271 {
11272 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11273 Assert(uVector == X86_XCPT_DB);
11274 /* no break */
11275 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11276 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11277 /* no break */
11278 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11279 {
11280 /*
11281 * If there's any exception caused as a result of event injection, go back to
11282 * the interpreter. The page-fault case is complicated and we manually handle
11283 * any currently pending event in hmR0VmxExitXcptPF. Nested #ACs are already
11284 * handled in hmR0VmxCheckExitDueToEventDelivery.
11285 */
11286 if (!pVCpu->hm.s.Event.fPending)
11287 { /* likely */ }
11288 else if ( uVector != X86_XCPT_PF
11289 && uVector != X86_XCPT_AC)
11290 {
11291 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
11292 rc = VERR_EM_INTERPRETER;
11293 break;
11294 }
11295
11296 switch (uVector)
11297 {
11298 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11299 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11300 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11301 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11302 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11303 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11304 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11305
11306 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11307 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11308 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11309 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11310 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11311 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11312 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11313 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11314 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11315 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11316 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11317 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11318 default:
11319 {
11320 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11321 AssertRCReturn(rc, rc);
11322
11323 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11324 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11325 {
11326 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11327 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11328 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11329
11330 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11331 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11332 AssertRCReturn(rc, rc);
11333 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11334 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11335 0 /* GCPtrFaultAddress */);
11336 AssertRCReturn(rc, rc);
11337 }
11338 else
11339 {
11340 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11341 pVCpu->hm.s.u32HMError = uVector;
11342 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11343 }
11344 break;
11345 }
11346 }
11347 break;
11348 }
11349
11350 default:
11351 {
11352 pVCpu->hm.s.u32HMError = uExitIntInfo;
11353 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11354 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11355 break;
11356 }
11357 }
11358 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11359 return rc;
11360}
11361
11362
11363/**
11364 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11365 */
11366HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11367{
11368 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11369
11370 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11371 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11372
11373 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11374 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11375 return VINF_SUCCESS;
11376}
11377
11378
11379/**
11380 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11381 */
11382HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11383{
11384 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11385 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11386 {
11387 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11388 HMVMX_RETURN_UNEXPECTED_EXIT();
11389 }
11390
11391 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11392
11393 /*
11394 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11395 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11396 */
11397 uint32_t uIntrState = 0;
11398 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11399 AssertRCReturn(rc, rc);
11400
11401 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11402 if ( fBlockSti
11403 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11404 {
11405 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11406 }
11407
11408 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11409 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11410
11411 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11412 return VINF_SUCCESS;
11413}
11414
11415
11416/**
11417 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11418 */
11419HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11420{
11421 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11422 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11423 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11424}
11425
11426
11427/**
11428 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11429 */
11430HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11431{
11432 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11433 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11434 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11435}
11436
11437
11438/**
11439 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11440 */
11441HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11442{
11443 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11444 PVM pVM = pVCpu->CTX_SUFF(pVM);
11445 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11446 if (RT_LIKELY(rc == VINF_SUCCESS))
11447 {
11448 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11449 Assert(pVmxTransient->cbInstr == 2);
11450 }
11451 else
11452 {
11453 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11454 rc = VERR_EM_INTERPRETER;
11455 }
11456 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11457 return rc;
11458}
11459
11460
11461/**
11462 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11463 */
11464HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11465{
11466 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11467 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11468 AssertRCReturn(rc, rc);
11469
11470 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11471 return VINF_EM_RAW_EMULATE_INSTR;
11472
11473 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11474 HMVMX_RETURN_UNEXPECTED_EXIT();
11475}
11476
11477
11478/**
11479 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11480 */
11481HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11482{
11483 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11484 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11485 AssertRCReturn(rc, rc);
11486
11487 PVM pVM = pVCpu->CTX_SUFF(pVM);
11488 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11489 if (RT_LIKELY(rc == VINF_SUCCESS))
11490 {
11491 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11492 Assert(pVmxTransient->cbInstr == 2);
11493 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11494 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11495 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11496 }
11497 else
11498 rc = VERR_EM_INTERPRETER;
11499 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11500 return rc;
11501}
11502
11503
11504/**
11505 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11506 */
11507HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11508{
11509 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11510 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11511 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11512 AssertRCReturn(rc, rc);
11513
11514 PVM pVM = pVCpu->CTX_SUFF(pVM);
11515 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11516 if (RT_SUCCESS(rc))
11517 {
11518 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11519 Assert(pVmxTransient->cbInstr == 3);
11520 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11521 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11522 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11523 }
11524 else
11525 {
11526 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11527 rc = VERR_EM_INTERPRETER;
11528 }
11529 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11530 return rc;
11531}
11532
11533
11534/**
11535 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11536 */
11537HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11538{
11539 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11540 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11541 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11542 AssertRCReturn(rc, rc);
11543
11544 PVM pVM = pVCpu->CTX_SUFF(pVM);
11545 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11546 if (RT_LIKELY(rc == VINF_SUCCESS))
11547 {
11548 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11549 Assert(pVmxTransient->cbInstr == 2);
11550 }
11551 else
11552 {
11553 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11554 rc = VERR_EM_INTERPRETER;
11555 }
11556 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11557 return rc;
11558}
11559
11560
11561/**
11562 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11563 */
11564HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11565{
11566 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11567 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11568
11569 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11570 if (pVCpu->hm.s.fHypercallsEnabled)
11571 {
11572#if 0
11573 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11574#else
11575 /* Aggressive state sync. for now. */
11576 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11577 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11578 AssertRCReturn(rc, rc);
11579#endif
11580
11581 /* Perform the hypercall. */
11582 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11583 if (rcStrict == VINF_SUCCESS)
11584 {
11585 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11586 AssertRCReturn(rc, rc);
11587 }
11588 else
11589 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11590 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11591 || RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
11592
11593 /* If the hypercall changes anything other than guest's general-purpose registers,
11594 we would need to reload the guest changed bits here before VM-entry. */
11595 }
11596 else
11597 Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11598
11599 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11600 if (RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)))
11601 {
11602 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11603 rcStrict = VINF_SUCCESS;
11604 }
11605
11606 return rcStrict;
11607}
11608
11609
11610/**
11611 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11612 */
11613HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11614{
11615 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11616 PVM pVM = pVCpu->CTX_SUFF(pVM);
11617 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11618
11619 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11620 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11621 AssertRCReturn(rc, rc);
11622
11623 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11624 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11625 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11626 else
11627 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11628 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11629 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
11630 return rcStrict;
11631}
11632
11633
11634/**
11635 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11636 */
11637HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11638{
11639 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11640 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11641 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11642 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11643 AssertRCReturn(rc, rc);
11644
11645 PVM pVM = pVCpu->CTX_SUFF(pVM);
11646 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11647 if (RT_LIKELY(rc == VINF_SUCCESS))
11648 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11649 else
11650 {
11651 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11652 rc = VERR_EM_INTERPRETER;
11653 }
11654 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11655 return rc;
11656}
11657
11658
11659/**
11660 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11661 */
11662HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11663{
11664 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11665 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11666 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11667 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11668 AssertRCReturn(rc, rc);
11669
11670 PVM pVM = pVCpu->CTX_SUFF(pVM);
11671 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11672 rc = VBOXSTRICTRC_VAL(rc2);
11673 if (RT_LIKELY( rc == VINF_SUCCESS
11674 || rc == VINF_EM_HALT))
11675 {
11676 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11677 AssertRCReturn(rc3, rc3);
11678
11679 if ( rc == VINF_EM_HALT
11680 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11681 {
11682 rc = VINF_SUCCESS;
11683 }
11684 }
11685 else
11686 {
11687 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11688 rc = VERR_EM_INTERPRETER;
11689 }
11690 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11691 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11692 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11693 return rc;
11694}
11695
11696
11697/**
11698 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11699 */
11700HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11701{
11702 /*
11703 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
11704 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
11705 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
11706 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
11707 */
11708 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11709 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11710 HMVMX_RETURN_UNEXPECTED_EXIT();
11711}
11712
11713
11714/**
11715 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11716 */
11717HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11718{
11719 /*
11720 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
11721 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
11722 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
11723 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
11724 */
11725 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11726 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11727 HMVMX_RETURN_UNEXPECTED_EXIT();
11728}
11729
11730
11731/**
11732 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11733 */
11734HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11735{
11736 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11737 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11738 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11739 HMVMX_RETURN_UNEXPECTED_EXIT();
11740}
11741
11742
11743/**
11744 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11745 */
11746HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11747{
11748 /*
11749 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
11750 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
11751 * See Intel spec. 25.3 "Other Causes of VM-exits".
11752 */
11753 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11754 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11755 HMVMX_RETURN_UNEXPECTED_EXIT();
11756}
11757
11758
11759/**
11760 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11761 * VM-exit.
11762 */
11763HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11764{
11765 /*
11766 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11767 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11768 *
11769 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11770 * See Intel spec. "23.8 Restrictions on VMX operation".
11771 */
11772 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11773 return VINF_SUCCESS;
11774}
11775
11776
11777/**
11778 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11779 * VM-exit.
11780 */
11781HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11782{
11783 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11784 return VINF_EM_RESET;
11785}
11786
11787
11788/**
11789 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11790 */
11791HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11792{
11793 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11794 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11795
11796 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11797 AssertRCReturn(rc, rc);
11798
11799 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11800 rc = VINF_SUCCESS;
11801 else
11802 rc = VINF_EM_HALT;
11803
11804 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11805 if (rc != VINF_SUCCESS)
11806 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11807 return rc;
11808}
11809
11810
11811/**
11812 * VM-exit handler for instructions that result in a \#UD exception delivered to
11813 * the guest.
11814 */
11815HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11816{
11817 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11818 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11819 return VINF_SUCCESS;
11820}
11821
11822
11823/**
11824 * VM-exit handler for expiry of the VMX preemption timer.
11825 */
11826HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11827{
11828 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11829
11830 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11831 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11832
11833 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11834 PVM pVM = pVCpu->CTX_SUFF(pVM);
11835 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11836 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11837 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11838}
11839
11840
11841/**
11842 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11843 */
11844HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11845{
11846 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11847
11848 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11849 rc |= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, false /*fNeedRsp*/);
11850 rc |= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11851 AssertRCReturn(rc, rc);
11852
11853 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11854 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
11855
11856 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11857
11858 return rcStrict;
11859}
11860
11861
11862/**
11863 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11864 */
11865HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11866{
11867 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11868
11869 /* The guest should not invalidate the host CPU's TLBs, fallback to interpreter. */
11870 /** @todo implement EMInterpretInvpcid() */
11871 return VERR_EM_INTERPRETER;
11872}
11873
11874
11875/**
11876 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11877 * Error VM-exit.
11878 */
11879HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11880{
11881 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11882 AssertRCReturn(rc, rc);
11883
11884 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11885 AssertRCReturn(rc, rc);
11886
11887 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11888 NOREF(uInvalidReason);
11889
11890#ifdef VBOX_STRICT
11891 uint32_t uIntrState;
11892 RTHCUINTREG uHCReg;
11893 uint64_t u64Val;
11894 uint32_t u32Val;
11895
11896 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11897 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11898 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11899 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11900 AssertRCReturn(rc, rc);
11901
11902 Log4(("uInvalidReason %u\n", uInvalidReason));
11903 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11904 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11905 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11906 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
11907
11908 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11909 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11910 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11911 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11912 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11913 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11914 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11915 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11916 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11917 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11918 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11919 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11920#else
11921 NOREF(pVmxTransient);
11922#endif
11923
11924 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11925 return VERR_VMX_INVALID_GUEST_STATE;
11926}
11927
11928
11929/**
11930 * VM-exit handler for VM-entry failure due to an MSR-load
11931 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11932 */
11933HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11934{
11935 NOREF(pVmxTransient);
11936 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11937 HMVMX_RETURN_UNEXPECTED_EXIT();
11938}
11939
11940
11941/**
11942 * VM-exit handler for VM-entry failure due to a machine-check event
11943 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11944 */
11945HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11946{
11947 NOREF(pVmxTransient);
11948 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11949 HMVMX_RETURN_UNEXPECTED_EXIT();
11950}
11951
11952
11953/**
11954 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11955 * theory.
11956 */
11957HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11958{
11959 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
11960 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
11961 return VERR_VMX_UNDEFINED_EXIT_CODE;
11962}
11963
11964
11965/**
11966 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11967 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11968 * Conditional VM-exit.
11969 */
11970HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11971{
11972 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11973
11974 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
11975 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11976 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
11977 return VERR_EM_INTERPRETER;
11978 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11979 HMVMX_RETURN_UNEXPECTED_EXIT();
11980}
11981
11982
11983/**
11984 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11985 */
11986HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11987{
11988 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11989
11990 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
11991 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
11992 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
11993 return VERR_EM_INTERPRETER;
11994 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11995 HMVMX_RETURN_UNEXPECTED_EXIT();
11996}
11997
11998
11999/**
12000 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
12001 */
12002HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12003{
12004 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12005
12006 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
12007 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12008 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12009 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12010 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12011 {
12012 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12013 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12014 }
12015 AssertRCReturn(rc, rc);
12016 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
12017
12018#ifdef VBOX_STRICT
12019 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
12020 {
12021 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
12022 && pMixedCtx->ecx != MSR_K6_EFER)
12023 {
12024 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12025 pMixedCtx->ecx));
12026 HMVMX_RETURN_UNEXPECTED_EXIT();
12027 }
12028 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12029 {
12030 VMXMSREXITREAD enmRead;
12031 VMXMSREXITWRITE enmWrite;
12032 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12033 AssertRCReturn(rc2, rc2);
12034 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
12035 {
12036 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12037 HMVMX_RETURN_UNEXPECTED_EXIT();
12038 }
12039 }
12040 }
12041#endif
12042
12043 PVM pVM = pVCpu->CTX_SUFF(pVM);
12044 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12045 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
12046 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
12047 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
12048 if (RT_SUCCESS(rc))
12049 {
12050 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12051 Assert(pVmxTransient->cbInstr == 2);
12052 }
12053 return rc;
12054}
12055
12056
12057/**
12058 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12059 */
12060HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12061{
12062 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12063 PVM pVM = pVCpu->CTX_SUFF(pVM);
12064 int rc = VINF_SUCCESS;
12065
12066 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
12067 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12068 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12069 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12070 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12071 {
12072 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12073 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12074 }
12075 AssertRCReturn(rc, rc);
12076 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
12077
12078 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12079 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
12080 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12081
12082 if (RT_SUCCESS(rc))
12083 {
12084 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12085
12086 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
12087 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
12088 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
12089 {
12090 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12091 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
12092 EMInterpretWrmsr() changes it. */
12093 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12094 }
12095 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12096 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12097 else if (pMixedCtx->ecx == MSR_K6_EFER)
12098 {
12099 /*
12100 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12101 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12102 * the other bits as well, SCE and NXE. See @bugref{7368}.
12103 */
12104 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
12105 }
12106
12107 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12108 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12109 {
12110 switch (pMixedCtx->ecx)
12111 {
12112 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
12113 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
12114 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
12115 case MSR_K8_FS_BASE: /* no break */
12116 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
12117 case MSR_K6_EFER: /* already handled above */ break;
12118 default:
12119 {
12120 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12121 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12122 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12123 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
12124 break;
12125 }
12126 }
12127 }
12128#ifdef VBOX_STRICT
12129 else
12130 {
12131 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12132 switch (pMixedCtx->ecx)
12133 {
12134 case MSR_IA32_SYSENTER_CS:
12135 case MSR_IA32_SYSENTER_EIP:
12136 case MSR_IA32_SYSENTER_ESP:
12137 case MSR_K8_FS_BASE:
12138 case MSR_K8_GS_BASE:
12139 {
12140 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12141 HMVMX_RETURN_UNEXPECTED_EXIT();
12142 }
12143
12144 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12145 default:
12146 {
12147 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12148 {
12149 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12150 if (pMixedCtx->ecx != MSR_K6_EFER)
12151 {
12152 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12153 pMixedCtx->ecx));
12154 HMVMX_RETURN_UNEXPECTED_EXIT();
12155 }
12156 }
12157
12158 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12159 {
12160 VMXMSREXITREAD enmRead;
12161 VMXMSREXITWRITE enmWrite;
12162 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12163 AssertRCReturn(rc2, rc2);
12164 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12165 {
12166 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12167 HMVMX_RETURN_UNEXPECTED_EXIT();
12168 }
12169 }
12170 break;
12171 }
12172 }
12173 }
12174#endif /* VBOX_STRICT */
12175 }
12176 return rc;
12177}
12178
12179
12180/**
12181 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12182 */
12183HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12184{
12185 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12186
12187 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12188 return VINF_EM_RAW_INTERRUPT;
12189}
12190
12191
12192/**
12193 * VM-exit handler for when the TPR value is lowered below the specified
12194 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12195 */
12196HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12197{
12198 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12199 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12200
12201 /*
12202 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
12203 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
12204 * resume guest execution.
12205 */
12206 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12207 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12208 return VINF_SUCCESS;
12209}
12210
12211
12212/**
12213 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12214 * VM-exit.
12215 *
12216 * @retval VINF_SUCCESS when guest execution can continue.
12217 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12218 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12219 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12220 * interpreter.
12221 */
12222HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12223{
12224 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12225 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12226 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12227 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12228 AssertRCReturn(rc, rc);
12229
12230 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12231 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12232 PVM pVM = pVCpu->CTX_SUFF(pVM);
12233 VBOXSTRICTRC rcStrict;
12234 rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, true /*fNeedRsp*/);
12235 switch (uAccessType)
12236 {
12237 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12238 {
12239 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12240 AssertRCReturn(rc, rc);
12241
12242 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12243 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12244 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12245 AssertMsg( rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE
12246 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12247 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12248 {
12249 case 0: /* CR0 */
12250 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12251 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12252 break;
12253 case 2: /* CR2 */
12254 /* Nothing to do here, CR2 it's not part of the VMCS. */
12255 break;
12256 case 3: /* CR3 */
12257 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
12258 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12259 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12260 break;
12261 case 4: /* CR4 */
12262 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12263 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12264 VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12265 break;
12266 case 8: /* CR8 */
12267 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12268 /* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12269 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12270 break;
12271 default:
12272 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12273 break;
12274 }
12275
12276 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12277 break;
12278 }
12279
12280 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12281 {
12282 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12283 AssertRCReturn(rc, rc);
12284
12285 Assert( !pVM->hm.s.fNestedPaging
12286 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12287 || pVCpu->hm.s.fUsingDebugLoop
12288 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12289
12290 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12291 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12292 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12293
12294 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12295 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12296 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12297 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12298 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12299 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12300 VBOXSTRICTRC_VAL(rcStrict)));
12301 break;
12302 }
12303
12304 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12305 {
12306 AssertRCReturn(rc, rc);
12307 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12308 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12309 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12310 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12311 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12312 break;
12313 }
12314
12315 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12316 {
12317 AssertRCReturn(rc, rc);
12318 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12319 VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12320 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE,
12321 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12322 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12323 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12324 break;
12325 }
12326
12327 default:
12328 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12329 VERR_VMX_UNEXPECTED_EXCEPTION);
12330 }
12331
12332 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12333 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12334 NOREF(pVM);
12335 return rcStrict;
12336}
12337
12338
12339/**
12340 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12341 * VM-exit.
12342 */
12343HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12344{
12345 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12346 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12347
12348 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12349 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12350 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12351 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12352 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12353 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12354 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12355 AssertRCReturn(rc2, rc2);
12356
12357 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12358 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12359 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12360 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12361 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12362 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12363 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12364 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12365 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12366
12367 /* I/O operation lookup arrays. */
12368 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12369 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12370
12371 VBOXSTRICTRC rcStrict;
12372 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12373 uint32_t const cbInstr = pVmxTransient->cbInstr;
12374 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12375 PVM pVM = pVCpu->CTX_SUFF(pVM);
12376 if (fIOString)
12377 {
12378#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12379 See @bugref{5752#c158}. Should work now. */
12380 /*
12381 * INS/OUTS - I/O String instruction.
12382 *
12383 * Use instruction-information if available, otherwise fall back on
12384 * interpreting the instruction.
12385 */
12386 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12387 fIOWrite ? 'w' : 'r'));
12388 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12389 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12390 {
12391 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12392 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12393 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12394 AssertRCReturn(rc2, rc2);
12395 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12396 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12397 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12398 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12399 if (fIOWrite)
12400 {
12401 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12402 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12403 }
12404 else
12405 {
12406 /*
12407 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12408 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12409 * See Intel Instruction spec. for "INS".
12410 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12411 */
12412 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12413 }
12414 }
12415 else
12416 {
12417 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12418 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12419 AssertRCReturn(rc2, rc2);
12420 rcStrict = IEMExecOne(pVCpu);
12421 }
12422 /** @todo IEM needs to be setting these flags somehow. */
12423 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12424 fUpdateRipAlready = true;
12425#else
12426 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12427 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12428 if (RT_SUCCESS(rcStrict))
12429 {
12430 if (fIOWrite)
12431 {
12432 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12433 (DISCPUMODE)pDis->uAddrMode, cbValue);
12434 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12435 }
12436 else
12437 {
12438 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12439 (DISCPUMODE)pDis->uAddrMode, cbValue);
12440 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12441 }
12442 }
12443 else
12444 {
12445 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12446 pMixedCtx->rip));
12447 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12448 }
12449#endif
12450 }
12451 else
12452 {
12453 /*
12454 * IN/OUT - I/O instruction.
12455 */
12456 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12457 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12458 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12459 if (fIOWrite)
12460 {
12461 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12462 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12463 }
12464 else
12465 {
12466 uint32_t u32Result = 0;
12467 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12468 if (IOM_SUCCESS(rcStrict))
12469 {
12470 /* Save result of I/O IN instr. in AL/AX/EAX. */
12471 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12472 }
12473 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12474 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12475 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12476 }
12477 }
12478
12479 if (IOM_SUCCESS(rcStrict))
12480 {
12481 if (!fUpdateRipAlready)
12482 {
12483 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12484 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12485 }
12486
12487 /*
12488 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12489 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12490 */
12491 if (fIOString)
12492 {
12493 /** @todo Single-step for INS/OUTS with REP prefix? */
12494 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12495 }
12496 else if ( !fDbgStepping
12497 && fGstStepping)
12498 {
12499 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12500 }
12501
12502 /*
12503 * If any I/O breakpoints are armed, we need to check if one triggered
12504 * and take appropriate action.
12505 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12506 */
12507 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12508 AssertRCReturn(rc2, rc2);
12509
12510 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12511 * execution engines about whether hyper BPs and such are pending. */
12512 uint32_t const uDr7 = pMixedCtx->dr[7];
12513 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12514 && X86_DR7_ANY_RW_IO(uDr7)
12515 && (pMixedCtx->cr4 & X86_CR4_DE))
12516 || DBGFBpIsHwIoArmed(pVM)))
12517 {
12518 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12519
12520 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12521 VMMRZCallRing3Disable(pVCpu);
12522 HM_DISABLE_PREEMPT();
12523
12524 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12525
12526 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12527 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12528 {
12529 /* Raise #DB. */
12530 if (fIsGuestDbgActive)
12531 ASMSetDR6(pMixedCtx->dr[6]);
12532 if (pMixedCtx->dr[7] != uDr7)
12533 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12534
12535 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12536 }
12537 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12538 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12539 else if ( rcStrict2 != VINF_SUCCESS
12540 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12541 rcStrict = rcStrict2;
12542 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12543
12544 HM_RESTORE_PREEMPT();
12545 VMMRZCallRing3Enable(pVCpu);
12546 }
12547 }
12548
12549#ifdef VBOX_STRICT
12550 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12551 Assert(!fIOWrite);
12552 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12553 Assert(fIOWrite);
12554 else
12555 {
12556#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12557 * statuses, that the VMM device and some others may return. See
12558 * IOM_SUCCESS() for guidance. */
12559 AssertMsg( RT_FAILURE(rcStrict)
12560 || rcStrict == VINF_SUCCESS
12561 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12562 || rcStrict == VINF_EM_DBG_BREAKPOINT
12563 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12564 || rcStrict == VINF_EM_RAW_TO_R3
12565 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12566#endif
12567 }
12568#endif
12569
12570 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12571 return rcStrict;
12572}
12573
12574
12575/**
12576 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12577 * VM-exit.
12578 */
12579HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12580{
12581 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12582
12583 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12584 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12585 AssertRCReturn(rc, rc);
12586 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12587 {
12588 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12589 AssertRCReturn(rc, rc);
12590 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12591 {
12592 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12593
12594 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12595 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12596
12597 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
12598 Assert(!pVCpu->hm.s.Event.fPending);
12599 pVCpu->hm.s.Event.fPending = true;
12600 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
12601 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12602 AssertRCReturn(rc, rc);
12603 if (fErrorCodeValid)
12604 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12605 else
12606 pVCpu->hm.s.Event.u32ErrCode = 0;
12607 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12608 && uVector == X86_XCPT_PF)
12609 {
12610 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
12611 }
12612
12613 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12614 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12615 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12616 }
12617 }
12618
12619 /* Fall back to the interpreter to emulate the task-switch. */
12620 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12621 return VERR_EM_INTERPRETER;
12622}
12623
12624
12625/**
12626 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12627 */
12628HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12629{
12630 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12631 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12632 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12633 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12634 AssertRCReturn(rc, rc);
12635 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12636 return VINF_EM_DBG_STEPPED;
12637}
12638
12639
12640/**
12641 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12642 */
12643HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12644{
12645 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12646
12647 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12648
12649 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12650 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12651 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12652 {
12653 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12654 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12655 {
12656 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12657 return VERR_EM_INTERPRETER;
12658 }
12659 }
12660 else
12661 {
12662 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12663 rcStrict1 = VINF_SUCCESS;
12664 return rcStrict1;
12665 }
12666
12667#if 0
12668 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
12669 * just sync the whole thing. */
12670 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12671#else
12672 /* Aggressive state sync. for now. */
12673 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12674 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12675 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12676#endif
12677 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12678 AssertRCReturn(rc, rc);
12679
12680 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12681 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12682 VBOXSTRICTRC rcStrict2;
12683 switch (uAccessType)
12684 {
12685 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12686 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12687 {
12688 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12689 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != 0x80,
12690 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12691
12692 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
12693 GCPhys &= PAGE_BASE_GC_MASK;
12694 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12695 PVM pVM = pVCpu->CTX_SUFF(pVM);
12696 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12697 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12698
12699 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12700 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12701 CPUMCTX2CORE(pMixedCtx), GCPhys);
12702 Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
12703 if ( rcStrict2 == VINF_SUCCESS
12704 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12705 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12706 {
12707 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12708 | HM_CHANGED_GUEST_RSP
12709 | HM_CHANGED_GUEST_RFLAGS
12710 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12711 rcStrict2 = VINF_SUCCESS;
12712 }
12713 break;
12714 }
12715
12716 default:
12717 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
12718 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12719 break;
12720 }
12721
12722 if (rcStrict2 != VINF_SUCCESS)
12723 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12724 return rcStrict2;
12725}
12726
12727
12728/**
12729 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12730 * VM-exit.
12731 */
12732HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12733{
12734 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12735
12736 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12737 if (pVmxTransient->fWasGuestDebugStateActive)
12738 {
12739 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12740 HMVMX_RETURN_UNEXPECTED_EXIT();
12741 }
12742
12743 if ( !pVCpu->hm.s.fSingleInstruction
12744 && !pVmxTransient->fWasHyperDebugStateActive)
12745 {
12746 Assert(!DBGFIsStepping(pVCpu));
12747 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12748
12749 /* Don't intercept MOV DRx any more. */
12750 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12751 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12752 AssertRCReturn(rc, rc);
12753
12754 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12755 VMMRZCallRing3Disable(pVCpu);
12756 HM_DISABLE_PREEMPT();
12757
12758 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12759 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12760 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12761
12762 HM_RESTORE_PREEMPT();
12763 VMMRZCallRing3Enable(pVCpu);
12764
12765#ifdef VBOX_WITH_STATISTICS
12766 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12767 AssertRCReturn(rc, rc);
12768 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12769 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12770 else
12771 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12772#endif
12773 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12774 return VINF_SUCCESS;
12775 }
12776
12777 /*
12778 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12779 * Update the segment registers and DR7 from the CPU.
12780 */
12781 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12782 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12783 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12784 AssertRCReturn(rc, rc);
12785 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12786
12787 PVM pVM = pVCpu->CTX_SUFF(pVM);
12788 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12789 {
12790 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12791 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
12792 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
12793 if (RT_SUCCESS(rc))
12794 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12795 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12796 }
12797 else
12798 {
12799 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12800 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
12801 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
12802 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12803 }
12804
12805 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12806 if (RT_SUCCESS(rc))
12807 {
12808 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12809 AssertRCReturn(rc2, rc2);
12810 return VINF_SUCCESS;
12811 }
12812 return rc;
12813}
12814
12815
12816/**
12817 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12818 * Conditional VM-exit.
12819 */
12820HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12821{
12822 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12823 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12824
12825 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12826 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12827 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12828 {
12829 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12830 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12831 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12832 {
12833 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12834 return VERR_EM_INTERPRETER;
12835 }
12836 }
12837 else
12838 {
12839 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12840 rcStrict1 = VINF_SUCCESS;
12841 return rcStrict1;
12842 }
12843
12844 RTGCPHYS GCPhys = 0;
12845 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12846
12847#if 0
12848 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12849#else
12850 /* Aggressive state sync. for now. */
12851 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12852 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12853 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12854#endif
12855 AssertRCReturn(rc, rc);
12856
12857 /*
12858 * If we succeed, resume guest execution.
12859 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12860 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12861 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12862 * weird case. See @bugref{6043}.
12863 */
12864 PVM pVM = pVCpu->CTX_SUFF(pVM);
12865 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12866 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
12867 if ( rcStrict2 == VINF_SUCCESS
12868 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12869 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12870 {
12871 /* Successfully handled MMIO operation. */
12872 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12873 | HM_CHANGED_GUEST_RSP
12874 | HM_CHANGED_GUEST_RFLAGS
12875 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12876 return VINF_SUCCESS;
12877 }
12878 return rcStrict2;
12879}
12880
12881
12882/**
12883 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12884 * VM-exit.
12885 */
12886HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12887{
12888 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12889 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12890
12891 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12892 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12893 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12894 {
12895 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12896 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12897 Log4(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12898 }
12899 else
12900 {
12901 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12902 rcStrict1 = VINF_SUCCESS;
12903 return rcStrict1;
12904 }
12905
12906 RTGCPHYS GCPhys = 0;
12907 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12908 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12909#if 0
12910 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12911#else
12912 /* Aggressive state sync. for now. */
12913 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12914 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12915 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12916#endif
12917 AssertRCReturn(rc, rc);
12918
12919 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12920 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12921
12922 RTGCUINT uErrorCode = 0;
12923 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
12924 uErrorCode |= X86_TRAP_PF_ID;
12925 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
12926 uErrorCode |= X86_TRAP_PF_RW;
12927 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
12928 uErrorCode |= X86_TRAP_PF_P;
12929
12930 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12931
12932 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12933 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12934
12935 /* Handle the pagefault trap for the nested shadow table. */
12936 PVM pVM = pVCpu->CTX_SUFF(pVM);
12937 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12938 TRPMResetTrap(pVCpu);
12939
12940 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12941 if ( rcStrict2 == VINF_SUCCESS
12942 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12943 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12944 {
12945 /* Successfully synced our nested page tables. */
12946 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12947 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12948 | HM_CHANGED_GUEST_RSP
12949 | HM_CHANGED_GUEST_RFLAGS);
12950 return VINF_SUCCESS;
12951 }
12952
12953 Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12954 return rcStrict2;
12955}
12956
12957/** @} */
12958
12959/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12960/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
12961/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12962
12963/** @name VM-exit exception handlers.
12964 * @{
12965 */
12966
12967/**
12968 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12969 */
12970static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12971{
12972 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12973 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12974
12975 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12976 AssertRCReturn(rc, rc);
12977
12978 if (!(pMixedCtx->cr0 & X86_CR0_NE))
12979 {
12980 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12981 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12982
12983 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12984 * provides VM-exit instruction length. If this causes problem later,
12985 * disassemble the instruction like it's done on AMD-V. */
12986 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12987 AssertRCReturn(rc2, rc2);
12988 return rc;
12989 }
12990
12991 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12992 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12993 return rc;
12994}
12995
12996
12997/**
12998 * VM-exit exception handler for \#BP (Breakpoint exception).
12999 */
13000static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13001{
13002 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13003 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13004
13005 /** @todo Try optimize this by not saving the entire guest state unless
13006 * really needed. */
13007 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13008 AssertRCReturn(rc, rc);
13009
13010 PVM pVM = pVCpu->CTX_SUFF(pVM);
13011 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
13012 if (rc == VINF_EM_RAW_GUEST_TRAP)
13013 {
13014 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13015 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13016 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13017 AssertRCReturn(rc, rc);
13018
13019 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13020 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13021 }
13022
13023 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
13024 return rc;
13025}
13026
13027
13028/**
13029 * VM-exit exception handler for \#AC (alignment check exception).
13030 */
13031static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13032{
13033 RT_NOREF_PV(pMixedCtx);
13034 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13035
13036 /*
13037 * Re-inject it. We'll detect any nesting before getting here.
13038 */
13039 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13040 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13041 AssertRCReturn(rc, rc);
13042 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13043
13044 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13045 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13046 return VINF_SUCCESS;
13047}
13048
13049
13050/**
13051 * VM-exit exception handler for \#DB (Debug exception).
13052 */
13053static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13054{
13055 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13056 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13057 Log6(("XcptDB\n"));
13058
13059 /*
13060 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13061 * for processing.
13062 */
13063 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13064 AssertRCReturn(rc, rc);
13065
13066 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13067 uint64_t uDR6 = X86_DR6_INIT_VAL;
13068 uDR6 |= ( pVmxTransient->uExitQualification
13069 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
13070
13071 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13072 if (rc == VINF_EM_RAW_GUEST_TRAP)
13073 {
13074 /*
13075 * The exception was for the guest. Update DR6, DR7.GD and
13076 * IA32_DEBUGCTL.LBR before forwarding it.
13077 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13078 */
13079 VMMRZCallRing3Disable(pVCpu);
13080 HM_DISABLE_PREEMPT();
13081
13082 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
13083 pMixedCtx->dr[6] |= uDR6;
13084 if (CPUMIsGuestDebugStateActive(pVCpu))
13085 ASMSetDR6(pMixedCtx->dr[6]);
13086
13087 HM_RESTORE_PREEMPT();
13088 VMMRZCallRing3Enable(pVCpu);
13089
13090 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13091 AssertRCReturn(rc, rc);
13092
13093 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13094 pMixedCtx->dr[7] &= ~X86_DR7_GD;
13095
13096 /* Paranoia. */
13097 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13098 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
13099
13100 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
13101 AssertRCReturn(rc, rc);
13102
13103 /*
13104 * Raise #DB in the guest.
13105 *
13106 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
13107 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
13108 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
13109 *
13110 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
13111 */
13112 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13113 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13114 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13115 AssertRCReturn(rc, rc);
13116 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13117 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13118 return VINF_SUCCESS;
13119 }
13120
13121 /*
13122 * Not a guest trap, must be a hypervisor related debug event then.
13123 * Update DR6 in case someone is interested in it.
13124 */
13125 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13126 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13127 CPUMSetHyperDR6(pVCpu, uDR6);
13128
13129 return rc;
13130}
13131
13132
13133/**
13134 * VM-exit exception handler for \#NM (Device-not-available exception: floating
13135 * point exception).
13136 */
13137static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13138{
13139 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13140
13141 /* We require CR0 and EFER. EFER is always up-to-date. */
13142 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13143 AssertRCReturn(rc, rc);
13144
13145 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
13146 VMMRZCallRing3Disable(pVCpu);
13147 HM_DISABLE_PREEMPT();
13148
13149 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
13150 if (pVmxTransient->fWasGuestFPUStateActive)
13151 {
13152 rc = VINF_EM_RAW_GUEST_TRAP;
13153 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
13154 }
13155 else
13156 {
13157#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13158 Assert(!pVmxTransient->fWasGuestFPUStateActive || pVCpu->hm.s.fUsingDebugLoop);
13159#endif
13160 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu);
13161 Assert( rc == VINF_EM_RAW_GUEST_TRAP
13162 || ((rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
13163 if (rc == VINF_CPUM_HOST_CR0_MODIFIED)
13164 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
13165 }
13166
13167 HM_RESTORE_PREEMPT();
13168 VMMRZCallRing3Enable(pVCpu);
13169
13170 if (rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED)
13171 {
13172 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
13173 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13174 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
13175 pVCpu->hm.s.fPreloadGuestFpu = true;
13176 }
13177 else
13178 {
13179 /* Forward #NM to the guest. */
13180 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
13181 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13182 AssertRCReturn(rc, rc);
13183 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13184 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
13185 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13186 }
13187
13188 return VINF_SUCCESS;
13189}
13190
13191
13192/**
13193 * VM-exit exception handler for \#GP (General-protection exception).
13194 *
13195 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13196 */
13197static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13198{
13199 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13200 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13201
13202 int rc;
13203 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13204 { /* likely */ }
13205 else
13206 {
13207#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13208 Assert(pVCpu->hm.s.fUsingDebugLoop);
13209#endif
13210 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13211 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13212 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13213 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13214 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13215 AssertRCReturn(rc, rc);
13216 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13217 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13218 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13219 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13220 return rc;
13221 }
13222
13223 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13224 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13225
13226 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13227 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13228 AssertRCReturn(rc, rc);
13229
13230 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13231 uint32_t cbOp = 0;
13232 PVM pVM = pVCpu->CTX_SUFF(pVM);
13233 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13234 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13235 if (RT_SUCCESS(rc))
13236 {
13237 rc = VINF_SUCCESS;
13238 Assert(cbOp == pDis->cbInstr);
13239 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13240 switch (pDis->pCurInstr->uOpcode)
13241 {
13242 case OP_CLI:
13243 {
13244 pMixedCtx->eflags.Bits.u1IF = 0;
13245 pMixedCtx->eflags.Bits.u1RF = 0;
13246 pMixedCtx->rip += pDis->cbInstr;
13247 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13248 if ( !fDbgStepping
13249 && pMixedCtx->eflags.Bits.u1TF)
13250 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13251 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13252 break;
13253 }
13254
13255 case OP_STI:
13256 {
13257 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13258 pMixedCtx->eflags.Bits.u1IF = 1;
13259 pMixedCtx->eflags.Bits.u1RF = 0;
13260 pMixedCtx->rip += pDis->cbInstr;
13261 if (!fOldIF)
13262 {
13263 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13264 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13265 }
13266 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13267 if ( !fDbgStepping
13268 && pMixedCtx->eflags.Bits.u1TF)
13269 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13270 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13271 break;
13272 }
13273
13274 case OP_HLT:
13275 {
13276 rc = VINF_EM_HALT;
13277 pMixedCtx->rip += pDis->cbInstr;
13278 pMixedCtx->eflags.Bits.u1RF = 0;
13279 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13280 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13281 break;
13282 }
13283
13284 case OP_POPF:
13285 {
13286 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13287 uint32_t cbParm;
13288 uint32_t uMask;
13289 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13290 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13291 {
13292 cbParm = 4;
13293 uMask = 0xffffffff;
13294 }
13295 else
13296 {
13297 cbParm = 2;
13298 uMask = 0xffff;
13299 }
13300
13301 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13302 RTGCPTR GCPtrStack = 0;
13303 X86EFLAGS Eflags;
13304 Eflags.u32 = 0;
13305 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13306 &GCPtrStack);
13307 if (RT_SUCCESS(rc))
13308 {
13309 Assert(sizeof(Eflags.u32) >= cbParm);
13310 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13311 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13312 }
13313 if (RT_FAILURE(rc))
13314 {
13315 rc = VERR_EM_INTERPRETER;
13316 break;
13317 }
13318 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13319 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13320 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13321 pMixedCtx->esp += cbParm;
13322 pMixedCtx->esp &= uMask;
13323 pMixedCtx->rip += pDis->cbInstr;
13324 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13325 | HM_CHANGED_GUEST_RSP
13326 | HM_CHANGED_GUEST_RFLAGS);
13327 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13328 POPF restores EFLAGS.TF. */
13329 if ( !fDbgStepping
13330 && fGstStepping)
13331 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13332 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13333 break;
13334 }
13335
13336 case OP_PUSHF:
13337 {
13338 uint32_t cbParm;
13339 uint32_t uMask;
13340 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13341 {
13342 cbParm = 4;
13343 uMask = 0xffffffff;
13344 }
13345 else
13346 {
13347 cbParm = 2;
13348 uMask = 0xffff;
13349 }
13350
13351 /* Get the stack pointer & push the contents of eflags onto the stack. */
13352 RTGCPTR GCPtrStack = 0;
13353 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13354 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13355 if (RT_FAILURE(rc))
13356 {
13357 rc = VERR_EM_INTERPRETER;
13358 break;
13359 }
13360 X86EFLAGS Eflags = pMixedCtx->eflags;
13361 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13362 Eflags.Bits.u1RF = 0;
13363 Eflags.Bits.u1VM = 0;
13364
13365 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13366 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13367 {
13368 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13369 rc = VERR_EM_INTERPRETER;
13370 break;
13371 }
13372 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13373 pMixedCtx->esp -= cbParm;
13374 pMixedCtx->esp &= uMask;
13375 pMixedCtx->rip += pDis->cbInstr;
13376 pMixedCtx->eflags.Bits.u1RF = 0;
13377 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13378 | HM_CHANGED_GUEST_RSP
13379 | HM_CHANGED_GUEST_RFLAGS);
13380 if ( !fDbgStepping
13381 && pMixedCtx->eflags.Bits.u1TF)
13382 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13383 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13384 break;
13385 }
13386
13387 case OP_IRET:
13388 {
13389 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13390 * instruction reference. */
13391 RTGCPTR GCPtrStack = 0;
13392 uint32_t uMask = 0xffff;
13393 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13394 uint16_t aIretFrame[3];
13395 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13396 {
13397 rc = VERR_EM_INTERPRETER;
13398 break;
13399 }
13400 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13401 &GCPtrStack);
13402 if (RT_SUCCESS(rc))
13403 {
13404 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13405 PGMACCESSORIGIN_HM));
13406 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13407 }
13408 if (RT_FAILURE(rc))
13409 {
13410 rc = VERR_EM_INTERPRETER;
13411 break;
13412 }
13413 pMixedCtx->eip = 0;
13414 pMixedCtx->ip = aIretFrame[0];
13415 pMixedCtx->cs.Sel = aIretFrame[1];
13416 pMixedCtx->cs.ValidSel = aIretFrame[1];
13417 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13418 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13419 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13420 pMixedCtx->sp += sizeof(aIretFrame);
13421 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13422 | HM_CHANGED_GUEST_SEGMENT_REGS
13423 | HM_CHANGED_GUEST_RSP
13424 | HM_CHANGED_GUEST_RFLAGS);
13425 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13426 if ( !fDbgStepping
13427 && fGstStepping)
13428 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13429 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13430 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13431 break;
13432 }
13433
13434 case OP_INT:
13435 {
13436 uint16_t uVector = pDis->Param1.uValue & 0xff;
13437 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13438 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13439 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13440 break;
13441 }
13442
13443 case OP_INTO:
13444 {
13445 if (pMixedCtx->eflags.Bits.u1OF)
13446 {
13447 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13448 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13449 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13450 }
13451 else
13452 {
13453 pMixedCtx->eflags.Bits.u1RF = 0;
13454 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13455 }
13456 break;
13457 }
13458
13459 default:
13460 {
13461 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13462 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13463 EMCODETYPE_SUPERVISOR);
13464 rc = VBOXSTRICTRC_VAL(rc2);
13465 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13466 /** @todo We have to set pending-debug exceptions here when the guest is
13467 * single-stepping depending on the instruction that was interpreted. */
13468 Log4(("#GP rc=%Rrc\n", rc));
13469 break;
13470 }
13471 }
13472 }
13473 else
13474 rc = VERR_EM_INTERPRETER;
13475
13476 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13477 ("#GP Unexpected rc=%Rrc\n", rc));
13478 return rc;
13479}
13480
13481
13482/**
13483 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13484 * the exception reported in the VMX transient structure back into the VM.
13485 *
13486 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13487 * up-to-date.
13488 */
13489static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13490{
13491 RT_NOREF_PV(pMixedCtx);
13492 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13493#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13494 Assert(pVCpu->hm.s.fUsingDebugLoop);
13495#endif
13496
13497 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13498 hmR0VmxCheckExitDueToEventDelivery(). */
13499 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13500 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13501 AssertRCReturn(rc, rc);
13502 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13503
13504#ifdef DEBUG_ramshankar
13505 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13506 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13507 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13508#endif
13509
13510 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13511 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13512 return VINF_SUCCESS;
13513}
13514
13515
13516/**
13517 * VM-exit exception handler for \#PF (Page-fault exception).
13518 */
13519static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13520{
13521 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13522 PVM pVM = pVCpu->CTX_SUFF(pVM);
13523 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13524 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13525 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13526 AssertRCReturn(rc, rc);
13527
13528 if (!pVM->hm.s.fNestedPaging)
13529 { /* likely */ }
13530 else
13531 {
13532#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13533 Assert(pVCpu->hm.s.fUsingDebugLoop);
13534#endif
13535 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13536 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13537 {
13538 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13539 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13540 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13541 }
13542 else
13543 {
13544 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13545 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13546 Log4(("Pending #DF due to vectoring #PF. NP\n"));
13547 }
13548 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13549 return rc;
13550 }
13551
13552 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13553 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13554 if (pVmxTransient->fVectoringPF)
13555 {
13556 Assert(pVCpu->hm.s.Event.fPending);
13557 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13558 }
13559
13560 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13561 AssertRCReturn(rc, rc);
13562
13563 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13564 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13565
13566 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13567 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13568 (RTGCPTR)pVmxTransient->uExitQualification);
13569
13570 Log4(("#PF: rc=%Rrc\n", rc));
13571 if (rc == VINF_SUCCESS)
13572 {
13573#if 0
13574 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13575 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
13576 * memory? We don't update the whole state here... */
13577 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13578 | HM_CHANGED_GUEST_RSP
13579 | HM_CHANGED_GUEST_RFLAGS
13580 | HM_CHANGED_VMX_GUEST_APIC_STATE);
13581#else
13582 /*
13583 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13584 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13585 */
13586 /** @todo take advantage of CPUM changed flags instead of brute forcing. */
13587 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13588#endif
13589 TRPMResetTrap(pVCpu);
13590 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13591 return rc;
13592 }
13593
13594 if (rc == VINF_EM_RAW_GUEST_TRAP)
13595 {
13596 if (!pVmxTransient->fVectoringDoublePF)
13597 {
13598 /* It's a guest page fault and needs to be reflected to the guest. */
13599 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13600 TRPMResetTrap(pVCpu);
13601 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13602 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13603 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13604 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13605 }
13606 else
13607 {
13608 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13609 TRPMResetTrap(pVCpu);
13610 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13611 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13612 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
13613 }
13614
13615 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13616 return VINF_SUCCESS;
13617 }
13618
13619 TRPMResetTrap(pVCpu);
13620 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13621 return rc;
13622}
13623
13624/** @} */
13625
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette