VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 62020

Last change on this file since 62020 was 62020, checked in by vboxsync, 8 years ago

VMM/HMVMXR0: Verify CPU feature before reading VMCS fields in debug-mode diagnostic code.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 580.3 KB
Line 
1/* $Id: HMVMXR0.cpp 62020 2016-07-05 08:18:23Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2015 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#include <iprt/x86.h>
24#include <iprt/asm-amd64-x86.h>
25#include <iprt/thread.h>
26
27#include <VBox/vmm/pdmapi.h>
28#include <VBox/vmm/dbgf.h>
29#include <VBox/vmm/iem.h>
30#include <VBox/vmm/iom.h>
31#include <VBox/vmm/selm.h>
32#include <VBox/vmm/tm.h>
33#include <VBox/vmm/gim.h>
34#ifdef VBOX_WITH_REM
35# include <VBox/vmm/rem.h>
36#endif
37#ifdef VBOX_WITH_NEW_APIC
38# include <VBox/vmm/apic.h>
39#endif
40#include "HMInternal.h"
41#include <VBox/vmm/vm.h>
42#include "HMVMXR0.h"
43#include "dtrace/VBoxVMM.h"
44
45#ifdef DEBUG_ramshankar
46# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
47# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
49# define HMVMX_ALWAYS_CHECK_GUEST_STATE
50# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
51# define HMVMX_ALWAYS_TRAP_PF
52# define HMVMX_ALWAYS_SWAP_FPU_STATE
53# define HMVMX_ALWAYS_FLUSH_TLB
54# define HMVMX_ALWAYS_SWAP_EFER
55#endif
56
57
58/*********************************************************************************************************************************
59* Defined Constants And Macros *
60*********************************************************************************************************************************/
61/** Use the function table. */
62#define HMVMX_USE_FUNCTION_TABLE
63
64/** Determine which tagged-TLB flush handler to use. */
65#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70/** @name Updated-guest-state flags.
71 * @{ */
72#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
73#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
74#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
75#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
76#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
77#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
78#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
79#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
80#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
81#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
82#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
83#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
84#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
85#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
86#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
87#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
88#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
89#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
90#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
91#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
92#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
93 | HMVMX_UPDATED_GUEST_RSP \
94 | HMVMX_UPDATED_GUEST_RFLAGS \
95 | HMVMX_UPDATED_GUEST_CR0 \
96 | HMVMX_UPDATED_GUEST_CR3 \
97 | HMVMX_UPDATED_GUEST_CR4 \
98 | HMVMX_UPDATED_GUEST_GDTR \
99 | HMVMX_UPDATED_GUEST_IDTR \
100 | HMVMX_UPDATED_GUEST_LDTR \
101 | HMVMX_UPDATED_GUEST_TR \
102 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
103 | HMVMX_UPDATED_GUEST_DEBUG \
104 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
105 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
106 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
107 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
108 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
109 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
110 | HMVMX_UPDATED_GUEST_INTR_STATE \
111 | HMVMX_UPDATED_GUEST_APIC_STATE)
112/** @} */
113
114/** @name
115 * Flags to skip redundant reads of some common VMCS fields that are not part of
116 * the guest-CPU state but are in the transient structure.
117 */
118#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
119#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
120#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
123#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
124#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
125/** @} */
126
127/** @name
128 * States of the VMCS.
129 *
130 * This does not reflect all possible VMCS states but currently only those
131 * needed for maintaining the VMCS consistently even when thread-context hooks
132 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
133 */
134#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
135#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
136#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
137/** @} */
138
139/**
140 * Exception bitmap mask for real-mode guests (real-on-v86).
141 *
142 * We need to intercept all exceptions manually except:
143 * - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
144 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
145 * due to bugs in Intel CPUs.
146 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
147 * support.
148 */
149#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
150 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
151 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
152 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
153 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
154 /* RT_BIT(X86_XCPT_MF) always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
155 | RT_BIT(X86_XCPT_XF))
156
157/**
158 * Exception bitmap mask for all contributory exceptions.
159 *
160 * Page fault is deliberately excluded here as it's conditional as to whether
161 * it's contributory or benign. Page faults are handled separately.
162 */
163#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
164 | RT_BIT(X86_XCPT_DE))
165
166/** Maximum VM-instruction error number. */
167#define HMVMX_INSTR_ERROR_MAX 28
168
169/** Profiling macro. */
170#ifdef HM_PROFILE_EXIT_DISPATCH
171# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
172# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
173#else
174# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
175# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
176#endif
177
178/** Assert that preemption is disabled or covered by thread-context hooks. */
179#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
180 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
181
182/** Assert that we haven't migrated CPUs when thread-context hooks are not
183 * used. */
184#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
185 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
186 ("Illegal migration! Entered on CPU %u Current %u\n", \
187 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
188
189/** Helper macro for VM-exit handlers called unexpectedly. */
190#define HMVMX_RETURN_UNEXPECTED_EXIT() \
191 do { \
192 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
193 return VERR_VMX_UNEXPECTED_EXIT; \
194 } while (0)
195
196
197/*********************************************************************************************************************************
198* Structures and Typedefs *
199*********************************************************************************************************************************/
200/**
201 * VMX transient state.
202 *
203 * A state structure for holding miscellaneous information across
204 * VMX non-root operation and restored after the transition.
205 */
206typedef struct VMXTRANSIENT
207{
208 /** The host's rflags/eflags. */
209 RTCCUINTREG fEFlags;
210#if HC_ARCH_BITS == 32
211 uint32_t u32Alignment0;
212#endif
213 /** The guest's TPR value used for TPR shadowing. */
214 uint8_t u8GuestTpr;
215 /** Alignment. */
216 uint8_t abAlignment0[7];
217
218 /** The basic VM-exit reason. */
219 uint16_t uExitReason;
220 /** Alignment. */
221 uint16_t u16Alignment0;
222 /** The VM-exit interruption error code. */
223 uint32_t uExitIntErrorCode;
224 /** The VM-exit exit code qualification. */
225 uint64_t uExitQualification;
226
227 /** The VM-exit interruption-information field. */
228 uint32_t uExitIntInfo;
229 /** The VM-exit instruction-length field. */
230 uint32_t cbInstr;
231 /** The VM-exit instruction-information field. */
232 union
233 {
234 /** Plain unsigned int representation. */
235 uint32_t u;
236 /** INS and OUTS information. */
237 struct
238 {
239 uint32_t u7Reserved0 : 7;
240 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
241 uint32_t u3AddrSize : 3;
242 uint32_t u5Reserved1 : 5;
243 /** The segment register (X86_SREG_XXX). */
244 uint32_t iSegReg : 3;
245 uint32_t uReserved2 : 14;
246 } StrIo;
247 } ExitInstrInfo;
248 /** Whether the VM-entry failed or not. */
249 bool fVMEntryFailed;
250 /** Alignment. */
251 uint8_t abAlignment1[3];
252
253 /** The VM-entry interruption-information field. */
254 uint32_t uEntryIntInfo;
255 /** The VM-entry exception error code field. */
256 uint32_t uEntryXcptErrorCode;
257 /** The VM-entry instruction length field. */
258 uint32_t cbEntryInstr;
259
260 /** IDT-vectoring information field. */
261 uint32_t uIdtVectoringInfo;
262 /** IDT-vectoring error code. */
263 uint32_t uIdtVectoringErrorCode;
264
265 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
266 uint32_t fVmcsFieldsRead;
267
268 /** Whether the guest FPU was active at the time of VM-exit. */
269 bool fWasGuestFPUStateActive;
270 /** Whether the guest debug state was active at the time of VM-exit. */
271 bool fWasGuestDebugStateActive;
272 /** Whether the hyper debug state was active at the time of VM-exit. */
273 bool fWasHyperDebugStateActive;
274 /** Whether TSC-offsetting should be setup before VM-entry. */
275 bool fUpdateTscOffsettingAndPreemptTimer;
276 /** Whether the VM-exit was caused by a page-fault during delivery of a
277 * contributory exception or a page-fault. */
278 bool fVectoringDoublePF;
279 /** Whether the VM-exit was caused by a page-fault during delivery of an
280 * external interrupt or NMI. */
281 bool fVectoringPF;
282} VMXTRANSIENT;
283AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
284AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
285AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
286AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
287AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
288/** Pointer to VMX transient state. */
289typedef VMXTRANSIENT *PVMXTRANSIENT;
290
291
292/**
293 * MSR-bitmap read permissions.
294 */
295typedef enum VMXMSREXITREAD
296{
297 /** Reading this MSR causes a VM-exit. */
298 VMXMSREXIT_INTERCEPT_READ = 0xb,
299 /** Reading this MSR does not cause a VM-exit. */
300 VMXMSREXIT_PASSTHRU_READ
301} VMXMSREXITREAD;
302/** Pointer to MSR-bitmap read permissions. */
303typedef VMXMSREXITREAD* PVMXMSREXITREAD;
304
305/**
306 * MSR-bitmap write permissions.
307 */
308typedef enum VMXMSREXITWRITE
309{
310 /** Writing to this MSR causes a VM-exit. */
311 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
312 /** Writing to this MSR does not cause a VM-exit. */
313 VMXMSREXIT_PASSTHRU_WRITE
314} VMXMSREXITWRITE;
315/** Pointer to MSR-bitmap write permissions. */
316typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
317
318
319/**
320 * VMX VM-exit handler.
321 *
322 * @returns Strict VBox status code (i.e. informational status codes too).
323 * @param pVCpu The cross context virtual CPU structure.
324 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
325 * out-of-sync. Make sure to update the required
326 * fields before using them.
327 * @param pVmxTransient Pointer to the VMX-transient structure.
328 */
329#ifndef HMVMX_USE_FUNCTION_TABLE
330typedef DECLINLINE(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
331#else
332typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
333/** Pointer to VM-exit handler. */
334typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
335#endif
336
337/**
338 * VMX VM-exit handler, non-strict status code.
339 *
340 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
341 *
342 * @returns VBox status code, no informational status code returned.
343 * @param pVCpu The cross context virtual CPU structure.
344 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
345 * out-of-sync. Make sure to update the required
346 * fields before using them.
347 * @param pVmxTransient Pointer to the VMX-transient structure.
348 *
349 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
350 * use of that status code will be replaced with VINF_EM_SOMETHING
351 * later when switching over to IEM.
352 */
353#ifndef HMVMX_USE_FUNCTION_TABLE
354typedef DECLINLINE(int) FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
355#else
356typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
357#endif
358
359
360/*********************************************************************************************************************************
361* Internal Functions *
362*********************************************************************************************************************************/
363static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
364static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
365static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
366static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
367 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
368 bool fStepping, uint32_t *puIntState);
369#if HC_ARCH_BITS == 32
370static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
371#endif
372#ifndef HMVMX_USE_FUNCTION_TABLE
373DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
374# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
375# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
376#else
377# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
378# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
379#endif
380
381
382/** @name VM-exit handlers.
383 * @{
384 */
385static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
386static FNVMXEXITHANDLER hmR0VmxExitExtInt;
387static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
388static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
389static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
390static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
391static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
392static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
393static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
394static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
395static FNVMXEXITHANDLER hmR0VmxExitCpuid;
396static FNVMXEXITHANDLER hmR0VmxExitGetsec;
397static FNVMXEXITHANDLER hmR0VmxExitHlt;
398static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
399static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
400static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
401static FNVMXEXITHANDLER hmR0VmxExitVmcall;
402static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
403static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
404static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
405static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
406static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
407static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
408static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
409static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
410static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
411static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
412static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
413static FNVMXEXITHANDLER hmR0VmxExitMwait;
414static FNVMXEXITHANDLER hmR0VmxExitMtf;
415static FNVMXEXITHANDLER hmR0VmxExitMonitor;
416static FNVMXEXITHANDLER hmR0VmxExitPause;
417static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
418static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
419static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
420static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
421static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
422static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
423static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
424static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
425static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
426static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
427static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
428static FNVMXEXITHANDLER hmR0VmxExitRdrand;
429static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
430/** @} */
431
432static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
433static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
434static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
435static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
436static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
437static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
438static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
439static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
440static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
441
442
443/*********************************************************************************************************************************
444* Global Variables *
445*********************************************************************************************************************************/
446#ifdef HMVMX_USE_FUNCTION_TABLE
447
448/**
449 * VMX_EXIT dispatch table.
450 */
451static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
452{
453 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
454 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
455 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
456 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
457 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
458 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
459 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
460 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
461 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
462 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
463 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
464 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
465 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
466 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
467 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
468 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
469 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
470 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
471 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
472 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
473 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
474 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
475 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
476 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
477 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
478 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
479 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
480 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
481 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
482 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
483 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
484 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
485 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
486 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
487 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
488 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
489 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
490 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
491 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
492 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
493 /* 40 UNDEFINED */ hmR0VmxExitPause,
494 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
495 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
496 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
497 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
498 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
499 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
500 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
501 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
502 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
503 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
504 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
505 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
506 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
507 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
508 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
509 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
510 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
511 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
512 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
513 /* 60 VMX_EXIT_RESERVED_60 */ hmR0VmxExitErrUndefined,
514 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
515 /* 62 VMX_EXIT_RESERVED_62 */ hmR0VmxExitErrUndefined,
516 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
517 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
518};
519#endif /* HMVMX_USE_FUNCTION_TABLE */
520
521#ifdef VBOX_STRICT
522static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
523{
524 /* 0 */ "(Not Used)",
525 /* 1 */ "VMCALL executed in VMX root operation.",
526 /* 2 */ "VMCLEAR with invalid physical address.",
527 /* 3 */ "VMCLEAR with VMXON pointer.",
528 /* 4 */ "VMLAUNCH with non-clear VMCS.",
529 /* 5 */ "VMRESUME with non-launched VMCS.",
530 /* 6 */ "VMRESUME after VMXOFF",
531 /* 7 */ "VM-entry with invalid control fields.",
532 /* 8 */ "VM-entry with invalid host state fields.",
533 /* 9 */ "VMPTRLD with invalid physical address.",
534 /* 10 */ "VMPTRLD with VMXON pointer.",
535 /* 11 */ "VMPTRLD with incorrect revision identifier.",
536 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
537 /* 13 */ "VMWRITE to read-only VMCS component.",
538 /* 14 */ "(Not Used)",
539 /* 15 */ "VMXON executed in VMX root operation.",
540 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
541 /* 17 */ "VM-entry with non-launched executing VMCS.",
542 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
543 /* 19 */ "VMCALL with non-clear VMCS.",
544 /* 20 */ "VMCALL with invalid VM-exit control fields.",
545 /* 21 */ "(Not Used)",
546 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
547 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
548 /* 24 */ "VMCALL with invalid SMM-monitor features.",
549 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
550 /* 26 */ "VM-entry with events blocked by MOV SS.",
551 /* 27 */ "(Not Used)",
552 /* 28 */ "Invalid operand to INVEPT/INVVPID."
553};
554#endif /* VBOX_STRICT */
555
556
557
558/**
559 * Updates the VM's last error record.
560 *
561 * If there was a VMX instruction error, reads the error data from the VMCS and
562 * updates VCPU's last error record as well.
563 *
564 * @param pVM The cross context VM structure.
565 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
566 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
567 * VERR_VMX_INVALID_VMCS_FIELD.
568 * @param rc The error code.
569 */
570static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
571{
572 AssertPtr(pVM);
573 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
574 || rc == VERR_VMX_UNABLE_TO_START_VM)
575 {
576 AssertPtrReturnVoid(pVCpu);
577 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
578 }
579 pVM->hm.s.lLastError = rc;
580}
581
582
583/**
584 * Reads the VM-entry interruption-information field from the VMCS into the VMX
585 * transient structure.
586 *
587 * @returns VBox status code.
588 * @param pVmxTransient Pointer to the VMX transient structure.
589 *
590 * @remarks No-long-jump zone!!!
591 */
592DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
593{
594 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
595 AssertRCReturn(rc, rc);
596 return VINF_SUCCESS;
597}
598
599
600/**
601 * Reads the VM-entry exception error code field from the VMCS into
602 * the VMX transient structure.
603 *
604 * @returns VBox status code.
605 * @param pVmxTransient Pointer to the VMX transient structure.
606 *
607 * @remarks No-long-jump zone!!!
608 */
609DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
610{
611 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
612 AssertRCReturn(rc, rc);
613 return VINF_SUCCESS;
614}
615
616
617/**
618 * Reads the VM-entry exception error code field from the VMCS into
619 * the VMX transient structure.
620 *
621 * @returns VBox status code.
622 * @param pVmxTransient Pointer to the VMX transient structure.
623 *
624 * @remarks No-long-jump zone!!!
625 */
626DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
627{
628 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
629 AssertRCReturn(rc, rc);
630 return VINF_SUCCESS;
631}
632
633
634/**
635 * Reads the VM-exit interruption-information field from the VMCS into the VMX
636 * transient structure.
637 *
638 * @returns VBox status code.
639 * @param pVmxTransient Pointer to the VMX transient structure.
640 */
641DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
642{
643 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
644 {
645 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
646 AssertRCReturn(rc, rc);
647 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
648 }
649 return VINF_SUCCESS;
650}
651
652
653/**
654 * Reads the VM-exit interruption error code from the VMCS into the VMX
655 * transient structure.
656 *
657 * @returns VBox status code.
658 * @param pVmxTransient Pointer to the VMX transient structure.
659 */
660DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
661{
662 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
663 {
664 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
665 AssertRCReturn(rc, rc);
666 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
667 }
668 return VINF_SUCCESS;
669}
670
671
672/**
673 * Reads the VM-exit instruction length field from the VMCS into the VMX
674 * transient structure.
675 *
676 * @returns VBox status code.
677 * @param pVmxTransient Pointer to the VMX transient structure.
678 */
679DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
680{
681 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
682 {
683 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
684 AssertRCReturn(rc, rc);
685 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
686 }
687 return VINF_SUCCESS;
688}
689
690
691/**
692 * Reads the VM-exit instruction-information field from the VMCS into
693 * the VMX transient structure.
694 *
695 * @returns VBox status code.
696 * @param pVmxTransient Pointer to the VMX transient structure.
697 */
698DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
699{
700 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
701 {
702 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
703 AssertRCReturn(rc, rc);
704 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
705 }
706 return VINF_SUCCESS;
707}
708
709
710/**
711 * Reads the exit code qualification from the VMCS into the VMX transient
712 * structure.
713 *
714 * @returns VBox status code.
715 * @param pVCpu The cross context virtual CPU structure of the
716 * calling EMT. (Required for the VMCS cache case.)
717 * @param pVmxTransient Pointer to the VMX transient structure.
718 */
719DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
720{
721 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
722 {
723 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
724 AssertRCReturn(rc, rc);
725 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
726 }
727 return VINF_SUCCESS;
728}
729
730
731/**
732 * Reads the IDT-vectoring information field from the VMCS into the VMX
733 * transient structure.
734 *
735 * @returns VBox status code.
736 * @param pVmxTransient Pointer to the VMX transient structure.
737 *
738 * @remarks No-long-jump zone!!!
739 */
740DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
741{
742 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
743 {
744 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
745 AssertRCReturn(rc, rc);
746 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
747 }
748 return VINF_SUCCESS;
749}
750
751
752/**
753 * Reads the IDT-vectoring error code from the VMCS into the VMX
754 * transient structure.
755 *
756 * @returns VBox status code.
757 * @param pVmxTransient Pointer to the VMX transient structure.
758 */
759DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
760{
761 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
762 {
763 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
764 AssertRCReturn(rc, rc);
765 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
766 }
767 return VINF_SUCCESS;
768}
769
770
771/**
772 * Enters VMX root mode operation on the current CPU.
773 *
774 * @returns VBox status code.
775 * @param pVM The cross context VM structure. Can be
776 * NULL, after a resume.
777 * @param HCPhysCpuPage Physical address of the VMXON region.
778 * @param pvCpuPage Pointer to the VMXON region.
779 */
780static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
781{
782 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
783 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
784 Assert(pvCpuPage);
785 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
786
787 if (pVM)
788 {
789 /* Write the VMCS revision dword to the VMXON region. */
790 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
791 }
792
793 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
794 RTCCUINTREG fEFlags = ASMIntDisableFlags();
795
796 /* Enable the VMX bit in CR4 if necessary. */
797 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
798
799 /* Enter VMX root mode. */
800 int rc = VMXEnable(HCPhysCpuPage);
801 if (RT_FAILURE(rc))
802 {
803 if (!(uOldCr4 & X86_CR4_VMXE))
804 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
805
806 if (pVM)
807 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
808 }
809
810 /* Restore interrupts. */
811 ASMSetFlags(fEFlags);
812 return rc;
813}
814
815
816/**
817 * Exits VMX root mode operation on the current CPU.
818 *
819 * @returns VBox status code.
820 */
821static int hmR0VmxLeaveRootMode(void)
822{
823 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
824
825 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
826 RTCCUINTREG fEFlags = ASMIntDisableFlags();
827
828 /* If we're for some reason not in VMX root mode, then don't leave it. */
829 RTCCUINTREG uHostCR4 = ASMGetCR4();
830
831 int rc;
832 if (uHostCR4 & X86_CR4_VMXE)
833 {
834 /* Exit VMX root mode and clear the VMX bit in CR4. */
835 VMXDisable();
836 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
837 rc = VINF_SUCCESS;
838 }
839 else
840 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
841
842 /* Restore interrupts. */
843 ASMSetFlags(fEFlags);
844 return rc;
845}
846
847
848/**
849 * Allocates and maps one physically contiguous page. The allocated page is
850 * zero'd out. (Used by various VT-x structures).
851 *
852 * @returns IPRT status code.
853 * @param pMemObj Pointer to the ring-0 memory object.
854 * @param ppVirt Where to store the virtual address of the
855 * allocation.
856 * @param pHCPhys Where to store the physical address of the
857 * allocation.
858 */
859DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
860{
861 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
862 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
863 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
864
865 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
866 if (RT_FAILURE(rc))
867 return rc;
868 *ppVirt = RTR0MemObjAddress(*pMemObj);
869 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
870 ASMMemZero32(*ppVirt, PAGE_SIZE);
871 return VINF_SUCCESS;
872}
873
874
875/**
876 * Frees and unmaps an allocated physical page.
877 *
878 * @param pMemObj Pointer to the ring-0 memory object.
879 * @param ppVirt Where to re-initialize the virtual address of
880 * allocation as 0.
881 * @param pHCPhys Where to re-initialize the physical address of the
882 * allocation as 0.
883 */
884DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
885{
886 AssertPtr(pMemObj);
887 AssertPtr(ppVirt);
888 AssertPtr(pHCPhys);
889 if (*pMemObj != NIL_RTR0MEMOBJ)
890 {
891 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
892 AssertRC(rc);
893 *pMemObj = NIL_RTR0MEMOBJ;
894 *ppVirt = 0;
895 *pHCPhys = 0;
896 }
897}
898
899
900/**
901 * Worker function to free VT-x related structures.
902 *
903 * @returns IPRT status code.
904 * @param pVM The cross context VM structure.
905 */
906static void hmR0VmxStructsFree(PVM pVM)
907{
908 for (VMCPUID i = 0; i < pVM->cCpus; i++)
909 {
910 PVMCPU pVCpu = &pVM->aCpus[i];
911 AssertPtr(pVCpu);
912
913 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
914 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
915
916 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
917 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
918
919 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
920 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
921 }
922
923 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
924#ifdef VBOX_WITH_CRASHDUMP_MAGIC
925 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
926#endif
927}
928
929
930/**
931 * Worker function to allocate VT-x related VM structures.
932 *
933 * @returns IPRT status code.
934 * @param pVM The cross context VM structure.
935 */
936static int hmR0VmxStructsAlloc(PVM pVM)
937{
938 /*
939 * Initialize members up-front so we can cleanup properly on allocation failure.
940 */
941#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
942 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
943 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
944 pVM->hm.s.vmx.HCPhys##a_Name = 0;
945
946#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
947 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
948 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
949 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
950
951#ifdef VBOX_WITH_CRASHDUMP_MAGIC
952 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
953#endif
954 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
955
956 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
957 for (VMCPUID i = 0; i < pVM->cCpus; i++)
958 {
959 PVMCPU pVCpu = &pVM->aCpus[i];
960 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
961 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
962 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
963 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
964 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
965 }
966#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
967#undef VMXLOCAL_INIT_VM_MEMOBJ
968
969 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
970 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
971 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
972 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
973
974 /*
975 * Allocate all the VT-x structures.
976 */
977 int rc = VINF_SUCCESS;
978#ifdef VBOX_WITH_CRASHDUMP_MAGIC
979 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
980 if (RT_FAILURE(rc))
981 goto cleanup;
982 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
983 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
984#endif
985
986 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
987 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
988 {
989 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
990 &pVM->hm.s.vmx.HCPhysApicAccess);
991 if (RT_FAILURE(rc))
992 goto cleanup;
993 }
994
995 /*
996 * Initialize per-VCPU VT-x structures.
997 */
998 for (VMCPUID i = 0; i < pVM->cCpus; i++)
999 {
1000 PVMCPU pVCpu = &pVM->aCpus[i];
1001 AssertPtr(pVCpu);
1002
1003 /* Allocate the VM control structure (VMCS). */
1004 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1005 if (RT_FAILURE(rc))
1006 goto cleanup;
1007
1008 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
1009 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1010 {
1011 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1012 &pVCpu->hm.s.vmx.HCPhysVirtApic);
1013 if (RT_FAILURE(rc))
1014 goto cleanup;
1015 }
1016
1017 /*
1018 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1019 * transparent accesses of specific MSRs.
1020 *
1021 * If the condition for enabling MSR bitmaps changes here, don't forget to
1022 * update HMAreMsrBitmapsAvailable().
1023 */
1024 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1025 {
1026 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1027 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1028 if (RT_FAILURE(rc))
1029 goto cleanup;
1030 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1031 }
1032
1033 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1034 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1035 if (RT_FAILURE(rc))
1036 goto cleanup;
1037
1038 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1039 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1040 if (RT_FAILURE(rc))
1041 goto cleanup;
1042 }
1043
1044 return VINF_SUCCESS;
1045
1046cleanup:
1047 hmR0VmxStructsFree(pVM);
1048 return rc;
1049}
1050
1051
1052/**
1053 * Does global VT-x initialization (called during module initialization).
1054 *
1055 * @returns VBox status code.
1056 */
1057VMMR0DECL(int) VMXR0GlobalInit(void)
1058{
1059#ifdef HMVMX_USE_FUNCTION_TABLE
1060 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1061# ifdef VBOX_STRICT
1062 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1063 Assert(g_apfnVMExitHandlers[i]);
1064# endif
1065#endif
1066 return VINF_SUCCESS;
1067}
1068
1069
1070/**
1071 * Does global VT-x termination (called during module termination).
1072 */
1073VMMR0DECL(void) VMXR0GlobalTerm()
1074{
1075 /* Nothing to do currently. */
1076}
1077
1078
1079/**
1080 * Sets up and activates VT-x on the current CPU.
1081 *
1082 * @returns VBox status code.
1083 * @param pCpu Pointer to the global CPU info struct.
1084 * @param pVM The cross context VM structure. Can be
1085 * NULL after a host resume operation.
1086 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1087 * fEnabledByHost is @c true).
1088 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1089 * @a fEnabledByHost is @c true).
1090 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1091 * enable VT-x on the host.
1092 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1093 */
1094VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1095 void *pvMsrs)
1096{
1097 Assert(pCpu);
1098 Assert(pvMsrs);
1099 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1100
1101 /* Enable VT-x if it's not already enabled by the host. */
1102 if (!fEnabledByHost)
1103 {
1104 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1105 if (RT_FAILURE(rc))
1106 return rc;
1107 }
1108
1109 /*
1110 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1111 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1112 */
1113 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1114 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1115 {
1116 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1117 pCpu->fFlushAsidBeforeUse = false;
1118 }
1119 else
1120 pCpu->fFlushAsidBeforeUse = true;
1121
1122 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1123 ++pCpu->cTlbFlushes;
1124
1125 return VINF_SUCCESS;
1126}
1127
1128
1129/**
1130 * Deactivates VT-x on the current CPU.
1131 *
1132 * @returns VBox status code.
1133 * @param pCpu Pointer to the global CPU info struct.
1134 * @param pvCpuPage Pointer to the VMXON region.
1135 * @param HCPhysCpuPage Physical address of the VMXON region.
1136 *
1137 * @remarks This function should never be called when SUPR0EnableVTx() or
1138 * similar was used to enable VT-x on the host.
1139 */
1140VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1141{
1142 NOREF(pCpu);
1143 NOREF(pvCpuPage);
1144 NOREF(HCPhysCpuPage);
1145
1146 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1147 return hmR0VmxLeaveRootMode();
1148}
1149
1150
1151/**
1152 * Sets the permission bits for the specified MSR in the MSR bitmap.
1153 *
1154 * @param pVCpu The cross context virtual CPU structure.
1155 * @param uMsr The MSR value.
1156 * @param enmRead Whether reading this MSR causes a VM-exit.
1157 * @param enmWrite Whether writing this MSR causes a VM-exit.
1158 */
1159static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1160{
1161 int32_t iBit;
1162 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1163
1164 /*
1165 * Layout:
1166 * 0x000 - 0x3ff - Low MSR read bits
1167 * 0x400 - 0x7ff - High MSR read bits
1168 * 0x800 - 0xbff - Low MSR write bits
1169 * 0xc00 - 0xfff - High MSR write bits
1170 */
1171 if (uMsr <= 0x00001FFF)
1172 iBit = uMsr;
1173 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1174 {
1175 iBit = uMsr - UINT32_C(0xC0000000);
1176 pbMsrBitmap += 0x400;
1177 }
1178 else
1179 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1180
1181 Assert(iBit <= 0x1fff);
1182 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1183 ASMBitSet(pbMsrBitmap, iBit);
1184 else
1185 ASMBitClear(pbMsrBitmap, iBit);
1186
1187 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1188 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1189 else
1190 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1191}
1192
1193
1194#ifdef VBOX_STRICT
1195/**
1196 * Gets the permission bits for the specified MSR in the MSR bitmap.
1197 *
1198 * @returns VBox status code.
1199 * @retval VINF_SUCCESS if the specified MSR is found.
1200 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1201 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1202 *
1203 * @param pVCpu The cross context virtual CPU structure.
1204 * @param uMsr The MSR.
1205 * @param penmRead Where to store the read permissions.
1206 * @param penmWrite Where to store the write permissions.
1207 */
1208static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1209{
1210 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1211 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1212 int32_t iBit;
1213 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1214
1215 /* See hmR0VmxSetMsrPermission() for the layout. */
1216 if (uMsr <= 0x00001FFF)
1217 iBit = uMsr;
1218 else if ( uMsr >= 0xC0000000
1219 && uMsr <= 0xC0001FFF)
1220 {
1221 iBit = (uMsr - 0xC0000000);
1222 pbMsrBitmap += 0x400;
1223 }
1224 else
1225 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1226
1227 Assert(iBit <= 0x1fff);
1228 if (ASMBitTest(pbMsrBitmap, iBit))
1229 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1230 else
1231 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1232
1233 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1234 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1235 else
1236 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1237 return VINF_SUCCESS;
1238}
1239#endif /* VBOX_STRICT */
1240
1241
1242/**
1243 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1244 * area.
1245 *
1246 * @returns VBox status code.
1247 * @param pVCpu The cross context virtual CPU structure.
1248 * @param cMsrs The number of MSRs.
1249 */
1250DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1251{
1252 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1253 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1254 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1255 {
1256 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1257 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1258 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1259 }
1260
1261 /* Update number of guest MSRs to load/store across the world-switch. */
1262 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1263 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1264
1265 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1266 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1267 AssertRCReturn(rc, rc);
1268
1269 /* Update the VCPU's copy of the MSR count. */
1270 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1271
1272 return VINF_SUCCESS;
1273}
1274
1275
1276/**
1277 * Adds a new (or updates the value of an existing) guest/host MSR
1278 * pair to be swapped during the world-switch as part of the
1279 * auto-load/store MSR area in the VMCS.
1280 *
1281 * @returns VBox status code.
1282 * @param pVCpu The cross context virtual CPU structure.
1283 * @param uMsr The MSR.
1284 * @param uGuestMsrValue Value of the guest MSR.
1285 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1286 * necessary.
1287 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1288 * its value was updated. Optional, can be NULL.
1289 */
1290static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1291 bool *pfAddedAndUpdated)
1292{
1293 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1294 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1295 uint32_t i;
1296 for (i = 0; i < cMsrs; i++)
1297 {
1298 if (pGuestMsr->u32Msr == uMsr)
1299 break;
1300 pGuestMsr++;
1301 }
1302
1303 bool fAdded = false;
1304 if (i == cMsrs)
1305 {
1306 ++cMsrs;
1307 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1308 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1309
1310 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1311 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1312 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1313
1314 fAdded = true;
1315 }
1316
1317 /* Update the MSR values in the auto-load/store MSR area. */
1318 pGuestMsr->u32Msr = uMsr;
1319 pGuestMsr->u64Value = uGuestMsrValue;
1320
1321 /* Create/update the MSR slot in the host MSR area. */
1322 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1323 pHostMsr += i;
1324 pHostMsr->u32Msr = uMsr;
1325
1326 /*
1327 * Update the host MSR only when requested by the caller AND when we're
1328 * adding it to the auto-load/store area. Otherwise, it would have been
1329 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1330 */
1331 bool fUpdatedMsrValue = false;
1332 if ( fAdded
1333 && fUpdateHostMsr)
1334 {
1335 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1336 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1337 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1338 fUpdatedMsrValue = true;
1339 }
1340
1341 if (pfAddedAndUpdated)
1342 *pfAddedAndUpdated = fUpdatedMsrValue;
1343 return VINF_SUCCESS;
1344}
1345
1346
1347/**
1348 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1349 * auto-load/store MSR area in the VMCS.
1350 *
1351 * @returns VBox status code.
1352 * @param pVCpu The cross context virtual CPU structure.
1353 * @param uMsr The MSR.
1354 */
1355static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1356{
1357 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1358 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1359 for (uint32_t i = 0; i < cMsrs; i++)
1360 {
1361 /* Find the MSR. */
1362 if (pGuestMsr->u32Msr == uMsr)
1363 {
1364 /* If it's the last MSR, simply reduce the count. */
1365 if (i == cMsrs - 1)
1366 {
1367 --cMsrs;
1368 break;
1369 }
1370
1371 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1372 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1373 pLastGuestMsr += cMsrs - 1;
1374 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1375 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1376
1377 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1378 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1379 pLastHostMsr += cMsrs - 1;
1380 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1381 pHostMsr->u64Value = pLastHostMsr->u64Value;
1382 --cMsrs;
1383 break;
1384 }
1385 pGuestMsr++;
1386 }
1387
1388 /* Update the VMCS if the count changed (meaning the MSR was found). */
1389 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1390 {
1391 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1392 AssertRCReturn(rc, rc);
1393
1394 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1395 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1396 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1397
1398 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1399 return VINF_SUCCESS;
1400 }
1401
1402 return VERR_NOT_FOUND;
1403}
1404
1405
1406/**
1407 * Checks if the specified guest MSR is part of the auto-load/store area in
1408 * the VMCS.
1409 *
1410 * @returns true if found, false otherwise.
1411 * @param pVCpu The cross context virtual CPU structure.
1412 * @param uMsr The MSR to find.
1413 */
1414static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1415{
1416 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1417 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1418
1419 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1420 {
1421 if (pGuestMsr->u32Msr == uMsr)
1422 return true;
1423 }
1424 return false;
1425}
1426
1427
1428/**
1429 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1430 *
1431 * @param pVCpu The cross context virtual CPU structure.
1432 *
1433 * @remarks No-long-jump zone!!!
1434 */
1435static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1436{
1437 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1438 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1439 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1440 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1441
1442 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1443 {
1444 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1445
1446 /*
1447 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1448 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1449 */
1450 if (pHostMsr->u32Msr == MSR_K6_EFER)
1451 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1452 else
1453 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1454 }
1455
1456 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1457}
1458
1459
1460/**
1461 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1462 * perform lazy restoration of the host MSRs while leaving VT-x.
1463 *
1464 * @param pVCpu The cross context virtual CPU structure.
1465 *
1466 * @remarks No-long-jump zone!!!
1467 */
1468static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1469{
1470 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1471
1472 /*
1473 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1474 */
1475 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1476 {
1477 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1478#if HC_ARCH_BITS == 64
1479 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1480 {
1481 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1482 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1483 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1484 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1485 }
1486#endif
1487 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1488 }
1489}
1490
1491
1492/**
1493 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1494 * lazily while leaving VT-x.
1495 *
1496 * @returns true if it does, false otherwise.
1497 * @param pVCpu The cross context virtual CPU structure.
1498 * @param uMsr The MSR to check.
1499 */
1500static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1501{
1502 NOREF(pVCpu);
1503#if HC_ARCH_BITS == 64
1504 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1505 {
1506 switch (uMsr)
1507 {
1508 case MSR_K8_LSTAR:
1509 case MSR_K6_STAR:
1510 case MSR_K8_SF_MASK:
1511 case MSR_K8_KERNEL_GS_BASE:
1512 return true;
1513 }
1514 }
1515#endif
1516 return false;
1517}
1518
1519
1520/**
1521 * Saves a set of guest MSRs back into the guest-CPU context.
1522 *
1523 * @param pVCpu The cross context virtual CPU structure.
1524 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1525 * out-of-sync. Make sure to update the required fields
1526 * before using them.
1527 *
1528 * @remarks No-long-jump zone!!!
1529 */
1530static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1531{
1532 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1533 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1534
1535 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1536 {
1537 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1538#if HC_ARCH_BITS == 64
1539 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1540 {
1541 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1542 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1543 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1544 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1545 }
1546#endif
1547 }
1548}
1549
1550
1551/**
1552 * Loads a set of guests MSRs to allow read/passthru to the guest.
1553 *
1554 * The name of this function is slightly confusing. This function does NOT
1555 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1556 * common prefix for functions dealing with "lazy restoration" of the shared
1557 * MSRs.
1558 *
1559 * @param pVCpu The cross context virtual CPU structure.
1560 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1561 * out-of-sync. Make sure to update the required fields
1562 * before using them.
1563 *
1564 * @remarks No-long-jump zone!!!
1565 */
1566static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1567{
1568 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1569 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1570
1571#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1572 do { \
1573 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1574 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1575 else \
1576 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1577 } while (0)
1578
1579 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1580 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1581 {
1582#if HC_ARCH_BITS == 64
1583 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1584 {
1585 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1586 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1587 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1588 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1589 }
1590#endif
1591 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1592 }
1593
1594#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1595}
1596
1597
1598/**
1599 * Performs lazy restoration of the set of host MSRs if they were previously
1600 * loaded with guest MSR values.
1601 *
1602 * @param pVCpu The cross context virtual CPU structure.
1603 *
1604 * @remarks No-long-jump zone!!!
1605 * @remarks The guest MSRs should have been saved back into the guest-CPU
1606 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1607 */
1608static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1609{
1610 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1611 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1612
1613 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1614 {
1615 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1616#if HC_ARCH_BITS == 64
1617 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1618 {
1619 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1620 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1621 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1622 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1623 }
1624#endif
1625 }
1626 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1627}
1628
1629
1630/**
1631 * Verifies that our cached values of the VMCS controls are all
1632 * consistent with what's actually present in the VMCS.
1633 *
1634 * @returns VBox status code.
1635 * @param pVCpu The cross context virtual CPU structure.
1636 */
1637static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1638{
1639 uint32_t u32Val;
1640 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1641 AssertRCReturn(rc, rc);
1642 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1643 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1644
1645 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1646 AssertRCReturn(rc, rc);
1647 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1648 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1649
1650 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1651 AssertRCReturn(rc, rc);
1652 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1653 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1654
1655 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1656 AssertRCReturn(rc, rc);
1657 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1658 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1659
1660 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1661 {
1662 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1663 AssertRCReturn(rc, rc);
1664 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1665 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1666 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1667 }
1668
1669 return VINF_SUCCESS;
1670}
1671
1672
1673#ifdef VBOX_STRICT
1674/**
1675 * Verifies that our cached host EFER value has not changed
1676 * since we cached it.
1677 *
1678 * @param pVCpu The cross context virtual CPU structure.
1679 */
1680static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1681{
1682 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1683
1684 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1685 {
1686 uint64_t u64Val;
1687 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1688 AssertRC(rc);
1689
1690 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1691 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1692 }
1693}
1694
1695
1696/**
1697 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1698 * VMCS are correct.
1699 *
1700 * @param pVCpu The cross context virtual CPU structure.
1701 */
1702static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1703{
1704 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1705
1706 /* Verify MSR counts in the VMCS are what we think it should be. */
1707 uint32_t cMsrs;
1708 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1709 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1710
1711 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1712 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1713
1714 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1715 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1716
1717 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1718 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1719 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1720 {
1721 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1722 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1723 pGuestMsr->u32Msr, cMsrs));
1724
1725 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1726 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1727 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1728
1729 /* Verify that the permissions are as expected in the MSR bitmap. */
1730 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1731 {
1732 VMXMSREXITREAD enmRead;
1733 VMXMSREXITWRITE enmWrite;
1734 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1735 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1736 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1737 {
1738 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1739 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1740 }
1741 else
1742 {
1743 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1744 pGuestMsr->u32Msr, cMsrs));
1745 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1746 pGuestMsr->u32Msr, cMsrs));
1747 }
1748 }
1749 }
1750}
1751#endif /* VBOX_STRICT */
1752
1753
1754/**
1755 * Flushes the TLB using EPT.
1756 *
1757 * @returns VBox status code.
1758 * @param pVCpu The cross context virtual CPU structure of the calling
1759 * EMT. Can be NULL depending on @a enmFlush.
1760 * @param enmFlush Type of flush.
1761 *
1762 * @remarks Caller is responsible for making sure this function is called only
1763 * when NestedPaging is supported and providing @a enmFlush that is
1764 * supported by the CPU.
1765 * @remarks Can be called with interrupts disabled.
1766 */
1767static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1768{
1769 uint64_t au64Descriptor[2];
1770 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1771 au64Descriptor[0] = 0;
1772 else
1773 {
1774 Assert(pVCpu);
1775 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1776 }
1777 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1778
1779 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1780 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1781 rc));
1782 if ( RT_SUCCESS(rc)
1783 && pVCpu)
1784 {
1785 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1786 }
1787}
1788
1789
1790/**
1791 * Flushes the TLB using VPID.
1792 *
1793 * @returns VBox status code.
1794 * @param pVM The cross context VM structure.
1795 * @param pVCpu The cross context virtual CPU structure of the calling
1796 * EMT. Can be NULL depending on @a enmFlush.
1797 * @param enmFlush Type of flush.
1798 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1799 * on @a enmFlush).
1800 *
1801 * @remarks Can be called with interrupts disabled.
1802 */
1803static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1804{
1805 NOREF(pVM);
1806 AssertPtr(pVM);
1807 Assert(pVM->hm.s.vmx.fVpid);
1808
1809 uint64_t au64Descriptor[2];
1810 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1811 {
1812 au64Descriptor[0] = 0;
1813 au64Descriptor[1] = 0;
1814 }
1815 else
1816 {
1817 AssertPtr(pVCpu);
1818 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1819 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1820 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1821 au64Descriptor[1] = GCPtr;
1822 }
1823
1824 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1825 AssertMsg(rc == VINF_SUCCESS,
1826 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1827 if ( RT_SUCCESS(rc)
1828 && pVCpu)
1829 {
1830 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1831 }
1832}
1833
1834
1835/**
1836 * Invalidates a guest page by guest virtual address. Only relevant for
1837 * EPT/VPID, otherwise there is nothing really to invalidate.
1838 *
1839 * @returns VBox status code.
1840 * @param pVM The cross context VM structure.
1841 * @param pVCpu The cross context virtual CPU structure.
1842 * @param GCVirt Guest virtual address of the page to invalidate.
1843 */
1844VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1845{
1846 AssertPtr(pVM);
1847 AssertPtr(pVCpu);
1848 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1849
1850 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1851 if (!fFlushPending)
1852 {
1853 /*
1854 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1855 * See @bugref{6043} and @bugref{6177}.
1856 *
1857 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1858 * function maybe called in a loop with individual addresses.
1859 */
1860 if (pVM->hm.s.vmx.fVpid)
1861 {
1862 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1863 {
1864 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1865 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1866 }
1867 else
1868 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1869 }
1870 else if (pVM->hm.s.fNestedPaging)
1871 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1872 }
1873
1874 return VINF_SUCCESS;
1875}
1876
1877
1878/**
1879 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1880 * otherwise there is nothing really to invalidate.
1881 *
1882 * @returns VBox status code.
1883 * @param pVM The cross context VM structure.
1884 * @param pVCpu The cross context virtual CPU structure.
1885 * @param GCPhys Guest physical address of the page to invalidate.
1886 */
1887VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1888{
1889 NOREF(pVM); NOREF(GCPhys);
1890 LogFlowFunc(("%RGp\n", GCPhys));
1891
1892 /*
1893 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1894 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1895 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1896 */
1897 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1898 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1899 return VINF_SUCCESS;
1900}
1901
1902
1903/**
1904 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1905 * case where neither EPT nor VPID is supported by the CPU.
1906 *
1907 * @param pVM The cross context VM structure.
1908 * @param pVCpu The cross context virtual CPU structure.
1909 * @param pCpu Pointer to the global HM struct.
1910 *
1911 * @remarks Called with interrupts disabled.
1912 */
1913static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1914{
1915 AssertPtr(pVCpu);
1916 AssertPtr(pCpu);
1917 NOREF(pVM);
1918
1919 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1920
1921 Assert(pCpu->idCpu != NIL_RTCPUID);
1922 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1923 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1924 pVCpu->hm.s.fForceTLBFlush = false;
1925 return;
1926}
1927
1928
1929/**
1930 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1931 *
1932 * @param pVM The cross context VM structure.
1933 * @param pVCpu The cross context virtual CPU structure.
1934 * @param pCpu Pointer to the global HM CPU struct.
1935 * @remarks All references to "ASID" in this function pertains to "VPID" in
1936 * Intel's nomenclature. The reason is, to avoid confusion in compare
1937 * statements since the host-CPU copies are named "ASID".
1938 *
1939 * @remarks Called with interrupts disabled.
1940 */
1941static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1942{
1943#ifdef VBOX_WITH_STATISTICS
1944 bool fTlbFlushed = false;
1945# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1946# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1947 if (!fTlbFlushed) \
1948 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1949 } while (0)
1950#else
1951# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1952# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1953#endif
1954
1955 AssertPtr(pVM);
1956 AssertPtr(pCpu);
1957 AssertPtr(pVCpu);
1958 Assert(pCpu->idCpu != NIL_RTCPUID);
1959
1960 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1961 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1962 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1963
1964 /*
1965 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1966 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1967 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1968 */
1969 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1970 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1971 {
1972 ++pCpu->uCurrentAsid;
1973 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1974 {
1975 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1976 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1977 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1978 }
1979
1980 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1981 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1982 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1983
1984 /*
1985 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1986 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1987 */
1988 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1989 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1990 HMVMX_SET_TAGGED_TLB_FLUSHED();
1991 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1992 }
1993
1994 /* Check for explicit TLB flushes. */
1995 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1996 {
1997 /*
1998 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1999 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2000 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2001 * but not guest-physical mappings.
2002 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2003 */
2004 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2005 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2006 HMVMX_SET_TAGGED_TLB_FLUSHED();
2007 }
2008
2009 pVCpu->hm.s.fForceTLBFlush = false;
2010 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2011
2012 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2013 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2014 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2015 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2016 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2017 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2018 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2019 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2020 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2021
2022 /* Update VMCS with the VPID. */
2023 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2024 AssertRC(rc);
2025
2026#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2027}
2028
2029
2030/**
2031 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2032 *
2033 * @returns VBox status code.
2034 * @param pVM The cross context VM structure.
2035 * @param pVCpu The cross context virtual CPU structure.
2036 * @param pCpu Pointer to the global HM CPU struct.
2037 *
2038 * @remarks Called with interrupts disabled.
2039 */
2040static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2041{
2042 AssertPtr(pVM);
2043 AssertPtr(pVCpu);
2044 AssertPtr(pCpu);
2045 Assert(pCpu->idCpu != NIL_RTCPUID);
2046 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2047 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2048
2049 /*
2050 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2051 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2052 */
2053 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2054 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2055 {
2056 pVCpu->hm.s.fForceTLBFlush = true;
2057 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2058 }
2059
2060 /* Check for explicit TLB flushes. */
2061 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2062 {
2063 pVCpu->hm.s.fForceTLBFlush = true;
2064 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2065 }
2066
2067 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2068 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2069
2070 if (pVCpu->hm.s.fForceTLBFlush)
2071 {
2072 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2073 pVCpu->hm.s.fForceTLBFlush = false;
2074 }
2075}
2076
2077
2078/**
2079 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2080 *
2081 * @returns VBox status code.
2082 * @param pVM The cross context VM structure.
2083 * @param pVCpu The cross context virtual CPU structure.
2084 * @param pCpu Pointer to the global HM CPU struct.
2085 *
2086 * @remarks Called with interrupts disabled.
2087 */
2088static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2089{
2090 AssertPtr(pVM);
2091 AssertPtr(pVCpu);
2092 AssertPtr(pCpu);
2093 Assert(pCpu->idCpu != NIL_RTCPUID);
2094 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2095 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2096
2097 /*
2098 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2099 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2100 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2101 */
2102 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2103 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2104 {
2105 pVCpu->hm.s.fForceTLBFlush = true;
2106 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2107 }
2108
2109 /* Check for explicit TLB flushes. */
2110 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2111 {
2112 /*
2113 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2114 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2115 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2116 */
2117 pVCpu->hm.s.fForceTLBFlush = true;
2118 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2119 }
2120
2121 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2122 if (pVCpu->hm.s.fForceTLBFlush)
2123 {
2124 ++pCpu->uCurrentAsid;
2125 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2126 {
2127 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2128 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2129 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2130 }
2131
2132 pVCpu->hm.s.fForceTLBFlush = false;
2133 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2134 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2135 if (pCpu->fFlushAsidBeforeUse)
2136 {
2137 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2138 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2139 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2140 {
2141 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2142 pCpu->fFlushAsidBeforeUse = false;
2143 }
2144 else
2145 {
2146 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2147 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2148 }
2149 }
2150 }
2151
2152 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2153 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2154 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2155 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2156 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2157 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2158 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2159
2160 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2161 AssertRC(rc);
2162}
2163
2164
2165/**
2166 * Flushes the guest TLB entry based on CPU capabilities.
2167 *
2168 * @param pVCpu The cross context virtual CPU structure.
2169 * @param pCpu Pointer to the global HM CPU struct.
2170 */
2171DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2172{
2173#ifdef HMVMX_ALWAYS_FLUSH_TLB
2174 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2175#endif
2176 PVM pVM = pVCpu->CTX_SUFF(pVM);
2177 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2178 {
2179 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2180 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2181 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2182 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2183 default:
2184 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2185 break;
2186 }
2187
2188 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2189}
2190
2191
2192/**
2193 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2194 * TLB entries from the host TLB before VM-entry.
2195 *
2196 * @returns VBox status code.
2197 * @param pVM The cross context VM structure.
2198 */
2199static int hmR0VmxSetupTaggedTlb(PVM pVM)
2200{
2201 /*
2202 * Determine optimal flush type for Nested Paging.
2203 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2204 * guest execution (see hmR3InitFinalizeR0()).
2205 */
2206 if (pVM->hm.s.fNestedPaging)
2207 {
2208 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2209 {
2210 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2211 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2212 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2213 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2214 else
2215 {
2216 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2217 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2218 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2219 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2220 }
2221
2222 /* Make sure the write-back cacheable memory type for EPT is supported. */
2223 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2224 {
2225 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2226 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2227 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2228 }
2229
2230 /* EPT requires a page-walk length of 4. */
2231 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2232 {
2233 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2234 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2235 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2236 }
2237 }
2238 else
2239 {
2240 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2241 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2242 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2243 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2244 }
2245 }
2246
2247 /*
2248 * Determine optimal flush type for VPID.
2249 */
2250 if (pVM->hm.s.vmx.fVpid)
2251 {
2252 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2253 {
2254 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2255 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2256 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2257 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2258 else
2259 {
2260 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2261 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2262 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2263 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2264 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2265 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2266 pVM->hm.s.vmx.fVpid = false;
2267 }
2268 }
2269 else
2270 {
2271 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2272 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2273 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2274 pVM->hm.s.vmx.fVpid = false;
2275 }
2276 }
2277
2278 /*
2279 * Setup the handler for flushing tagged-TLBs.
2280 */
2281 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2282 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2283 else if (pVM->hm.s.fNestedPaging)
2284 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2285 else if (pVM->hm.s.vmx.fVpid)
2286 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2287 else
2288 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2289 return VINF_SUCCESS;
2290}
2291
2292
2293/**
2294 * Sets up pin-based VM-execution controls in the VMCS.
2295 *
2296 * @returns VBox status code.
2297 * @param pVM The cross context VM structure.
2298 * @param pVCpu The cross context virtual CPU structure.
2299 */
2300static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2301{
2302 AssertPtr(pVM);
2303 AssertPtr(pVCpu);
2304
2305 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2306 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2307
2308 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2309 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2310
2311 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2312 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2313
2314 /* Enable the VMX preemption timer. */
2315 if (pVM->hm.s.vmx.fUsePreemptTimer)
2316 {
2317 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2318 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2319 }
2320
2321#ifdef VBOX_WITH_NEW_APIC
2322#if 0
2323 /* Enable posted-interrupt processing. */
2324 if (pVM->hm.s.fPostedIntrs)
2325 {
2326 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2327 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2328 val |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2329 }
2330#endif
2331#endif
2332
2333 if ((val & zap) != val)
2334 {
2335 LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2336 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2337 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2338 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2339 }
2340
2341 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2342 AssertRCReturn(rc, rc);
2343
2344 pVCpu->hm.s.vmx.u32PinCtls = val;
2345 return rc;
2346}
2347
2348
2349/**
2350 * Sets up processor-based VM-execution controls in the VMCS.
2351 *
2352 * @returns VBox status code.
2353 * @param pVM The cross context VM structure.
2354 * @param pVCpu The cross context virtual CPU structure.
2355 */
2356static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2357{
2358 AssertPtr(pVM);
2359 AssertPtr(pVCpu);
2360
2361 int rc = VERR_INTERNAL_ERROR_5;
2362 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2363 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2364
2365 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2366 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2367 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2368 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2369 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2370 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2371 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2372
2373 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2374 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2375 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2376 {
2377 LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2378 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2379 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2380 }
2381
2382 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2383 if (!pVM->hm.s.fNestedPaging)
2384 {
2385 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2386 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2387 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2388 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2389 }
2390
2391 /* Use TPR shadowing if supported by the CPU. */
2392 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2393 {
2394 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2395 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2396 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2397 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2398 AssertRCReturn(rc, rc);
2399
2400 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2401 /* CR8 writes cause a VM-exit based on TPR threshold. */
2402 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2403 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2404 }
2405 else
2406 {
2407 /*
2408 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2409 * Set this control only for 64-bit guests.
2410 */
2411 if (pVM->hm.s.fAllow64BitGuests)
2412 {
2413 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2414 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2415 }
2416 }
2417
2418 /* Use MSR-bitmaps if supported by the CPU. */
2419 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2420 {
2421 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2422
2423 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2424 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2425 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2426 AssertRCReturn(rc, rc);
2427
2428 /*
2429 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2430 * automatically using dedicated fields in the VMCS.
2431 */
2432 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2433 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2434 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2435 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2436 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2437
2438#if HC_ARCH_BITS == 64
2439 /*
2440 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2441 */
2442 if (pVM->hm.s.fAllow64BitGuests)
2443 {
2444 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2445 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2446 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2447 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2448 }
2449#endif
2450 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2451 }
2452
2453 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2454 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2455 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2456
2457 if ((val & zap) != val)
2458 {
2459 LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2460 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2461 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2462 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2463 }
2464
2465 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2466 AssertRCReturn(rc, rc);
2467
2468 pVCpu->hm.s.vmx.u32ProcCtls = val;
2469
2470 /*
2471 * Secondary processor-based VM-execution controls.
2472 */
2473 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2474 {
2475 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2476 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2477
2478 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2479 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2480
2481 if (pVM->hm.s.fNestedPaging)
2482 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2483 else
2484 {
2485 /*
2486 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2487 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2488 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2489 */
2490 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2491 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2492 }
2493
2494 if (pVM->hm.s.vmx.fVpid)
2495 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2496
2497 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2498 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2499
2500#ifdef VBOX_WITH_NEW_APIC
2501#if 0
2502 if (pVM->hm.s.fVirtApicRegs)
2503 {
2504 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2505 val |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2506
2507 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2508 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2509 }
2510#endif
2511#endif
2512
2513 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2514 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2515 * done dynamically. */
2516 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2517 {
2518 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2519 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2520 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2521 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2522 AssertRCReturn(rc, rc);
2523 }
2524
2525 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2526 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2527
2528 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2529 && pVM->hm.s.vmx.cPleGapTicks
2530 && pVM->hm.s.vmx.cPleWindowTicks)
2531 {
2532 val |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2533
2534 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2535 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2536 AssertRCReturn(rc, rc);
2537 }
2538
2539 if ((val & zap) != val)
2540 {
2541 LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2542 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2543 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2544 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2545 }
2546
2547 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2548 AssertRCReturn(rc, rc);
2549
2550 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2551 }
2552 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2553 {
2554 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2555 "available\n"));
2556 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2557 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2558 }
2559
2560 return VINF_SUCCESS;
2561}
2562
2563
2564/**
2565 * Sets up miscellaneous (everything other than Pin & Processor-based
2566 * VM-execution) control fields in the VMCS.
2567 *
2568 * @returns VBox status code.
2569 * @param pVM The cross context VM structure.
2570 * @param pVCpu The cross context virtual CPU structure.
2571 */
2572static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2573{
2574 NOREF(pVM);
2575 AssertPtr(pVM);
2576 AssertPtr(pVCpu);
2577
2578 int rc = VERR_GENERAL_FAILURE;
2579
2580 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2581#if 0
2582 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2583 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2584 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2585
2586 /*
2587 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2588 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2589 * We thus use the exception bitmap to control it rather than use both.
2590 */
2591 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2592 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2593
2594 /** @todo Explore possibility of using IO-bitmaps. */
2595 /* All IO & IOIO instructions cause VM-exits. */
2596 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2597 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2598
2599 /* Initialize the MSR-bitmap area. */
2600 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2601 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2602 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2603 AssertRCReturn(rc, rc);
2604#endif
2605
2606 /* Setup MSR auto-load/store area. */
2607 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2608 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2609 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2610 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2611 AssertRCReturn(rc, rc);
2612
2613 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2614 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2615 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2616 AssertRCReturn(rc, rc);
2617
2618 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2619 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2620 AssertRCReturn(rc, rc);
2621
2622 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2623#if 0
2624 /* Setup debug controls */
2625 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2626 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2627 AssertRCReturn(rc, rc);
2628#endif
2629
2630 return rc;
2631}
2632
2633
2634/**
2635 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2636 *
2637 * @returns VBox status code.
2638 * @param pVM The cross context VM structure.
2639 * @param pVCpu The cross context virtual CPU structure.
2640 */
2641static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2642{
2643 AssertPtr(pVM);
2644 AssertPtr(pVCpu);
2645
2646 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2647
2648 uint32_t u32XcptBitmap = pVCpu->hm.s.fGIMTrapXcptUD ? RT_BIT(X86_XCPT_UD) : 0;
2649
2650 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2651 u32XcptBitmap |= RT_BIT_32(X86_XCPT_AC);
2652
2653 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2654 and writes, and because recursive #DBs can cause the CPU hang, we must always
2655 intercept #DB. */
2656 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2657
2658 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2659 if (!pVM->hm.s.fNestedPaging)
2660 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2661
2662 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2663 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2664 AssertRCReturn(rc, rc);
2665 return rc;
2666}
2667
2668
2669/**
2670 * Sets up the initial guest-state mask. The guest-state mask is consulted
2671 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2672 * for the nested virtualization case (as it would cause a VM-exit).
2673 *
2674 * @param pVCpu The cross context virtual CPU structure.
2675 */
2676static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2677{
2678 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2679 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2680 return VINF_SUCCESS;
2681}
2682
2683
2684/**
2685 * Does per-VM VT-x initialization.
2686 *
2687 * @returns VBox status code.
2688 * @param pVM The cross context VM structure.
2689 */
2690VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2691{
2692 LogFlowFunc(("pVM=%p\n", pVM));
2693
2694 int rc = hmR0VmxStructsAlloc(pVM);
2695 if (RT_FAILURE(rc))
2696 {
2697 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2698 return rc;
2699 }
2700
2701 return VINF_SUCCESS;
2702}
2703
2704
2705/**
2706 * Does per-VM VT-x termination.
2707 *
2708 * @returns VBox status code.
2709 * @param pVM The cross context VM structure.
2710 */
2711VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2712{
2713 LogFlowFunc(("pVM=%p\n", pVM));
2714
2715#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2716 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2717 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2718#endif
2719 hmR0VmxStructsFree(pVM);
2720 return VINF_SUCCESS;
2721}
2722
2723
2724/**
2725 * Sets up the VM for execution under VT-x.
2726 * This function is only called once per-VM during initialization.
2727 *
2728 * @returns VBox status code.
2729 * @param pVM The cross context VM structure.
2730 */
2731VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2732{
2733 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2734 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2735
2736 LogFlowFunc(("pVM=%p\n", pVM));
2737
2738 /*
2739 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2740 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2741 */
2742 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2743 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2744 || !pVM->hm.s.vmx.pRealModeTSS))
2745 {
2746 LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2747 return VERR_INTERNAL_ERROR;
2748 }
2749
2750 /* Initialize these always, see hmR3InitFinalizeR0().*/
2751 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2752 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2753
2754 /* Setup the tagged-TLB flush handlers. */
2755 int rc = hmR0VmxSetupTaggedTlb(pVM);
2756 if (RT_FAILURE(rc))
2757 {
2758 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2759 return rc;
2760 }
2761
2762 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2763 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2764#if HC_ARCH_BITS == 64
2765 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2766 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2767 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2768 {
2769 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2770 }
2771#endif
2772
2773 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2774 RTCCUINTREG uHostCR4 = ASMGetCR4();
2775 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2776 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2777
2778 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2779 {
2780 PVMCPU pVCpu = &pVM->aCpus[i];
2781 AssertPtr(pVCpu);
2782 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2783
2784 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2785 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2786
2787 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2788 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2789 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2790
2791 /* Set revision dword at the beginning of the VMCS structure. */
2792 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2793
2794 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2795 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2796 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2797 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2798
2799 /* Load this VMCS as the current VMCS. */
2800 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2801 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2802 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2803
2804 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2805 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2806 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2807
2808 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2809 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2810 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2811
2812 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2813 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2814 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2815
2816 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2817 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2818 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2819
2820 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2821 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2822 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2823
2824#if HC_ARCH_BITS == 32
2825 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2826 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2827 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2828#endif
2829
2830 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2831 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2832 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2833 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2834
2835 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2836
2837 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2838 }
2839
2840 return VINF_SUCCESS;
2841}
2842
2843
2844/**
2845 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2846 * the VMCS.
2847 *
2848 * @returns VBox status code.
2849 * @param pVM The cross context VM structure.
2850 * @param pVCpu The cross context virtual CPU structure.
2851 */
2852DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2853{
2854 NOREF(pVM); NOREF(pVCpu);
2855
2856 RTCCUINTREG uReg = ASMGetCR0();
2857 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2858 AssertRCReturn(rc, rc);
2859
2860 uReg = ASMGetCR3();
2861 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2862 AssertRCReturn(rc, rc);
2863
2864 uReg = ASMGetCR4();
2865 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2866 AssertRCReturn(rc, rc);
2867 return rc;
2868}
2869
2870
2871#if HC_ARCH_BITS == 64
2872/**
2873 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2874 * requirements. See hmR0VmxSaveHostSegmentRegs().
2875 */
2876# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2877 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2878 { \
2879 bool fValidSelector = true; \
2880 if ((selValue) & X86_SEL_LDT) \
2881 { \
2882 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2883 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2884 } \
2885 if (fValidSelector) \
2886 { \
2887 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2888 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2889 } \
2890 (selValue) = 0; \
2891 }
2892#endif
2893
2894
2895/**
2896 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2897 * the host-state area in the VMCS.
2898 *
2899 * @returns VBox status code.
2900 * @param pVM The cross context VM structure.
2901 * @param pVCpu The cross context virtual CPU structure.
2902 */
2903DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2904{
2905 int rc = VERR_INTERNAL_ERROR_5;
2906
2907#if HC_ARCH_BITS == 64
2908 /*
2909 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2910 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2911 *
2912 * This apparently can happen (most likely the FPU changes), deal with it rather than asserting.
2913 * Was observed booting Solaris10u10 32-bit guest.
2914 */
2915 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2916 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2917 {
2918 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2919 pVCpu->idCpu));
2920 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2921 }
2922 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2923#endif
2924
2925 /*
2926 * Host DS, ES, FS and GS segment registers.
2927 */
2928#if HC_ARCH_BITS == 64
2929 RTSEL uSelDS = ASMGetDS();
2930 RTSEL uSelES = ASMGetES();
2931 RTSEL uSelFS = ASMGetFS();
2932 RTSEL uSelGS = ASMGetGS();
2933#else
2934 RTSEL uSelDS = 0;
2935 RTSEL uSelES = 0;
2936 RTSEL uSelFS = 0;
2937 RTSEL uSelGS = 0;
2938#endif
2939
2940 /*
2941 * Host CS and SS segment registers.
2942 */
2943 RTSEL uSelCS = ASMGetCS();
2944 RTSEL uSelSS = ASMGetSS();
2945
2946 /*
2947 * Host TR segment register.
2948 */
2949 RTSEL uSelTR = ASMGetTR();
2950
2951#if HC_ARCH_BITS == 64
2952 /*
2953 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2954 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2955 */
2956 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2957 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2958 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2959 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2960# undef VMXLOCAL_ADJUST_HOST_SEG
2961#endif
2962
2963 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2964 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2965 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2966 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2967 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2968 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2969 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2970 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2971 Assert(uSelCS);
2972 Assert(uSelTR);
2973
2974 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2975#if 0
2976 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2977 Assert(uSelSS != 0);
2978#endif
2979
2980 /* Write these host selector fields into the host-state area in the VMCS. */
2981 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2982 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2983#if HC_ARCH_BITS == 64
2984 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2985 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2986 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2987 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2988#else
2989 NOREF(uSelDS);
2990 NOREF(uSelES);
2991 NOREF(uSelFS);
2992 NOREF(uSelGS);
2993#endif
2994 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2995 AssertRCReturn(rc, rc);
2996
2997 /*
2998 * Host GDTR and IDTR.
2999 */
3000 RTGDTR Gdtr;
3001 RTIDTR Idtr;
3002 RT_ZERO(Gdtr);
3003 RT_ZERO(Idtr);
3004 ASMGetGDTR(&Gdtr);
3005 ASMGetIDTR(&Idtr);
3006 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3007 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3008 AssertRCReturn(rc, rc);
3009
3010#if HC_ARCH_BITS == 64
3011 /*
3012 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3013 * maximum limit (0xffff) on every VM-exit.
3014 */
3015 if (Gdtr.cbGdt != 0xffff)
3016 {
3017 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3018 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3019 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3020 }
3021
3022 /*
3023 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3024 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3025 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3026 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3027 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3028 * hosts where we are pretty sure it won't cause trouble.
3029 */
3030# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3031 if (Idtr.cbIdt < 0x0fff)
3032# else
3033 if (Idtr.cbIdt != 0xffff)
3034# endif
3035 {
3036 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3037 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3038 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3039 }
3040#endif
3041
3042 /*
3043 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3044 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3045 */
3046 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3047 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3048 VERR_VMX_INVALID_HOST_STATE);
3049
3050 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3051#if HC_ARCH_BITS == 64
3052 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3053
3054 /*
3055 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3056 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3057 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3058 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3059 *
3060 * [1] See Intel spec. 3.5 "System Descriptor Types".
3061 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3062 */
3063 Assert(pDesc->System.u4Type == 11);
3064 if ( pDesc->System.u16LimitLow != 0x67
3065 || pDesc->System.u4LimitHigh)
3066 {
3067 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3068 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3069 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3070 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3071 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3072
3073 /* Store the GDTR here as we need it while restoring TR. */
3074 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3075 }
3076#else
3077 NOREF(pVM);
3078 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3079#endif
3080 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3081 AssertRCReturn(rc, rc);
3082
3083 /*
3084 * Host FS base and GS base.
3085 */
3086#if HC_ARCH_BITS == 64
3087 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3088 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3089 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3090 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3091 AssertRCReturn(rc, rc);
3092
3093 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3094 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3095 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3096 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3097 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3098#endif
3099 return rc;
3100}
3101
3102
3103/**
3104 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3105 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3106 * the host after every successful VM-exit.
3107 *
3108 * @returns VBox status code.
3109 * @param pVM The cross context VM structure.
3110 * @param pVCpu The cross context virtual CPU structure.
3111 *
3112 * @remarks No-long-jump zone!!!
3113 */
3114DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3115{
3116 NOREF(pVM);
3117
3118 AssertPtr(pVCpu);
3119 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3120
3121 /*
3122 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3123 * rather than swapping them on every VM-entry.
3124 */
3125 hmR0VmxLazySaveHostMsrs(pVCpu);
3126
3127 /*
3128 * Host Sysenter MSRs.
3129 */
3130 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3131#if HC_ARCH_BITS == 32
3132 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3133 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3134#else
3135 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3136 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3137#endif
3138 AssertRCReturn(rc, rc);
3139
3140 /*
3141 * Host EFER MSR.
3142 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3143 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3144 */
3145 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3146 {
3147 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3148 AssertRCReturn(rc, rc);
3149 }
3150
3151 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3152 * hmR0VmxLoadGuestExitCtls() !! */
3153
3154 return rc;
3155}
3156
3157
3158/**
3159 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3160 *
3161 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3162 * these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3163 * hmR0VMxLoadGuestEntryCtls().
3164 *
3165 * @returns true if we need to load guest EFER, false otherwise.
3166 * @param pVCpu The cross context virtual CPU structure.
3167 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3168 * out-of-sync. Make sure to update the required fields
3169 * before using them.
3170 *
3171 * @remarks Requires EFER, CR4.
3172 * @remarks No-long-jump zone!!!
3173 */
3174static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3175{
3176#ifdef HMVMX_ALWAYS_SWAP_EFER
3177 return true;
3178#endif
3179
3180#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3181 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3182 if (CPUMIsGuestInLongMode(pVCpu))
3183 return false;
3184#endif
3185
3186 PVM pVM = pVCpu->CTX_SUFF(pVM);
3187 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3188 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3189
3190 /*
3191 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3192 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3193 */
3194 if ( CPUMIsGuestInLongMode(pVCpu)
3195 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3196 {
3197 return true;
3198 }
3199
3200 /*
3201 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3202 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3203 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3204 */
3205 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3206 && (pMixedCtx->cr0 & X86_CR0_PG)
3207 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3208 {
3209 /* Assert that host is PAE capable. */
3210 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3211 return true;
3212 }
3213
3214 /** @todo Check the latest Intel spec. for any other bits,
3215 * like SMEP/SMAP? */
3216 return false;
3217}
3218
3219
3220/**
3221 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3222 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3223 * controls".
3224 *
3225 * @returns VBox status code.
3226 * @param pVCpu The cross context virtual CPU structure.
3227 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3228 * out-of-sync. Make sure to update the required fields
3229 * before using them.
3230 *
3231 * @remarks Requires EFER.
3232 * @remarks No-long-jump zone!!!
3233 */
3234DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3235{
3236 int rc = VINF_SUCCESS;
3237 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3238 {
3239 PVM pVM = pVCpu->CTX_SUFF(pVM);
3240 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3241 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3242
3243 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3244 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3245
3246 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3247 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3248 {
3249 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3250 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3251 }
3252 else
3253 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3254
3255 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3256 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3257 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3258 {
3259 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3260 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3261 }
3262
3263 /*
3264 * The following should -not- be set (since we're not in SMM mode):
3265 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3266 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3267 */
3268
3269 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3270 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3271
3272 if ((val & zap) != val)
3273 {
3274 LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3275 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3276 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3277 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3278 }
3279
3280 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3281 AssertRCReturn(rc, rc);
3282
3283 pVCpu->hm.s.vmx.u32EntryCtls = val;
3284 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3285 }
3286 return rc;
3287}
3288
3289
3290/**
3291 * Sets up the VM-exit controls in the VMCS.
3292 *
3293 * @returns VBox status code.
3294 * @param pVCpu The cross context virtual CPU structure.
3295 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3296 * out-of-sync. Make sure to update the required fields
3297 * before using them.
3298 *
3299 * @remarks Requires EFER.
3300 */
3301DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3302{
3303 NOREF(pMixedCtx);
3304
3305 int rc = VINF_SUCCESS;
3306 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3307 {
3308 PVM pVM = pVCpu->CTX_SUFF(pVM);
3309 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3310 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3311
3312 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3313 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3314
3315 /*
3316 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3317 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3318 */
3319#if HC_ARCH_BITS == 64
3320 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3321 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3322#else
3323 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3324 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3325 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3326 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3327 {
3328 /* The switcher returns to long mode, EFER is managed by the switcher. */
3329 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3330 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3331 }
3332 else
3333 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3334#endif
3335
3336 /* If the newer VMCS fields for managing EFER exists, use it. */
3337 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3338 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3339 {
3340 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3341 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3342 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3343 }
3344
3345 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3346 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3347
3348 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3349 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3350 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3351
3352 if ( pVM->hm.s.vmx.fUsePreemptTimer
3353 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3354 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3355
3356 if ((val & zap) != val)
3357 {
3358 LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3359 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3360 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3361 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3362 }
3363
3364 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3365 AssertRCReturn(rc, rc);
3366
3367 pVCpu->hm.s.vmx.u32ExitCtls = val;
3368 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3369 }
3370 return rc;
3371}
3372
3373
3374/**
3375 * Sets the TPR threshold in the VMCS.
3376 *
3377 * @returns VBox status code.
3378 * @param pVCpu The cross context virtual CPU structure.
3379 * @param u32TprThreshold The TPR threshold (task-priority class only).
3380 */
3381DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3382{
3383 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3384 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
3385 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3386}
3387
3388
3389/**
3390 * Loads the guest APIC and related state.
3391 *
3392 * @returns VBox status code.
3393 * @param pVCpu The cross context virtual CPU structure.
3394 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3395 * out-of-sync. Make sure to update the required fields
3396 * before using them.
3397 */
3398DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3399{
3400 NOREF(pMixedCtx);
3401
3402 int rc = VINF_SUCCESS;
3403 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3404 {
3405 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3406 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3407 {
3408 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3409
3410 bool fPendingIntr = false;
3411 uint8_t u8Tpr = 0;
3412 uint8_t u8PendingIntr = 0;
3413 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3414 AssertRCReturn(rc, rc);
3415
3416 /*
3417 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3418 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3419 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3420 * the interrupt when we VM-exit for other reasons.
3421 */
3422 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3423 uint32_t u32TprThreshold = 0;
3424 if (fPendingIntr)
3425 {
3426 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3427 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3428 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3429 if (u8PendingPriority <= u8TprPriority)
3430 u32TprThreshold = u8PendingPriority;
3431 else
3432 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3433 }
3434
3435 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3436 AssertRCReturn(rc, rc);
3437 }
3438
3439 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3440 }
3441 return rc;
3442}
3443
3444
3445/**
3446 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3447 *
3448 * @returns Guest's interruptibility-state.
3449 * @param pVCpu The cross context virtual CPU structure.
3450 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3451 * out-of-sync. Make sure to update the required fields
3452 * before using them.
3453 *
3454 * @remarks No-long-jump zone!!!
3455 */
3456DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3457{
3458 /*
3459 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3460 */
3461 uint32_t uIntrState = 0;
3462 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3463 {
3464 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3465 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3466 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3467 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3468 {
3469 if (pMixedCtx->eflags.Bits.u1IF)
3470 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3471 else
3472 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3473 }
3474 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3475 {
3476 /*
3477 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3478 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3479 */
3480 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3481 }
3482 }
3483
3484 /*
3485 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3486 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3487 * setting this would block host-NMIs and IRET will not clear the blocking.
3488 *
3489 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3490 */
3491 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3492 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3493 {
3494 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3495 }
3496
3497 return uIntrState;
3498}
3499
3500
3501/**
3502 * Loads the guest's interruptibility-state into the guest-state area in the
3503 * VMCS.
3504 *
3505 * @returns VBox status code.
3506 * @param pVCpu The cross context virtual CPU structure.
3507 * @param uIntrState The interruptibility-state to set.
3508 */
3509static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3510{
3511 NOREF(pVCpu);
3512 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3513 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3514 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3515 AssertRC(rc);
3516 return rc;
3517}
3518
3519
3520/**
3521 * Loads the exception intercepts required for guest execution in the VMCS.
3522 *
3523 * @returns VBox status code.
3524 * @param pVCpu The cross context virtual CPU structure.
3525 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3526 * out-of-sync. Make sure to update the required fields
3527 * before using them.
3528 */
3529static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3530{
3531 NOREF(pMixedCtx);
3532 int rc = VINF_SUCCESS;
3533 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3534 {
3535 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3536 if (pVCpu->hm.s.fGIMTrapXcptUD)
3537 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3538#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3539 else
3540 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3541#endif
3542
3543 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3544 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3545
3546 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3547 AssertRCReturn(rc, rc);
3548
3549 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3550 Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3551 pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3552 }
3553 return rc;
3554}
3555
3556
3557/**
3558 * Loads the guest's RIP into the guest-state area in the VMCS.
3559 *
3560 * @returns VBox status code.
3561 * @param pVCpu The cross context virtual CPU structure.
3562 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3563 * out-of-sync. Make sure to update the required fields
3564 * before using them.
3565 *
3566 * @remarks No-long-jump zone!!!
3567 */
3568static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3569{
3570 int rc = VINF_SUCCESS;
3571 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3572 {
3573 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3574 AssertRCReturn(rc, rc);
3575
3576 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3577 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3578 HMCPU_CF_VALUE(pVCpu)));
3579 }
3580 return rc;
3581}
3582
3583
3584/**
3585 * Loads the guest's RSP into the guest-state area in the VMCS.
3586 *
3587 * @returns VBox status code.
3588 * @param pVCpu The cross context virtual CPU structure.
3589 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3590 * out-of-sync. Make sure to update the required fields
3591 * before using them.
3592 *
3593 * @remarks No-long-jump zone!!!
3594 */
3595static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3596{
3597 int rc = VINF_SUCCESS;
3598 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3599 {
3600 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3601 AssertRCReturn(rc, rc);
3602
3603 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3604 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3605 }
3606 return rc;
3607}
3608
3609
3610/**
3611 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3612 *
3613 * @returns VBox status code.
3614 * @param pVCpu The cross context virtual CPU structure.
3615 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3616 * out-of-sync. Make sure to update the required fields
3617 * before using them.
3618 *
3619 * @remarks No-long-jump zone!!!
3620 */
3621static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3622{
3623 int rc = VINF_SUCCESS;
3624 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3625 {
3626 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3627 Let us assert it as such and use 32-bit VMWRITE. */
3628 Assert(!(pMixedCtx->rflags.u64 >> 32));
3629 X86EFLAGS Eflags = pMixedCtx->eflags;
3630 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3631 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3632 * These will never be cleared/set, unless some other part of the VMM
3633 * code is buggy - in which case we're better of finding and fixing
3634 * those bugs than hiding them. */
3635 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3636 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3637 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3638 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3639
3640 /*
3641 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3642 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3643 */
3644 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3645 {
3646 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3647 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3648 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3649 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3650 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3651 }
3652
3653 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3654 AssertRCReturn(rc, rc);
3655
3656 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3657 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3658 }
3659 return rc;
3660}
3661
3662
3663/**
3664 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3665 *
3666 * @returns VBox status code.
3667 * @param pVCpu The cross context virtual CPU structure.
3668 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3669 * out-of-sync. Make sure to update the required fields
3670 * before using them.
3671 *
3672 * @remarks No-long-jump zone!!!
3673 */
3674DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3675{
3676 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3677 rc |= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3678 rc |= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3679 AssertRCReturn(rc, rc);
3680 return rc;
3681}
3682
3683
3684/**
3685 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3686 * CR0 is partially shared with the host and we have to consider the FPU bits.
3687 *
3688 * @returns VBox status code.
3689 * @param pVCpu The cross context virtual CPU structure.
3690 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3691 * out-of-sync. Make sure to update the required fields
3692 * before using them.
3693 *
3694 * @remarks No-long-jump zone!!!
3695 */
3696static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3697{
3698 /*
3699 * Guest CR0.
3700 * Guest FPU.
3701 */
3702 int rc = VINF_SUCCESS;
3703 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3704 {
3705 Assert(!(pMixedCtx->cr0 >> 32));
3706 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3707 PVM pVM = pVCpu->CTX_SUFF(pVM);
3708
3709 /* The guest's view (read access) of its CR0 is unblemished. */
3710 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3711 AssertRCReturn(rc, rc);
3712 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3713
3714 /* Setup VT-x's view of the guest CR0. */
3715 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3716 if (pVM->hm.s.fNestedPaging)
3717 {
3718 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3719 {
3720 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3721 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3722 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3723 }
3724 else
3725 {
3726 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3727 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3728 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3729 }
3730
3731 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3732 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3733 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3734
3735 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3736 AssertRCReturn(rc, rc);
3737 }
3738 else
3739 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3740
3741 /*
3742 * Guest FPU bits.
3743 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3744 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3745 */
3746 u32GuestCR0 |= X86_CR0_NE;
3747 bool fInterceptNM = false;
3748 if (CPUMIsGuestFPUStateActive(pVCpu))
3749 {
3750 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3751 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3752 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3753 }
3754 else
3755 {
3756 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3757 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3758 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3759 }
3760
3761 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3762 bool fInterceptMF = false;
3763 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3764 fInterceptMF = true;
3765
3766 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3767 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3768 {
3769 Assert(PDMVmmDevHeapIsEnabled(pVM));
3770 Assert(pVM->hm.s.vmx.pRealModeTSS);
3771 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3772 fInterceptNM = true;
3773 fInterceptMF = true;
3774 }
3775 else
3776 {
3777 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3778 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3779 }
3780 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3781
3782 if (fInterceptNM)
3783 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3784 else
3785 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3786
3787 if (fInterceptMF)
3788 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3789 else
3790 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3791
3792 /* Additional intercepts for debugging, define these yourself explicitly. */
3793#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3794 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3795 | RT_BIT(X86_XCPT_BP)
3796 | RT_BIT(X86_XCPT_DE)
3797 | RT_BIT(X86_XCPT_NM)
3798 | RT_BIT(X86_XCPT_TS)
3799 | RT_BIT(X86_XCPT_UD)
3800 | RT_BIT(X86_XCPT_NP)
3801 | RT_BIT(X86_XCPT_SS)
3802 | RT_BIT(X86_XCPT_GP)
3803 | RT_BIT(X86_XCPT_PF)
3804 | RT_BIT(X86_XCPT_MF)
3805 ;
3806#elif defined(HMVMX_ALWAYS_TRAP_PF)
3807 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3808#endif
3809
3810 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3811
3812 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3813 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3814 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3815 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3816 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3817 else
3818 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3819
3820 u32GuestCR0 |= uSetCR0;
3821 u32GuestCR0 &= uZapCR0;
3822 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3823
3824 /* Write VT-x's view of the guest CR0 into the VMCS. */
3825 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3826 AssertRCReturn(rc, rc);
3827 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3828 uZapCR0));
3829
3830 /*
3831 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3832 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3833 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3834 */
3835 uint32_t u32CR0Mask = 0;
3836 u32CR0Mask = X86_CR0_PE
3837 | X86_CR0_NE
3838 | X86_CR0_WP
3839 | X86_CR0_PG
3840 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3841 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3842 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3843
3844 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3845 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3846 * and @bugref{6944}. */
3847#if 0
3848 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3849 u32CR0Mask &= ~X86_CR0_PE;
3850#endif
3851 if (pVM->hm.s.fNestedPaging)
3852 u32CR0Mask &= ~X86_CR0_WP;
3853
3854 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3855 if (fInterceptNM)
3856 {
3857 u32CR0Mask |= X86_CR0_TS
3858 | X86_CR0_MP;
3859 }
3860
3861 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3862 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3863 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3864 AssertRCReturn(rc, rc);
3865 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3866
3867 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3868 }
3869 return rc;
3870}
3871
3872
3873/**
3874 * Loads the guest control registers (CR3, CR4) into the guest-state area
3875 * in the VMCS.
3876 *
3877 * @returns VBox strict status code.
3878 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3879 * without unrestricted guest access and the VMMDev is not presently
3880 * mapped (e.g. EFI32).
3881 *
3882 * @param pVCpu The cross context virtual CPU structure.
3883 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3884 * out-of-sync. Make sure to update the required fields
3885 * before using them.
3886 *
3887 * @remarks No-long-jump zone!!!
3888 */
3889static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3890{
3891 int rc = VINF_SUCCESS;
3892 PVM pVM = pVCpu->CTX_SUFF(pVM);
3893
3894 /*
3895 * Guest CR2.
3896 * It's always loaded in the assembler code. Nothing to do here.
3897 */
3898
3899 /*
3900 * Guest CR3.
3901 */
3902 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3903 {
3904 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3905 if (pVM->hm.s.fNestedPaging)
3906 {
3907 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3908
3909 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3910 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3911 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3912 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3913
3914 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3915 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3916 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3917
3918 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3919 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3920 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3921 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3922 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3923 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3924 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3925
3926 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3927 AssertRCReturn(rc, rc);
3928 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3929
3930 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3931 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3932 {
3933 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3934 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3935 {
3936 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3937 AssertRCReturn(rc, rc);
3938 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3939 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3940 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3941 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3942 AssertRCReturn(rc, rc);
3943 }
3944
3945 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3946 have Unrestricted Execution to handle the guest when it's not using paging. */
3947 GCPhysGuestCR3 = pMixedCtx->cr3;
3948 }
3949 else
3950 {
3951 /*
3952 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3953 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3954 * EPT takes care of translating it to host-physical addresses.
3955 */
3956 RTGCPHYS GCPhys;
3957 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3958
3959 /* We obtain it here every time as the guest could have relocated this PCI region. */
3960 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3961 if (RT_SUCCESS(rc))
3962 { /* likely */ }
3963 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3964 {
3965 Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
3966 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3967 }
3968 else
3969 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3970
3971 GCPhysGuestCR3 = GCPhys;
3972 }
3973
3974 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3975 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3976 }
3977 else
3978 {
3979 /* Non-nested paging case, just use the hypervisor's CR3. */
3980 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3981
3982 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
3983 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3984 }
3985 AssertRCReturn(rc, rc);
3986
3987 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3988 }
3989
3990 /*
3991 * Guest CR4.
3992 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3993 */
3994 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3995 {
3996 Assert(!(pMixedCtx->cr4 >> 32));
3997 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3998
3999 /* The guest's view of its CR4 is unblemished. */
4000 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4001 AssertRCReturn(rc, rc);
4002 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4003
4004 /* Setup VT-x's view of the guest CR4. */
4005 /*
4006 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4007 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4008 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4009 */
4010 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4011 {
4012 Assert(pVM->hm.s.vmx.pRealModeTSS);
4013 Assert(PDMVmmDevHeapIsEnabled(pVM));
4014 u32GuestCR4 &= ~X86_CR4_VME;
4015 }
4016
4017 if (pVM->hm.s.fNestedPaging)
4018 {
4019 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4020 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4021 {
4022 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4023 u32GuestCR4 |= X86_CR4_PSE;
4024 /* Our identity mapping is a 32-bit page directory. */
4025 u32GuestCR4 &= ~X86_CR4_PAE;
4026 }
4027 /* else use guest CR4.*/
4028 }
4029 else
4030 {
4031 /*
4032 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4033 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4034 */
4035 switch (pVCpu->hm.s.enmShadowMode)
4036 {
4037 case PGMMODE_REAL: /* Real-mode. */
4038 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4039 case PGMMODE_32_BIT: /* 32-bit paging. */
4040 {
4041 u32GuestCR4 &= ~X86_CR4_PAE;
4042 break;
4043 }
4044
4045 case PGMMODE_PAE: /* PAE paging. */
4046 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4047 {
4048 u32GuestCR4 |= X86_CR4_PAE;
4049 break;
4050 }
4051
4052 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4053 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4054#ifdef VBOX_ENABLE_64_BITS_GUESTS
4055 break;
4056#endif
4057 default:
4058 AssertFailed();
4059 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4060 }
4061 }
4062
4063 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4064 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4065 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4066 u32GuestCR4 |= uSetCR4;
4067 u32GuestCR4 &= uZapCR4;
4068
4069 /* Write VT-x's view of the guest CR4 into the VMCS. */
4070 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4071 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4072 AssertRCReturn(rc, rc);
4073
4074 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4075 uint32_t u32CR4Mask = X86_CR4_VME
4076 | X86_CR4_PAE
4077 | X86_CR4_PGE
4078 | X86_CR4_PSE
4079 | X86_CR4_VMXE;
4080 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4081 u32CR4Mask |= X86_CR4_OSXSAVE;
4082 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4083 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4084 AssertRCReturn(rc, rc);
4085
4086 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4087 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4088
4089 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4090 }
4091 return rc;
4092}
4093
4094
4095/**
4096 * Loads the guest debug registers into the guest-state area in the VMCS.
4097 *
4098 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4099 *
4100 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4101 *
4102 * @returns VBox status code.
4103 * @param pVCpu The cross context virtual CPU structure.
4104 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4105 * out-of-sync. Make sure to update the required fields
4106 * before using them.
4107 *
4108 * @remarks No-long-jump zone!!!
4109 */
4110static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4111{
4112 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4113 return VINF_SUCCESS;
4114
4115#ifdef VBOX_STRICT
4116 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4117 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4118 {
4119 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4120 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4121 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4122 }
4123#endif
4124
4125 int rc;
4126 PVM pVM = pVCpu->CTX_SUFF(pVM);
4127 bool fSteppingDB = false;
4128 bool fInterceptMovDRx = false;
4129 if (pVCpu->hm.s.fSingleInstruction)
4130 {
4131 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4132 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4133 {
4134 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4135 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4136 AssertRCReturn(rc, rc);
4137 Assert(fSteppingDB == false);
4138 }
4139 else
4140 {
4141 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4142 pVCpu->hm.s.fClearTrapFlag = true;
4143 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4144 fSteppingDB = true;
4145 }
4146 }
4147
4148 if ( fSteppingDB
4149 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4150 {
4151 /*
4152 * Use the combined guest and host DRx values found in the hypervisor
4153 * register set because the debugger has breakpoints active or someone
4154 * is single stepping on the host side without a monitor trap flag.
4155 *
4156 * Note! DBGF expects a clean DR6 state before executing guest code.
4157 */
4158#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4159 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4160 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4161 {
4162 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4163 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4164 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4165 }
4166 else
4167#endif
4168 if (!CPUMIsHyperDebugStateActive(pVCpu))
4169 {
4170 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4171 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4172 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4173 }
4174
4175 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4176 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4177 AssertRCReturn(rc, rc);
4178
4179 pVCpu->hm.s.fUsingHyperDR7 = true;
4180 fInterceptMovDRx = true;
4181 }
4182 else
4183 {
4184 /*
4185 * If the guest has enabled debug registers, we need to load them prior to
4186 * executing guest code so they'll trigger at the right time.
4187 */
4188 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4189 {
4190#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4191 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4192 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4193 {
4194 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4195 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4196 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4197 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4198 }
4199 else
4200#endif
4201 if (!CPUMIsGuestDebugStateActive(pVCpu))
4202 {
4203 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4204 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4205 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4206 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4207 }
4208 Assert(!fInterceptMovDRx);
4209 }
4210 /*
4211 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4212 * must intercept #DB in order to maintain a correct DR6 guest value, and
4213 * because we need to intercept it to prevent nested #DBs from hanging the
4214 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4215 */
4216#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4217 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4218 && !CPUMIsGuestDebugStateActive(pVCpu))
4219#else
4220 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4221#endif
4222 {
4223 fInterceptMovDRx = true;
4224 }
4225
4226 /* Update guest DR7. */
4227 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4228 AssertRCReturn(rc, rc);
4229
4230 pVCpu->hm.s.fUsingHyperDR7 = false;
4231 }
4232
4233 /*
4234 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4235 */
4236 if (fInterceptMovDRx)
4237 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4238 else
4239 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4240 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4241 AssertRCReturn(rc, rc);
4242
4243 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4244 return VINF_SUCCESS;
4245}
4246
4247
4248#ifdef VBOX_STRICT
4249/**
4250 * Strict function to validate segment registers.
4251 *
4252 * @remarks ASSUMES CR0 is up to date.
4253 */
4254static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4255{
4256 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4257 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4258 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4259 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4260 && ( !CPUMIsGuestInRealModeEx(pCtx)
4261 && !CPUMIsGuestInV86ModeEx(pCtx)))
4262 {
4263 /* Protected mode checks */
4264 /* CS */
4265 Assert(pCtx->cs.Attr.n.u1Present);
4266 Assert(!(pCtx->cs.Attr.u & 0xf00));
4267 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4268 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4269 || !(pCtx->cs.Attr.n.u1Granularity));
4270 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4271 || (pCtx->cs.Attr.n.u1Granularity));
4272 /* CS cannot be loaded with NULL in protected mode. */
4273 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4274 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4275 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4276 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4277 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4278 else
4279 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4280 /* SS */
4281 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4282 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4283 if ( !(pCtx->cr0 & X86_CR0_PE)
4284 || pCtx->cs.Attr.n.u4Type == 3)
4285 {
4286 Assert(!pCtx->ss.Attr.n.u2Dpl);
4287 }
4288 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4289 {
4290 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4291 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4292 Assert(pCtx->ss.Attr.n.u1Present);
4293 Assert(!(pCtx->ss.Attr.u & 0xf00));
4294 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4295 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4296 || !(pCtx->ss.Attr.n.u1Granularity));
4297 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4298 || (pCtx->ss.Attr.n.u1Granularity));
4299 }
4300 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4301 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4302 {
4303 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4304 Assert(pCtx->ds.Attr.n.u1Present);
4305 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4306 Assert(!(pCtx->ds.Attr.u & 0xf00));
4307 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4308 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4309 || !(pCtx->ds.Attr.n.u1Granularity));
4310 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4311 || (pCtx->ds.Attr.n.u1Granularity));
4312 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4313 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4314 }
4315 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4316 {
4317 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4318 Assert(pCtx->es.Attr.n.u1Present);
4319 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4320 Assert(!(pCtx->es.Attr.u & 0xf00));
4321 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4322 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4323 || !(pCtx->es.Attr.n.u1Granularity));
4324 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4325 || (pCtx->es.Attr.n.u1Granularity));
4326 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4327 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4328 }
4329 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4330 {
4331 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4332 Assert(pCtx->fs.Attr.n.u1Present);
4333 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4334 Assert(!(pCtx->fs.Attr.u & 0xf00));
4335 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4336 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4337 || !(pCtx->fs.Attr.n.u1Granularity));
4338 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4339 || (pCtx->fs.Attr.n.u1Granularity));
4340 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4341 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4342 }
4343 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4344 {
4345 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4346 Assert(pCtx->gs.Attr.n.u1Present);
4347 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4348 Assert(!(pCtx->gs.Attr.u & 0xf00));
4349 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4350 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4351 || !(pCtx->gs.Attr.n.u1Granularity));
4352 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4353 || (pCtx->gs.Attr.n.u1Granularity));
4354 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4355 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4356 }
4357 /* 64-bit capable CPUs. */
4358# if HC_ARCH_BITS == 64
4359 Assert(!(pCtx->cs.u64Base >> 32));
4360 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4361 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4362 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4363# endif
4364 }
4365 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4366 || ( CPUMIsGuestInRealModeEx(pCtx)
4367 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4368 {
4369 /* Real and v86 mode checks. */
4370 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4371 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4372 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4373 {
4374 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4375 }
4376 else
4377 {
4378 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4379 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4380 }
4381
4382 /* CS */
4383 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4384 Assert(pCtx->cs.u32Limit == 0xffff);
4385 Assert(u32CSAttr == 0xf3);
4386 /* SS */
4387 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4388 Assert(pCtx->ss.u32Limit == 0xffff);
4389 Assert(u32SSAttr == 0xf3);
4390 /* DS */
4391 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4392 Assert(pCtx->ds.u32Limit == 0xffff);
4393 Assert(u32DSAttr == 0xf3);
4394 /* ES */
4395 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4396 Assert(pCtx->es.u32Limit == 0xffff);
4397 Assert(u32ESAttr == 0xf3);
4398 /* FS */
4399 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4400 Assert(pCtx->fs.u32Limit == 0xffff);
4401 Assert(u32FSAttr == 0xf3);
4402 /* GS */
4403 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4404 Assert(pCtx->gs.u32Limit == 0xffff);
4405 Assert(u32GSAttr == 0xf3);
4406 /* 64-bit capable CPUs. */
4407# if HC_ARCH_BITS == 64
4408 Assert(!(pCtx->cs.u64Base >> 32));
4409 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4410 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4411 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4412# endif
4413 }
4414}
4415#endif /* VBOX_STRICT */
4416
4417
4418/**
4419 * Writes a guest segment register into the guest-state area in the VMCS.
4420 *
4421 * @returns VBox status code.
4422 * @param pVCpu The cross context virtual CPU structure.
4423 * @param idxSel Index of the selector in the VMCS.
4424 * @param idxLimit Index of the segment limit in the VMCS.
4425 * @param idxBase Index of the segment base in the VMCS.
4426 * @param idxAccess Index of the access rights of the segment in the VMCS.
4427 * @param pSelReg Pointer to the segment selector.
4428 *
4429 * @remarks No-long-jump zone!!!
4430 */
4431static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4432 uint32_t idxAccess, PCPUMSELREG pSelReg)
4433{
4434 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4435 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4436 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4437 AssertRCReturn(rc, rc);
4438
4439 uint32_t u32Access = pSelReg->Attr.u;
4440 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4441 {
4442 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4443 u32Access = 0xf3;
4444 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4445 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4446 }
4447 else
4448 {
4449 /*
4450 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4451 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4452 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4453 * loaded in protected-mode have their attribute as 0.
4454 */
4455 if (!u32Access)
4456 u32Access = X86DESCATTR_UNUSABLE;
4457 }
4458
4459 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4460 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4461 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4462
4463 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4464 AssertRCReturn(rc, rc);
4465 return rc;
4466}
4467
4468
4469/**
4470 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4471 * into the guest-state area in the VMCS.
4472 *
4473 * @returns VBox status code.
4474 * @param pVCpu The cross context virtual CPU structure.
4475 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4476 * out-of-sync. Make sure to update the required fields
4477 * before using them.
4478 *
4479 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4480 * @remarks No-long-jump zone!!!
4481 */
4482static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4483{
4484 int rc = VERR_INTERNAL_ERROR_5;
4485 PVM pVM = pVCpu->CTX_SUFF(pVM);
4486
4487 /*
4488 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4489 */
4490 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4491 {
4492 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4493 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4494 {
4495 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4496 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4497 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4498 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4499 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4500 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4501 }
4502
4503#ifdef VBOX_WITH_REM
4504 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4505 {
4506 Assert(pVM->hm.s.vmx.pRealModeTSS);
4507 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4508 if ( pVCpu->hm.s.vmx.fWasInRealMode
4509 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4510 {
4511 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4512 in real-mode (e.g. OpenBSD 4.0) */
4513 REMFlushTBs(pVM);
4514 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4515 pVCpu->hm.s.vmx.fWasInRealMode = false;
4516 }
4517 }
4518#endif
4519 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4520 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4521 AssertRCReturn(rc, rc);
4522 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4523 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4524 AssertRCReturn(rc, rc);
4525 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4526 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4527 AssertRCReturn(rc, rc);
4528 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4529 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4530 AssertRCReturn(rc, rc);
4531 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4532 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4533 AssertRCReturn(rc, rc);
4534 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4535 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4536 AssertRCReturn(rc, rc);
4537
4538#ifdef VBOX_STRICT
4539 /* Validate. */
4540 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4541#endif
4542
4543 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4544 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4545 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4546 }
4547
4548 /*
4549 * Guest TR.
4550 */
4551 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4552 {
4553 /*
4554 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4555 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4556 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4557 */
4558 uint16_t u16Sel = 0;
4559 uint32_t u32Limit = 0;
4560 uint64_t u64Base = 0;
4561 uint32_t u32AccessRights = 0;
4562
4563 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4564 {
4565 u16Sel = pMixedCtx->tr.Sel;
4566 u32Limit = pMixedCtx->tr.u32Limit;
4567 u64Base = pMixedCtx->tr.u64Base;
4568 u32AccessRights = pMixedCtx->tr.Attr.u;
4569 }
4570 else
4571 {
4572 Assert(pVM->hm.s.vmx.pRealModeTSS);
4573 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4574
4575 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4576 RTGCPHYS GCPhys;
4577 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4578 AssertRCReturn(rc, rc);
4579
4580 X86DESCATTR DescAttr;
4581 DescAttr.u = 0;
4582 DescAttr.n.u1Present = 1;
4583 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4584
4585 u16Sel = 0;
4586 u32Limit = HM_VTX_TSS_SIZE;
4587 u64Base = GCPhys; /* in real-mode phys = virt. */
4588 u32AccessRights = DescAttr.u;
4589 }
4590
4591 /* Validate. */
4592 Assert(!(u16Sel & RT_BIT(2)));
4593 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4594 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4595 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4596 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4597 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4598 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4599 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4600 Assert( (u32Limit & 0xfff) == 0xfff
4601 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4602 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4603 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4604
4605 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4606 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4607 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4608 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4609 AssertRCReturn(rc, rc);
4610
4611 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4612 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4613 }
4614
4615 /*
4616 * Guest GDTR.
4617 */
4618 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4619 {
4620 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4621 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4622 AssertRCReturn(rc, rc);
4623
4624 /* Validate. */
4625 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4626
4627 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4628 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4629 }
4630
4631 /*
4632 * Guest LDTR.
4633 */
4634 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4635 {
4636 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4637 uint32_t u32Access = 0;
4638 if (!pMixedCtx->ldtr.Attr.u)
4639 u32Access = X86DESCATTR_UNUSABLE;
4640 else
4641 u32Access = pMixedCtx->ldtr.Attr.u;
4642
4643 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4644 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4645 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4646 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4647 AssertRCReturn(rc, rc);
4648
4649 /* Validate. */
4650 if (!(u32Access & X86DESCATTR_UNUSABLE))
4651 {
4652 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4653 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4654 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4655 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4656 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4657 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4658 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4659 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4660 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4661 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4662 }
4663
4664 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4665 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4666 }
4667
4668 /*
4669 * Guest IDTR.
4670 */
4671 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4672 {
4673 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4674 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4675 AssertRCReturn(rc, rc);
4676
4677 /* Validate. */
4678 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4679
4680 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4681 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4682 }
4683
4684 return VINF_SUCCESS;
4685}
4686
4687
4688/**
4689 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4690 * areas.
4691 *
4692 * These MSRs will automatically be loaded to the host CPU on every successful
4693 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4694 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4695 * -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4696 *
4697 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4698 *
4699 * @returns VBox status code.
4700 * @param pVCpu The cross context virtual CPU structure.
4701 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4702 * out-of-sync. Make sure to update the required fields
4703 * before using them.
4704 *
4705 * @remarks No-long-jump zone!!!
4706 */
4707static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4708{
4709 AssertPtr(pVCpu);
4710 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4711
4712 /*
4713 * MSRs that we use the auto-load/store MSR area in the VMCS.
4714 */
4715 PVM pVM = pVCpu->CTX_SUFF(pVM);
4716 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4717 {
4718 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4719#if HC_ARCH_BITS == 32
4720 if (pVM->hm.s.fAllow64BitGuests)
4721 {
4722 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4723 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4724 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4725 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4726 AssertRCReturn(rc, rc);
4727# ifdef LOG_ENABLED
4728 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4729 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4730 {
4731 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4732 pMsr->u64Value));
4733 }
4734# endif
4735 }
4736#endif
4737 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4738 }
4739
4740 /*
4741 * Guest Sysenter MSRs.
4742 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4743 * VM-exits on WRMSRs for these MSRs.
4744 */
4745 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4746 {
4747 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4748 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4749 }
4750
4751 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4752 {
4753 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4754 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4755 }
4756
4757 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4758 {
4759 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4760 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4761 }
4762
4763 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4764 {
4765 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4766 {
4767 /*
4768 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4769 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4770 */
4771 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4772 {
4773 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4774 AssertRCReturn(rc,rc);
4775 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4776 }
4777 else
4778 {
4779 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4780 NULL /* pfAddedAndUpdated */);
4781 AssertRCReturn(rc, rc);
4782
4783 /* We need to intercept reads too, see @bugref{7386#c16}. */
4784 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4785 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4786 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4787 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4788 }
4789 }
4790 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4791 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4792 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4793 }
4794
4795 return VINF_SUCCESS;
4796}
4797
4798
4799/**
4800 * Loads the guest activity state into the guest-state area in the VMCS.
4801 *
4802 * @returns VBox status code.
4803 * @param pVCpu The cross context virtual CPU structure.
4804 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4805 * out-of-sync. Make sure to update the required fields
4806 * before using them.
4807 *
4808 * @remarks No-long-jump zone!!!
4809 */
4810static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4811{
4812 NOREF(pMixedCtx);
4813 /** @todo See if we can make use of other states, e.g.
4814 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4815 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4816 {
4817 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4818 AssertRCReturn(rc, rc);
4819
4820 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4821 }
4822 return VINF_SUCCESS;
4823}
4824
4825
4826/**
4827 * Sets up the appropriate function to run guest code.
4828 *
4829 * @returns VBox status code.
4830 * @param pVCpu The cross context virtual CPU structure.
4831 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4832 * out-of-sync. Make sure to update the required fields
4833 * before using them.
4834 *
4835 * @remarks No-long-jump zone!!!
4836 */
4837static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4838{
4839 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4840 {
4841#ifndef VBOX_ENABLE_64_BITS_GUESTS
4842 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4843#endif
4844 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4845#if HC_ARCH_BITS == 32
4846 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4847 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4848 {
4849 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4850 {
4851 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4852 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4853 | HM_CHANGED_VMX_ENTRY_CTLS
4854 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4855 }
4856 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4857
4858 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4859 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4860 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4861 }
4862#else
4863 /* 64-bit host. */
4864 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4865#endif
4866 }
4867 else
4868 {
4869 /* Guest is not in long mode, use the 32-bit handler. */
4870#if HC_ARCH_BITS == 32
4871 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4872 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4873 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4874 {
4875 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4876 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4877 | HM_CHANGED_VMX_ENTRY_CTLS
4878 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4879 }
4880# ifdef VBOX_ENABLE_64_BITS_GUESTS
4881 /* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel design. See @bugref{8432#c7}.
4882 * Except if Real-on-V86 is active, clear the 64-bit switcher flag because now we know the guest is in a sane
4883 * state where it's safe to use the 32-bit switcher again.
4884 */
4885 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4886 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4887
4888 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4889 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4890 else
4891 {
4892 Assert(!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active);
4893 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4894 }
4895# else
4896 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4897# endif
4898#else
4899 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4900#endif
4901 }
4902 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4903 return VINF_SUCCESS;
4904}
4905
4906
4907/**
4908 * Wrapper for running the guest code in VT-x.
4909 *
4910 * @returns VBox status code, no informational status codes.
4911 * @param pVM The cross context VM structure.
4912 * @param pVCpu The cross context virtual CPU structure.
4913 * @param pCtx Pointer to the guest-CPU context.
4914 *
4915 * @remarks No-long-jump zone!!!
4916 */
4917DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4918{
4919 /*
4920 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4921 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4922 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4923 */
4924 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4925 /** @todo Add stats for resume vs launch. */
4926#ifdef VBOX_WITH_KERNEL_USING_XMM
4927 int rc = HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4928#else
4929 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4930#endif
4931 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4932 return rc;
4933}
4934
4935
4936/**
4937 * Reports world-switch error and dumps some useful debug info.
4938 *
4939 * @param pVM The cross context VM structure.
4940 * @param pVCpu The cross context virtual CPU structure.
4941 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4942 * @param pCtx Pointer to the guest-CPU context.
4943 * @param pVmxTransient Pointer to the VMX transient structure (only
4944 * exitReason updated).
4945 */
4946static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4947{
4948 Assert(pVM);
4949 Assert(pVCpu);
4950 Assert(pCtx);
4951 Assert(pVmxTransient);
4952 HMVMX_ASSERT_PREEMPT_SAFE();
4953
4954 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4955 switch (rcVMRun)
4956 {
4957 case VERR_VMX_INVALID_VMXON_PTR:
4958 AssertFailed();
4959 break;
4960 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4961 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4962 {
4963 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4964 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4965 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4966 AssertRC(rc);
4967
4968 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4969 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4970 Cannot do it here as we may have been long preempted. */
4971
4972#ifdef VBOX_STRICT
4973 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4974 pVmxTransient->uExitReason));
4975 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4976 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4977 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4978 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4979 else
4980 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4981 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4982 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4983
4984 /* VMX control bits. */
4985 uint32_t u32Val;
4986 uint64_t u64Val;
4987 RTHCUINTREG uHCReg;
4988 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4989 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4990 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4991 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4992 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
4993 {
4994 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4995 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4996 }
4997 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4998 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4999 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5000 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5001 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5002 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5003 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5004 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5005 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5006 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5007 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5008 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5009 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5010 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5011 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5012 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5013 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5014 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5015 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5016 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5017 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5018 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5019 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5020 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5021 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5022 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5023 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5024 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5025 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5026 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5027 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5028 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5029 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5030 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5031 if (pVM->hm.s.fNestedPaging)
5032 {
5033 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5034 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5035 }
5036
5037 /* Guest bits. */
5038 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5039 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5040 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5041 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5042 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5043 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5044 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5045 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5046
5047 /* Host bits. */
5048 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5049 Log4(("Host CR0 %#RHr\n", uHCReg));
5050 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5051 Log4(("Host CR3 %#RHr\n", uHCReg));
5052 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5053 Log4(("Host CR4 %#RHr\n", uHCReg));
5054
5055 RTGDTR HostGdtr;
5056 PCX86DESCHC pDesc;
5057 ASMGetGDTR(&HostGdtr);
5058 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5059 Log4(("Host CS %#08x\n", u32Val));
5060 if (u32Val < HostGdtr.cbGdt)
5061 {
5062 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5063 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5064 }
5065
5066 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5067 Log4(("Host DS %#08x\n", u32Val));
5068 if (u32Val < HostGdtr.cbGdt)
5069 {
5070 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5071 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5072 }
5073
5074 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5075 Log4(("Host ES %#08x\n", u32Val));
5076 if (u32Val < HostGdtr.cbGdt)
5077 {
5078 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5079 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5080 }
5081
5082 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5083 Log4(("Host FS %#08x\n", u32Val));
5084 if (u32Val < HostGdtr.cbGdt)
5085 {
5086 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5087 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5088 }
5089
5090 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5091 Log4(("Host GS %#08x\n", u32Val));
5092 if (u32Val < HostGdtr.cbGdt)
5093 {
5094 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5095 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5096 }
5097
5098 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5099 Log4(("Host SS %#08x\n", u32Val));
5100 if (u32Val < HostGdtr.cbGdt)
5101 {
5102 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5103 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5104 }
5105
5106 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5107 Log4(("Host TR %#08x\n", u32Val));
5108 if (u32Val < HostGdtr.cbGdt)
5109 {
5110 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5111 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5112 }
5113
5114 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5115 Log4(("Host TR Base %#RHv\n", uHCReg));
5116 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5117 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5118 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5119 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5120 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5121 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5122 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5123 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5124 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5125 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5126 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5127 Log4(("Host RSP %#RHv\n", uHCReg));
5128 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5129 Log4(("Host RIP %#RHv\n", uHCReg));
5130# if HC_ARCH_BITS == 64
5131 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5132 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5133 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5134 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5135 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5136 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5137# endif
5138#endif /* VBOX_STRICT */
5139 break;
5140 }
5141
5142 default:
5143 /* Impossible */
5144 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5145 break;
5146 }
5147 NOREF(pVM); NOREF(pCtx);
5148}
5149
5150
5151#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5152#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5153# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5154#endif
5155#ifdef VBOX_STRICT
5156static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5157{
5158 switch (idxField)
5159 {
5160 case VMX_VMCS_GUEST_RIP:
5161 case VMX_VMCS_GUEST_RSP:
5162 case VMX_VMCS_GUEST_SYSENTER_EIP:
5163 case VMX_VMCS_GUEST_SYSENTER_ESP:
5164 case VMX_VMCS_GUEST_GDTR_BASE:
5165 case VMX_VMCS_GUEST_IDTR_BASE:
5166 case VMX_VMCS_GUEST_CS_BASE:
5167 case VMX_VMCS_GUEST_DS_BASE:
5168 case VMX_VMCS_GUEST_ES_BASE:
5169 case VMX_VMCS_GUEST_FS_BASE:
5170 case VMX_VMCS_GUEST_GS_BASE:
5171 case VMX_VMCS_GUEST_SS_BASE:
5172 case VMX_VMCS_GUEST_LDTR_BASE:
5173 case VMX_VMCS_GUEST_TR_BASE:
5174 case VMX_VMCS_GUEST_CR3:
5175 return true;
5176 }
5177 return false;
5178}
5179
5180static bool hmR0VmxIsValidReadField(uint32_t idxField)
5181{
5182 switch (idxField)
5183 {
5184 /* Read-only fields. */
5185 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5186 return true;
5187 }
5188 /* Remaining readable fields should also be writable. */
5189 return hmR0VmxIsValidWriteField(idxField);
5190}
5191#endif /* VBOX_STRICT */
5192
5193
5194/**
5195 * Executes the specified handler in 64-bit mode.
5196 *
5197 * @returns VBox status code (no informational status codes).
5198 * @param pVM The cross context VM structure.
5199 * @param pVCpu The cross context virtual CPU structure.
5200 * @param pCtx Pointer to the guest CPU context.
5201 * @param enmOp The operation to perform.
5202 * @param cParams Number of parameters.
5203 * @param paParam Array of 32-bit parameters.
5204 */
5205VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5206 uint32_t cParams, uint32_t *paParam)
5207{
5208 NOREF(pCtx);
5209
5210 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5211 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5212 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5213 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5214
5215#ifdef VBOX_STRICT
5216 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5217 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5218
5219 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5220 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5221#endif
5222
5223 /* Disable interrupts. */
5224 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5225
5226#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5227 RTCPUID idHostCpu = RTMpCpuId();
5228 CPUMR0SetLApic(pVCpu, idHostCpu);
5229#endif
5230
5231 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5232 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5233
5234 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5235 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5236
5237 /* Leave VMX Root Mode. */
5238 VMXDisable();
5239
5240 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5241
5242 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5243 CPUMSetHyperEIP(pVCpu, enmOp);
5244 for (int i = (int)cParams - 1; i >= 0; i--)
5245 CPUMPushHyper(pVCpu, paParam[i]);
5246
5247 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5248
5249 /* Call the switcher. */
5250 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5251 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5252
5253 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5254 /* Make sure the VMX instructions don't cause #UD faults. */
5255 SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
5256
5257 /* Re-enter VMX Root Mode */
5258 int rc2 = VMXEnable(HCPhysCpuPage);
5259 if (RT_FAILURE(rc2))
5260 {
5261 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5262 ASMSetFlags(fOldEFlags);
5263 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5264 return rc2;
5265 }
5266
5267 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5268 AssertRC(rc2);
5269 Assert(!(ASMGetFlags() & X86_EFL_IF));
5270 ASMSetFlags(fOldEFlags);
5271 return rc;
5272}
5273
5274
5275/**
5276 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5277 * supporting 64-bit guests.
5278 *
5279 * @returns VBox status code.
5280 * @param fResume Whether to VMLAUNCH or VMRESUME.
5281 * @param pCtx Pointer to the guest-CPU context.
5282 * @param pCache Pointer to the VMCS cache.
5283 * @param pVM The cross context VM structure.
5284 * @param pVCpu The cross context virtual CPU structure.
5285 */
5286DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5287{
5288 NOREF(fResume);
5289
5290 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5291 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5292
5293#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5294 pCache->uPos = 1;
5295 pCache->interPD = PGMGetInterPaeCR3(pVM);
5296 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5297#endif
5298
5299#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5300 pCache->TestIn.HCPhysCpuPage = 0;
5301 pCache->TestIn.HCPhysVmcs = 0;
5302 pCache->TestIn.pCache = 0;
5303 pCache->TestOut.HCPhysVmcs = 0;
5304 pCache->TestOut.pCache = 0;
5305 pCache->TestOut.pCtx = 0;
5306 pCache->TestOut.eflags = 0;
5307#else
5308 NOREF(pCache);
5309#endif
5310
5311 uint32_t aParam[10];
5312 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5313 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5314 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5315 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5316 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5317 aParam[5] = 0;
5318 aParam[6] = VM_RC_ADDR(pVM, pVM);
5319 aParam[7] = 0;
5320 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5321 aParam[9] = 0;
5322
5323#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5324 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5325 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5326#endif
5327 int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5328
5329#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5330 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5331 Assert(pCtx->dr[4] == 10);
5332 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5333#endif
5334
5335#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5336 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5337 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5338 pVCpu->hm.s.vmx.HCPhysVmcs));
5339 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5340 pCache->TestOut.HCPhysVmcs));
5341 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5342 pCache->TestOut.pCache));
5343 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5344 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5345 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5346 pCache->TestOut.pCtx));
5347 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5348#endif
5349 return rc;
5350}
5351
5352
5353/**
5354 * Initialize the VMCS-Read cache.
5355 *
5356 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5357 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5358 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5359 * (those that have a 32-bit FULL & HIGH part).
5360 *
5361 * @returns VBox status code.
5362 * @param pVM The cross context VM structure.
5363 * @param pVCpu The cross context virtual CPU structure.
5364 */
5365static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5366{
5367#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5368{ \
5369 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5370 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5371 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5372 ++cReadFields; \
5373}
5374
5375 AssertPtr(pVM);
5376 AssertPtr(pVCpu);
5377 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5378 uint32_t cReadFields = 0;
5379
5380 /*
5381 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5382 * and serve to indicate exceptions to the rules.
5383 */
5384
5385 /* Guest-natural selector base fields. */
5386#if 0
5387 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5388 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5389 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5390#endif
5391 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5392 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5393 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5394 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5395 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5396 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5397 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5398 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5399 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5400 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5401 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5402 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5403#if 0
5404 /* Unused natural width guest-state fields. */
5405 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5406 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5407#endif
5408 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5409 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5410
5411 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5412#if 0
5413 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5414 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5415 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5416 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5417 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5418 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5419 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5420 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5421 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5422#endif
5423
5424 /* Natural width guest-state fields. */
5425 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5426#if 0
5427 /* Currently unused field. */
5428 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5429#endif
5430
5431 if (pVM->hm.s.fNestedPaging)
5432 {
5433 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5434 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5435 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5436 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5437 }
5438 else
5439 {
5440 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5441 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5442 }
5443
5444#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5445 return VINF_SUCCESS;
5446}
5447
5448
5449/**
5450 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5451 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5452 * darwin, running 64-bit guests).
5453 *
5454 * @returns VBox status code.
5455 * @param pVCpu The cross context virtual CPU structure.
5456 * @param idxField The VMCS field encoding.
5457 * @param u64Val 16, 32 or 64-bit value.
5458 */
5459VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5460{
5461 int rc;
5462 switch (idxField)
5463 {
5464 /*
5465 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5466 */
5467 /* 64-bit Control fields. */
5468 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5469 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5470 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5471 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5472 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5473 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5474 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5475 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5476 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5477 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5478 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5479 case VMX_VMCS64_CTRL_EPTP_FULL:
5480 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5481 /* 64-bit Guest-state fields. */
5482 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5483 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5484 case VMX_VMCS64_GUEST_PAT_FULL:
5485 case VMX_VMCS64_GUEST_EFER_FULL:
5486 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5487 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5488 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5489 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5490 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5491 /* 64-bit Host-state fields. */
5492 case VMX_VMCS64_HOST_PAT_FULL:
5493 case VMX_VMCS64_HOST_EFER_FULL:
5494 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5495 {
5496 rc = VMXWriteVmcs32(idxField, u64Val);
5497 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5498 break;
5499 }
5500
5501 /*
5502 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5503 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5504 */
5505 /* Natural-width Guest-state fields. */
5506 case VMX_VMCS_GUEST_CR3:
5507 case VMX_VMCS_GUEST_ES_BASE:
5508 case VMX_VMCS_GUEST_CS_BASE:
5509 case VMX_VMCS_GUEST_SS_BASE:
5510 case VMX_VMCS_GUEST_DS_BASE:
5511 case VMX_VMCS_GUEST_FS_BASE:
5512 case VMX_VMCS_GUEST_GS_BASE:
5513 case VMX_VMCS_GUEST_LDTR_BASE:
5514 case VMX_VMCS_GUEST_TR_BASE:
5515 case VMX_VMCS_GUEST_GDTR_BASE:
5516 case VMX_VMCS_GUEST_IDTR_BASE:
5517 case VMX_VMCS_GUEST_RSP:
5518 case VMX_VMCS_GUEST_RIP:
5519 case VMX_VMCS_GUEST_SYSENTER_ESP:
5520 case VMX_VMCS_GUEST_SYSENTER_EIP:
5521 {
5522 if (!(u64Val >> 32))
5523 {
5524 /* If this field is 64-bit, VT-x will zero out the top bits. */
5525 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5526 }
5527 else
5528 {
5529 /* Assert that only the 32->64 switcher case should ever come here. */
5530 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5531 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5532 }
5533 break;
5534 }
5535
5536 default:
5537 {
5538 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5539 rc = VERR_INVALID_PARAMETER;
5540 break;
5541 }
5542 }
5543 AssertRCReturn(rc, rc);
5544 return rc;
5545}
5546
5547
5548/**
5549 * Queue up a VMWRITE by using the VMCS write cache.
5550 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5551 *
5552 * @param pVCpu The cross context virtual CPU structure.
5553 * @param idxField The VMCS field encoding.
5554 * @param u64Val 16, 32 or 64-bit value.
5555 */
5556VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5557{
5558 AssertPtr(pVCpu);
5559 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5560
5561 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5562 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5563
5564 /* Make sure there are no duplicates. */
5565 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5566 {
5567 if (pCache->Write.aField[i] == idxField)
5568 {
5569 pCache->Write.aFieldVal[i] = u64Val;
5570 return VINF_SUCCESS;
5571 }
5572 }
5573
5574 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5575 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5576 pCache->Write.cValidEntries++;
5577 return VINF_SUCCESS;
5578}
5579#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5580
5581
5582/**
5583 * Sets up the usage of TSC-offsetting and updates the VMCS.
5584 *
5585 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5586 * VMX preemption timer.
5587 *
5588 * @returns VBox status code.
5589 * @param pVM The cross context VM structure.
5590 * @param pVCpu The cross context virtual CPU structure.
5591 *
5592 * @remarks No-long-jump zone!!!
5593 */
5594static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5595{
5596 int rc;
5597 bool fOffsettedTsc;
5598 bool fParavirtTsc;
5599 if (pVM->hm.s.vmx.fUsePreemptTimer)
5600 {
5601 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5602 &fOffsettedTsc, &fParavirtTsc);
5603
5604 /* Make sure the returned values have sane upper and lower boundaries. */
5605 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5606 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5607 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5608 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5609
5610 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5611 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5612 }
5613 else
5614 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5615
5616 /** @todo later optimize this to be done elsewhere and not before every
5617 * VM-entry. */
5618 if (fParavirtTsc)
5619 {
5620 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5621 information before every VM-entry, hence disable it for performance sake. */
5622#if 0
5623 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5624 AssertRC(rc);
5625#endif
5626 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5627 }
5628
5629 if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5630 {
5631 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5632 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5633
5634 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5635 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5636 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5637 }
5638 else
5639 {
5640 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5641 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5642 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5643 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5644 }
5645}
5646
5647
5648/**
5649 * Determines if an exception is a contributory exception.
5650 *
5651 * Contributory exceptions are ones which can cause double-faults unless the
5652 * original exception was a benign exception. Page-fault is intentionally not
5653 * included here as it's a conditional contributory exception.
5654 *
5655 * @returns true if the exception is contributory, false otherwise.
5656 * @param uVector The exception vector.
5657 */
5658DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5659{
5660 switch (uVector)
5661 {
5662 case X86_XCPT_GP:
5663 case X86_XCPT_SS:
5664 case X86_XCPT_NP:
5665 case X86_XCPT_TS:
5666 case X86_XCPT_DE:
5667 return true;
5668 default:
5669 break;
5670 }
5671 return false;
5672}
5673
5674
5675/**
5676 * Sets an event as a pending event to be injected into the guest.
5677 *
5678 * @param pVCpu The cross context virtual CPU structure.
5679 * @param u32IntInfo The VM-entry interruption-information field.
5680 * @param cbInstr The VM-entry instruction length in bytes (for software
5681 * interrupts, exceptions and privileged software
5682 * exceptions).
5683 * @param u32ErrCode The VM-entry exception error code.
5684 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5685 * page-fault.
5686 *
5687 * @remarks Statistics counter assumes this is a guest event being injected or
5688 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5689 * always incremented.
5690 */
5691DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5692 RTGCUINTPTR GCPtrFaultAddress)
5693{
5694 Assert(!pVCpu->hm.s.Event.fPending);
5695 pVCpu->hm.s.Event.fPending = true;
5696 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5697 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5698 pVCpu->hm.s.Event.cbInstr = cbInstr;
5699 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5700}
5701
5702
5703/**
5704 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5705 *
5706 * @param pVCpu The cross context virtual CPU structure.
5707 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5708 * out-of-sync. Make sure to update the required fields
5709 * before using them.
5710 */
5711DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5712{
5713 NOREF(pMixedCtx);
5714 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5715 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5716 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5717 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5718}
5719
5720
5721/**
5722 * Handle a condition that occurred while delivering an event through the guest
5723 * IDT.
5724 *
5725 * @returns Strict VBox status code (i.e. informational status codes too).
5726 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5727 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5728 * to continue execution of the guest which will delivery the \#DF.
5729 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5730 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5731 *
5732 * @param pVCpu The cross context virtual CPU structure.
5733 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5734 * out-of-sync. Make sure to update the required fields
5735 * before using them.
5736 * @param pVmxTransient Pointer to the VMX transient structure.
5737 *
5738 * @remarks No-long-jump zone!!!
5739 */
5740static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5741{
5742 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5743
5744 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5745 rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5746
5747 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5748 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5749 {
5750 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5751 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5752
5753 typedef enum
5754 {
5755 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5756 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5757 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5758 VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
5759 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5760 } VMXREFLECTXCPT;
5761
5762 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5763 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5764 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5765 {
5766 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5767 {
5768 enmReflect = VMXREFLECTXCPT_XCPT;
5769#ifdef VBOX_STRICT
5770 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5771 && uExitVector == X86_XCPT_PF)
5772 {
5773 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5774 }
5775#endif
5776 if ( uExitVector == X86_XCPT_PF
5777 && uIdtVector == X86_XCPT_PF)
5778 {
5779 pVmxTransient->fVectoringDoublePF = true;
5780 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5781 }
5782 else if ( uExitVector == X86_XCPT_AC
5783 && uIdtVector == X86_XCPT_AC)
5784 {
5785 enmReflect = VMXREFLECTXCPT_HANG;
5786 Log4(("IDT: Nested #AC - Bad guest\n"));
5787 }
5788 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5789 && hmR0VmxIsContributoryXcpt(uExitVector)
5790 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5791 || uIdtVector == X86_XCPT_PF))
5792 {
5793 enmReflect = VMXREFLECTXCPT_DF;
5794 }
5795 else if (uIdtVector == X86_XCPT_DF)
5796 enmReflect = VMXREFLECTXCPT_TF;
5797 }
5798 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5799 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5800 {
5801 /*
5802 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5803 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5804 */
5805 enmReflect = VMXREFLECTXCPT_XCPT;
5806
5807 if (uExitVector == X86_XCPT_PF)
5808 {
5809 pVmxTransient->fVectoringPF = true;
5810 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5811 }
5812 }
5813 }
5814 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5815 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5816 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5817 {
5818 /*
5819 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5820 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5821 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5822 */
5823 enmReflect = VMXREFLECTXCPT_XCPT;
5824 }
5825
5826 /*
5827 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5828 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5829 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5830 *
5831 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5832 */
5833 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5834 && enmReflect == VMXREFLECTXCPT_XCPT
5835 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5836 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5837 {
5838 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5839 }
5840
5841 switch (enmReflect)
5842 {
5843 case VMXREFLECTXCPT_XCPT:
5844 {
5845 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5846 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5847 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5848
5849 uint32_t u32ErrCode = 0;
5850 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5851 {
5852 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5853 AssertRCReturn(rc2, rc2);
5854 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5855 }
5856
5857 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5858 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5859 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5860 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5861 rcStrict = VINF_SUCCESS;
5862 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5863 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5864
5865 break;
5866 }
5867
5868 case VMXREFLECTXCPT_DF:
5869 {
5870 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5871 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5872 rcStrict = VINF_HM_DOUBLE_FAULT;
5873 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5874 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5875
5876 break;
5877 }
5878
5879 case VMXREFLECTXCPT_TF:
5880 {
5881 rcStrict = VINF_EM_RESET;
5882 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5883 uExitVector));
5884 break;
5885 }
5886
5887 case VMXREFLECTXCPT_HANG:
5888 {
5889 rcStrict = VERR_EM_GUEST_CPU_HANG;
5890 break;
5891 }
5892
5893 default:
5894 Assert(rcStrict == VINF_SUCCESS);
5895 break;
5896 }
5897 }
5898 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5899 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5900 && uExitVector != X86_XCPT_DF
5901 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5902 {
5903 /*
5904 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5905 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5906 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5907 */
5908 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5909 {
5910 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
5911 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5912 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5913 }
5914 }
5915
5916 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
5917 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
5918 return rcStrict;
5919}
5920
5921
5922/**
5923 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5924 *
5925 * @returns VBox status code.
5926 * @param pVCpu The cross context virtual CPU structure.
5927 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5928 * out-of-sync. Make sure to update the required fields
5929 * before using them.
5930 *
5931 * @remarks No-long-jump zone!!!
5932 */
5933static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5934{
5935 NOREF(pMixedCtx);
5936
5937 /*
5938 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5939 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5940 */
5941 VMMRZCallRing3Disable(pVCpu);
5942 HM_DISABLE_PREEMPT();
5943
5944 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5945 {
5946 uint32_t uVal = 0;
5947 uint32_t uShadow = 0;
5948 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5949 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5950 AssertRCReturn(rc, rc);
5951
5952 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5953 CPUMSetGuestCR0(pVCpu, uVal);
5954 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5955 }
5956
5957 HM_RESTORE_PREEMPT();
5958 VMMRZCallRing3Enable(pVCpu);
5959 return VINF_SUCCESS;
5960}
5961
5962
5963/**
5964 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5965 *
5966 * @returns VBox status code.
5967 * @param pVCpu The cross context virtual CPU structure.
5968 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5969 * out-of-sync. Make sure to update the required fields
5970 * before using them.
5971 *
5972 * @remarks No-long-jump zone!!!
5973 */
5974static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5975{
5976 NOREF(pMixedCtx);
5977
5978 int rc = VINF_SUCCESS;
5979 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5980 {
5981 uint32_t uVal = 0;
5982 uint32_t uShadow = 0;
5983 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5984 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5985 AssertRCReturn(rc, rc);
5986
5987 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5988 CPUMSetGuestCR4(pVCpu, uVal);
5989 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5990 }
5991 return rc;
5992}
5993
5994
5995/**
5996 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5997 *
5998 * @returns VBox status code.
5999 * @param pVCpu The cross context virtual CPU structure.
6000 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6001 * out-of-sync. Make sure to update the required fields
6002 * before using them.
6003 *
6004 * @remarks No-long-jump zone!!!
6005 */
6006static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6007{
6008 int rc = VINF_SUCCESS;
6009 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6010 {
6011 uint64_t u64Val = 0;
6012 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6013 AssertRCReturn(rc, rc);
6014
6015 pMixedCtx->rip = u64Val;
6016 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6017 }
6018 return rc;
6019}
6020
6021
6022/**
6023 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6024 *
6025 * @returns VBox status code.
6026 * @param pVCpu The cross context virtual CPU structure.
6027 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6028 * out-of-sync. Make sure to update the required fields
6029 * before using them.
6030 *
6031 * @remarks No-long-jump zone!!!
6032 */
6033static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6034{
6035 int rc = VINF_SUCCESS;
6036 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6037 {
6038 uint64_t u64Val = 0;
6039 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6040 AssertRCReturn(rc, rc);
6041
6042 pMixedCtx->rsp = u64Val;
6043 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6044 }
6045 return rc;
6046}
6047
6048
6049/**
6050 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6051 *
6052 * @returns VBox status code.
6053 * @param pVCpu The cross context virtual CPU structure.
6054 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6055 * out-of-sync. Make sure to update the required fields
6056 * before using them.
6057 *
6058 * @remarks No-long-jump zone!!!
6059 */
6060static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6061{
6062 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6063 {
6064 uint32_t uVal = 0;
6065 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6066 AssertRCReturn(rc, rc);
6067
6068 pMixedCtx->eflags.u32 = uVal;
6069 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6070 {
6071 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6072 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6073
6074 pMixedCtx->eflags.Bits.u1VM = 0;
6075 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6076 }
6077
6078 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6079 }
6080 return VINF_SUCCESS;
6081}
6082
6083
6084/**
6085 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6086 * guest-CPU context.
6087 */
6088DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6089{
6090 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6091 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6092 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6093 return rc;
6094}
6095
6096
6097/**
6098 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6099 * from the guest-state area in the VMCS.
6100 *
6101 * @param pVCpu The cross context virtual CPU structure.
6102 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6103 * out-of-sync. Make sure to update the required fields
6104 * before using them.
6105 *
6106 * @remarks No-long-jump zone!!!
6107 */
6108static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6109{
6110 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6111 {
6112 uint32_t uIntrState = 0;
6113 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6114 AssertRC(rc);
6115
6116 if (!uIntrState)
6117 {
6118 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6119 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6120
6121 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6122 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6123 }
6124 else
6125 {
6126 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6127 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6128 {
6129 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6130 AssertRC(rc);
6131 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6132 AssertRC(rc);
6133
6134 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6135 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6136 }
6137 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6138 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6139
6140 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6141 {
6142 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6143 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6144 }
6145 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6146 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6147 }
6148
6149 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6150 }
6151}
6152
6153
6154/**
6155 * Saves the guest's activity state.
6156 *
6157 * @returns VBox status code.
6158 * @param pVCpu The cross context virtual CPU structure.
6159 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6160 * out-of-sync. Make sure to update the required fields
6161 * before using them.
6162 *
6163 * @remarks No-long-jump zone!!!
6164 */
6165static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6166{
6167 NOREF(pMixedCtx);
6168 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6169 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6170 return VINF_SUCCESS;
6171}
6172
6173
6174/**
6175 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6176 * the current VMCS into the guest-CPU context.
6177 *
6178 * @returns VBox status code.
6179 * @param pVCpu The cross context virtual CPU structure.
6180 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6181 * out-of-sync. Make sure to update the required fields
6182 * before using them.
6183 *
6184 * @remarks No-long-jump zone!!!
6185 */
6186static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6187{
6188 int rc = VINF_SUCCESS;
6189 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6190 {
6191 uint32_t u32Val = 0;
6192 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6193 pMixedCtx->SysEnter.cs = u32Val;
6194 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6195 }
6196
6197 uint64_t u64Val = 0;
6198 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6199 {
6200 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6201 pMixedCtx->SysEnter.eip = u64Val;
6202 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6203 }
6204 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6205 {
6206 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6207 pMixedCtx->SysEnter.esp = u64Val;
6208 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6209 }
6210 return rc;
6211}
6212
6213
6214/**
6215 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6216 * the CPU back into the guest-CPU context.
6217 *
6218 * @returns VBox status code.
6219 * @param pVCpu The cross context virtual CPU structure.
6220 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6221 * out-of-sync. Make sure to update the required fields
6222 * before using them.
6223 *
6224 * @remarks No-long-jump zone!!!
6225 */
6226static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6227{
6228 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6229 VMMRZCallRing3Disable(pVCpu);
6230 HM_DISABLE_PREEMPT();
6231
6232 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6233 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6234 {
6235 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6236 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6237 }
6238
6239 HM_RESTORE_PREEMPT();
6240 VMMRZCallRing3Enable(pVCpu);
6241
6242 return VINF_SUCCESS;
6243}
6244
6245
6246/**
6247 * Saves the auto load/store'd guest MSRs from the current VMCS into
6248 * the guest-CPU context.
6249 *
6250 * @returns VBox status code.
6251 * @param pVCpu The cross context virtual CPU structure.
6252 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6253 * out-of-sync. Make sure to update the required fields
6254 * before using them.
6255 *
6256 * @remarks No-long-jump zone!!!
6257 */
6258static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6259{
6260 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6261 return VINF_SUCCESS;
6262
6263 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6264 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6265 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6266 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6267 {
6268 switch (pMsr->u32Msr)
6269 {
6270 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6271 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6272 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6273 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6274 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6275 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6276 break;
6277
6278 default:
6279 {
6280 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6281 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6282 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6283 }
6284 }
6285 }
6286
6287 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6288 return VINF_SUCCESS;
6289}
6290
6291
6292/**
6293 * Saves the guest control registers from the current VMCS into the guest-CPU
6294 * context.
6295 *
6296 * @returns VBox status code.
6297 * @param pVCpu The cross context virtual CPU structure.
6298 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6299 * out-of-sync. Make sure to update the required fields
6300 * before using them.
6301 *
6302 * @remarks No-long-jump zone!!!
6303 */
6304static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6305{
6306 /* Guest CR0. Guest FPU. */
6307 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6308 AssertRCReturn(rc, rc);
6309
6310 /* Guest CR4. */
6311 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6312 AssertRCReturn(rc, rc);
6313
6314 /* Guest CR2 - updated always during the world-switch or in #PF. */
6315 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6316 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6317 {
6318 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6319 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6320
6321 PVM pVM = pVCpu->CTX_SUFF(pVM);
6322 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6323 || ( pVM->hm.s.fNestedPaging
6324 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6325 {
6326 uint64_t u64Val = 0;
6327 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6328 if (pMixedCtx->cr3 != u64Val)
6329 {
6330 CPUMSetGuestCR3(pVCpu, u64Val);
6331 if (VMMRZCallRing3IsEnabled(pVCpu))
6332 {
6333 PGMUpdateCR3(pVCpu, u64Val);
6334 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6335 }
6336 else
6337 {
6338 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6339 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6340 }
6341 }
6342
6343 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6344 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6345 {
6346 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6347 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6348 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6349 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6350 AssertRCReturn(rc, rc);
6351
6352 if (VMMRZCallRing3IsEnabled(pVCpu))
6353 {
6354 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6355 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6356 }
6357 else
6358 {
6359 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6360 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6361 }
6362 }
6363 }
6364
6365 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6366 }
6367
6368 /*
6369 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6370 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6371 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6372 *
6373 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6374 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6375 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6376 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6377 *
6378 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6379 */
6380 if (VMMRZCallRing3IsEnabled(pVCpu))
6381 {
6382 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6383 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6384
6385 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6386 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6387
6388 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6389 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6390 }
6391
6392 return rc;
6393}
6394
6395
6396/**
6397 * Reads a guest segment register from the current VMCS into the guest-CPU
6398 * context.
6399 *
6400 * @returns VBox status code.
6401 * @param pVCpu The cross context virtual CPU structure.
6402 * @param idxSel Index of the selector in the VMCS.
6403 * @param idxLimit Index of the segment limit in the VMCS.
6404 * @param idxBase Index of the segment base in the VMCS.
6405 * @param idxAccess Index of the access rights of the segment in the VMCS.
6406 * @param pSelReg Pointer to the segment selector.
6407 *
6408 * @remarks No-long-jump zone!!!
6409 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6410 * macro as that takes care of whether to read from the VMCS cache or
6411 * not.
6412 */
6413DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6414 PCPUMSELREG pSelReg)
6415{
6416 NOREF(pVCpu);
6417
6418 uint32_t u32Val = 0;
6419 int rc = VMXReadVmcs32(idxSel, &u32Val);
6420 AssertRCReturn(rc, rc);
6421 pSelReg->Sel = (uint16_t)u32Val;
6422 pSelReg->ValidSel = (uint16_t)u32Val;
6423 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6424
6425 rc = VMXReadVmcs32(idxLimit, &u32Val);
6426 AssertRCReturn(rc, rc);
6427 pSelReg->u32Limit = u32Val;
6428
6429 uint64_t u64Val = 0;
6430 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6431 AssertRCReturn(rc, rc);
6432 pSelReg->u64Base = u64Val;
6433
6434 rc = VMXReadVmcs32(idxAccess, &u32Val);
6435 AssertRCReturn(rc, rc);
6436 pSelReg->Attr.u = u32Val;
6437
6438 /*
6439 * If VT-x marks the segment as unusable, most other bits remain undefined:
6440 * - For CS the L, D and G bits have meaning.
6441 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6442 * - For the remaining data segments no bits are defined.
6443 *
6444 * The present bit and the unusable bit has been observed to be set at the
6445 * same time (the selector was supposed to be invalid as we started executing
6446 * a V8086 interrupt in ring-0).
6447 *
6448 * What should be important for the rest of the VBox code, is that the P bit is
6449 * cleared. Some of the other VBox code recognizes the unusable bit, but
6450 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6451 * safe side here, we'll strip off P and other bits we don't care about. If
6452 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6453 *
6454 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6455 */
6456 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6457 {
6458 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6459
6460 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6461 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6462 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6463
6464 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6465#ifdef DEBUG_bird
6466 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6467 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6468 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6469#endif
6470 }
6471 return VINF_SUCCESS;
6472}
6473
6474
6475#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6476# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6477 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6478 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6479#else
6480# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6481 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6482 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6483#endif
6484
6485
6486/**
6487 * Saves the guest segment registers from the current VMCS into the guest-CPU
6488 * context.
6489 *
6490 * @returns VBox status code.
6491 * @param pVCpu The cross context virtual CPU structure.
6492 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6493 * out-of-sync. Make sure to update the required fields
6494 * before using them.
6495 *
6496 * @remarks No-long-jump zone!!!
6497 */
6498static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6499{
6500 /* Guest segment registers. */
6501 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6502 {
6503 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6504 AssertRCReturn(rc, rc);
6505
6506 rc = VMXLOCAL_READ_SEG(CS, cs);
6507 rc |= VMXLOCAL_READ_SEG(SS, ss);
6508 rc |= VMXLOCAL_READ_SEG(DS, ds);
6509 rc |= VMXLOCAL_READ_SEG(ES, es);
6510 rc |= VMXLOCAL_READ_SEG(FS, fs);
6511 rc |= VMXLOCAL_READ_SEG(GS, gs);
6512 AssertRCReturn(rc, rc);
6513
6514 /* Restore segment attributes for real-on-v86 mode hack. */
6515 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6516 {
6517 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6518 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6519 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6520 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6521 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6522 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6523 }
6524 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6525 }
6526
6527 return VINF_SUCCESS;
6528}
6529
6530
6531/**
6532 * Saves the guest descriptor table registers and task register from the current
6533 * VMCS into the guest-CPU context.
6534 *
6535 * @returns VBox status code.
6536 * @param pVCpu The cross context virtual CPU structure.
6537 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6538 * out-of-sync. Make sure to update the required fields
6539 * before using them.
6540 *
6541 * @remarks No-long-jump zone!!!
6542 */
6543static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6544{
6545 int rc = VINF_SUCCESS;
6546
6547 /* Guest LDTR. */
6548 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6549 {
6550 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6551 AssertRCReturn(rc, rc);
6552 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6553 }
6554
6555 /* Guest GDTR. */
6556 uint64_t u64Val = 0;
6557 uint32_t u32Val = 0;
6558 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6559 {
6560 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6561 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6562 pMixedCtx->gdtr.pGdt = u64Val;
6563 pMixedCtx->gdtr.cbGdt = u32Val;
6564 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6565 }
6566
6567 /* Guest IDTR. */
6568 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6569 {
6570 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6571 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6572 pMixedCtx->idtr.pIdt = u64Val;
6573 pMixedCtx->idtr.cbIdt = u32Val;
6574 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6575 }
6576
6577 /* Guest TR. */
6578 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6579 {
6580 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6581 AssertRCReturn(rc, rc);
6582
6583 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6584 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6585 {
6586 rc = VMXLOCAL_READ_SEG(TR, tr);
6587 AssertRCReturn(rc, rc);
6588 }
6589 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6590 }
6591 return rc;
6592}
6593
6594#undef VMXLOCAL_READ_SEG
6595
6596
6597/**
6598 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6599 * context.
6600 *
6601 * @returns VBox status code.
6602 * @param pVCpu The cross context virtual CPU structure.
6603 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6604 * out-of-sync. Make sure to update the required fields
6605 * before using them.
6606 *
6607 * @remarks No-long-jump zone!!!
6608 */
6609static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6610{
6611 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6612 {
6613 if (!pVCpu->hm.s.fUsingHyperDR7)
6614 {
6615 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6616 uint32_t u32Val;
6617 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6618 pMixedCtx->dr[7] = u32Val;
6619 }
6620
6621 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6622 }
6623 return VINF_SUCCESS;
6624}
6625
6626
6627/**
6628 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6629 *
6630 * @returns VBox status code.
6631 * @param pVCpu The cross context virtual CPU structure.
6632 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6633 * out-of-sync. Make sure to update the required fields
6634 * before using them.
6635 *
6636 * @remarks No-long-jump zone!!!
6637 */
6638static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6639{
6640 NOREF(pMixedCtx);
6641
6642 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6643 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6644 return VINF_SUCCESS;
6645}
6646
6647
6648/**
6649 * Saves the entire guest state from the currently active VMCS into the
6650 * guest-CPU context.
6651 *
6652 * This essentially VMREADs all guest-data.
6653 *
6654 * @returns VBox status code.
6655 * @param pVCpu The cross context virtual CPU structure.
6656 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6657 * out-of-sync. Make sure to update the required fields
6658 * before using them.
6659 */
6660static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6661{
6662 Assert(pVCpu);
6663 Assert(pMixedCtx);
6664
6665 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6666 return VINF_SUCCESS;
6667
6668 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6669 again on the ring-3 callback path, there is no real need to. */
6670 if (VMMRZCallRing3IsEnabled(pVCpu))
6671 VMMR0LogFlushDisable(pVCpu);
6672 else
6673 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6674 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6675
6676 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6677 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6678
6679 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6680 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6681
6682 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6683 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6684
6685 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6686 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6687
6688 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6689 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6690
6691 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6692 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6693
6694 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6695 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6696
6697 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6698 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6699
6700 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6701 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6702
6703 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6704 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6705
6706 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6707 ("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
6708 HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
6709
6710 if (VMMRZCallRing3IsEnabled(pVCpu))
6711 VMMR0LogFlushEnable(pVCpu);
6712
6713 return VINF_SUCCESS;
6714}
6715
6716
6717/**
6718 * Saves basic guest registers needed for IEM instruction execution.
6719 *
6720 * @returns VBox status code (OR-able).
6721 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6722 * @param pMixedCtx Pointer to the CPU context of the guest.
6723 * @param fMemory Whether the instruction being executed operates on
6724 * memory or not. Only CR0 is synced up if clear.
6725 * @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
6726 */
6727static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
6728{
6729 /*
6730 * We assume all general purpose registers other than RSP are available.
6731 *
6732 * RIP is a must, as it will be incremented or otherwise changed.
6733 *
6734 * RFLAGS are always required to figure the CPL.
6735 *
6736 * RSP isn't always required, however it's a GPR, so frequently required.
6737 *
6738 * SS and CS are the only segment register needed if IEM doesn't do memory
6739 * access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
6740 *
6741 * CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
6742 * be required for memory accesses.
6743 *
6744 * Note! Before IEM dispatches an exception, it will call us to sync in everything.
6745 */
6746 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6747 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6748 if (fNeedRsp)
6749 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6750 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6751 if (!fMemory)
6752 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6753 else
6754 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6755 AssertRCReturn(rc, rc);
6756 return rc;
6757}
6758
6759
6760/**
6761 * Ensures that we've got a complete basic guest-context.
6762 *
6763 * This excludes the FPU, SSE, AVX, and similar extended state. The interface
6764 * is for the interpreter.
6765 *
6766 * @returns VBox status code.
6767 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6768 * @param pMixedCtx Pointer to the guest-CPU context which may have data
6769 * needing to be synced in.
6770 * @thread EMT(pVCpu)
6771 */
6772VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6773{
6774 /* Note! Since this is only applicable to VT-x, the implementation is placed
6775 in the VT-x part of the sources instead of the generic stuff. */
6776 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
6777 {
6778 /* For now, imply that the caller might change everything too. */
6779 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6780 return hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6781 }
6782 return VINF_SUCCESS;
6783}
6784
6785
6786/**
6787 * Check per-VM and per-VCPU force flag actions that require us to go back to
6788 * ring-3 for one reason or another.
6789 *
6790 * @returns Strict VBox status code (i.e. informational status codes too)
6791 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6792 * ring-3.
6793 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6794 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6795 * interrupts)
6796 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6797 * all EMTs to be in ring-3.
6798 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6799 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6800 * to the EM loop.
6801 *
6802 * @param pVM The cross context VM structure.
6803 * @param pVCpu The cross context virtual CPU structure.
6804 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6805 * out-of-sync. Make sure to update the required fields
6806 * before using them.
6807 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6808 */
6809static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6810{
6811 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6812
6813 /*
6814 * Anything pending? Should be more likely than not if we're doing a good job.
6815 */
6816 if ( !fStepping
6817 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6818 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6819 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6820 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6821 return VINF_SUCCESS;
6822
6823 /* We need the control registers now, make sure the guest-CPU context is updated. */
6824 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6825 AssertRCReturn(rc3, rc3);
6826
6827 /* Pending HM CR3 sync. */
6828 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6829 {
6830 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6831 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6832 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6833 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6834 }
6835
6836 /* Pending HM PAE PDPEs. */
6837 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6838 {
6839 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6840 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6841 }
6842
6843 /* Pending PGM C3 sync. */
6844 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6845 {
6846 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6847 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6848 if (rcStrict2 != VINF_SUCCESS)
6849 {
6850 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6851 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6852 return rcStrict2;
6853 }
6854 }
6855
6856 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6857 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6858 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6859 {
6860 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6861 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6862 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6863 return rc2;
6864 }
6865
6866 /* Pending VM request packets, such as hardware interrupts. */
6867 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6868 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6869 {
6870 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6871 return VINF_EM_PENDING_REQUEST;
6872 }
6873
6874 /* Pending PGM pool flushes. */
6875 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6876 {
6877 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6878 return VINF_PGM_POOL_FLUSH_PENDING;
6879 }
6880
6881 /* Pending DMA requests. */
6882 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6883 {
6884 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6885 return VINF_EM_RAW_TO_R3;
6886 }
6887
6888 return VINF_SUCCESS;
6889}
6890
6891
6892/**
6893 * Converts any TRPM trap into a pending HM event. This is typically used when
6894 * entering from ring-3 (not longjmp returns).
6895 *
6896 * @param pVCpu The cross context virtual CPU structure.
6897 */
6898static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6899{
6900 Assert(TRPMHasTrap(pVCpu));
6901 Assert(!pVCpu->hm.s.Event.fPending);
6902
6903 uint8_t uVector;
6904 TRPMEVENT enmTrpmEvent;
6905 RTGCUINT uErrCode;
6906 RTGCUINTPTR GCPtrFaultAddress;
6907 uint8_t cbInstr;
6908
6909 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6910 AssertRC(rc);
6911
6912 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6913 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6914 if (enmTrpmEvent == TRPM_TRAP)
6915 {
6916 switch (uVector)
6917 {
6918 case X86_XCPT_NMI:
6919 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6920 break;
6921
6922 case X86_XCPT_BP:
6923 case X86_XCPT_OF:
6924 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6925 break;
6926
6927 case X86_XCPT_PF:
6928 case X86_XCPT_DF:
6929 case X86_XCPT_TS:
6930 case X86_XCPT_NP:
6931 case X86_XCPT_SS:
6932 case X86_XCPT_GP:
6933 case X86_XCPT_AC:
6934 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6935 /* no break! */
6936 default:
6937 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6938 break;
6939 }
6940 }
6941 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6942 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6943 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6944 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6945 else
6946 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6947
6948 rc = TRPMResetTrap(pVCpu);
6949 AssertRC(rc);
6950 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6951 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6952
6953 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6954}
6955
6956
6957/**
6958 * Converts the pending HM event into a TRPM trap.
6959 *
6960 * @param pVCpu The cross context virtual CPU structure.
6961 */
6962static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6963{
6964 Assert(pVCpu->hm.s.Event.fPending);
6965
6966 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6967 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6968 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6969 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6970
6971 /* If a trap was already pending, we did something wrong! */
6972 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6973
6974 TRPMEVENT enmTrapType;
6975 switch (uVectorType)
6976 {
6977 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6978 enmTrapType = TRPM_HARDWARE_INT;
6979 break;
6980
6981 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6982 enmTrapType = TRPM_SOFTWARE_INT;
6983 break;
6984
6985 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6986 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6987 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6988 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6989 enmTrapType = TRPM_TRAP;
6990 break;
6991
6992 default:
6993 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6994 enmTrapType = TRPM_32BIT_HACK;
6995 break;
6996 }
6997
6998 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6999
7000 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7001 AssertRC(rc);
7002
7003 if (fErrorCodeValid)
7004 TRPMSetErrorCode(pVCpu, uErrorCode);
7005
7006 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7007 && uVector == X86_XCPT_PF)
7008 {
7009 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7010 }
7011 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7012 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7013 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7014 {
7015 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7016 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
7017 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7018 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7019 }
7020
7021 /* Clear any pending events from the VMCS. */
7022 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
7023 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
7024
7025 /* We're now done converting the pending event. */
7026 pVCpu->hm.s.Event.fPending = false;
7027}
7028
7029
7030/**
7031 * Does the necessary state syncing before returning to ring-3 for any reason
7032 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7033 *
7034 * @returns VBox status code.
7035 * @param pVM The cross context VM structure.
7036 * @param pVCpu The cross context virtual CPU structure.
7037 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7038 * be out-of-sync. Make sure to update the required
7039 * fields before using them.
7040 * @param fSaveGuestState Whether to save the guest state or not.
7041 *
7042 * @remarks No-long-jmp zone!!!
7043 */
7044static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7045{
7046 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7047 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7048
7049 RTCPUID idCpu = RTMpCpuId();
7050 Log4Func(("HostCpuId=%u\n", idCpu));
7051
7052 /*
7053 * !!! IMPORTANT !!!
7054 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7055 */
7056
7057 /* Save the guest state if necessary. */
7058 if ( fSaveGuestState
7059 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7060 {
7061 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7062 AssertRCReturn(rc, rc);
7063 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7064 }
7065
7066 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
7067 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7068 {
7069 if (fSaveGuestState)
7070 {
7071 /* We shouldn't reload CR0 without saving it first. */
7072 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7073 AssertRCReturn(rc, rc);
7074 }
7075 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7076 }
7077
7078 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7079#ifdef VBOX_STRICT
7080 if (CPUMIsHyperDebugStateActive(pVCpu))
7081 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7082#endif
7083 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7084 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7085 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7086 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7087
7088#if HC_ARCH_BITS == 64
7089 /* Restore host-state bits that VT-x only restores partially. */
7090 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7091 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7092 {
7093 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7094 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7095 }
7096 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7097#endif
7098
7099 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7100 if (pVCpu->hm.s.vmx.fLazyMsrs)
7101 {
7102 /* We shouldn't reload the guest MSRs without saving it first. */
7103 if (!fSaveGuestState)
7104 {
7105 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7106 AssertRCReturn(rc, rc);
7107 }
7108 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7109 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7110 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7111 }
7112
7113 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7114 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7115
7116 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7117 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7118 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7119 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7120 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7121 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7122 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7123 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7124
7125 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7126
7127 /** @todo This partially defeats the purpose of having preemption hooks.
7128 * The problem is, deregistering the hooks should be moved to a place that
7129 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7130 * context.
7131 */
7132 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7133 {
7134 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7135 AssertRCReturn(rc, rc);
7136
7137 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7138 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7139 }
7140 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7141 NOREF(idCpu);
7142
7143 return VINF_SUCCESS;
7144}
7145
7146
7147/**
7148 * Leaves the VT-x session.
7149 *
7150 * @returns VBox status code.
7151 * @param pVM The cross context VM structure.
7152 * @param pVCpu The cross context virtual CPU structure.
7153 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7154 * out-of-sync. Make sure to update the required fields
7155 * before using them.
7156 *
7157 * @remarks No-long-jmp zone!!!
7158 */
7159DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7160{
7161 HM_DISABLE_PREEMPT();
7162 HMVMX_ASSERT_CPU_SAFE();
7163 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7164 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7165
7166 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7167 and done this from the VMXR0ThreadCtxCallback(). */
7168 if (!pVCpu->hm.s.fLeaveDone)
7169 {
7170 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7171 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7172 pVCpu->hm.s.fLeaveDone = true;
7173 }
7174 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7175
7176 /*
7177 * !!! IMPORTANT !!!
7178 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7179 */
7180
7181 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7182 /** @todo Deregistering here means we need to VMCLEAR always
7183 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7184 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7185 VMMR0ThreadCtxHookDisable(pVCpu);
7186
7187 /* Leave HM context. This takes care of local init (term). */
7188 int rc = HMR0LeaveCpu(pVCpu);
7189
7190 HM_RESTORE_PREEMPT();
7191 return rc;
7192}
7193
7194
7195/**
7196 * Does the necessary state syncing before doing a longjmp to ring-3.
7197 *
7198 * @returns VBox status code.
7199 * @param pVM The cross context VM structure.
7200 * @param pVCpu The cross context virtual CPU structure.
7201 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7202 * out-of-sync. Make sure to update the required fields
7203 * before using them.
7204 *
7205 * @remarks No-long-jmp zone!!!
7206 */
7207DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7208{
7209 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7210}
7211
7212
7213/**
7214 * Take necessary actions before going back to ring-3.
7215 *
7216 * An action requires us to go back to ring-3. This function does the necessary
7217 * steps before we can safely return to ring-3. This is not the same as longjmps
7218 * to ring-3, this is voluntary and prepares the guest so it may continue
7219 * executing outside HM (recompiler/IEM).
7220 *
7221 * @returns VBox status code.
7222 * @param pVM The cross context VM structure.
7223 * @param pVCpu The cross context virtual CPU structure.
7224 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7225 * out-of-sync. Make sure to update the required fields
7226 * before using them.
7227 * @param rcExit The reason for exiting to ring-3. Can be
7228 * VINF_VMM_UNKNOWN_RING3_CALL.
7229 */
7230static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7231{
7232 Assert(pVM);
7233 Assert(pVCpu);
7234 Assert(pMixedCtx);
7235 HMVMX_ASSERT_PREEMPT_SAFE();
7236
7237 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7238 {
7239 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7240 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7241 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7242 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7243 }
7244
7245 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7246 VMMRZCallRing3Disable(pVCpu);
7247 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7248
7249 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7250 if (pVCpu->hm.s.Event.fPending)
7251 {
7252 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7253 Assert(!pVCpu->hm.s.Event.fPending);
7254 }
7255
7256 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7257 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7258
7259 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7260 and if we're injecting an event we should have a TRPM trap pending. */
7261 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7262#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7263 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7264#endif
7265
7266 /* Save guest state and restore host state bits. */
7267 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7268 AssertRCReturn(rc, rc);
7269 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7270 /* Thread-context hooks are unregistered at this point!!! */
7271
7272 /* Sync recompiler state. */
7273 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7274 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7275 | CPUM_CHANGED_LDTR
7276 | CPUM_CHANGED_GDTR
7277 | CPUM_CHANGED_IDTR
7278 | CPUM_CHANGED_TR
7279 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7280 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7281 if ( pVM->hm.s.fNestedPaging
7282 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7283 {
7284 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7285 }
7286
7287 Assert(!pVCpu->hm.s.fClearTrapFlag);
7288
7289 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7290 if (rcExit != VINF_EM_RAW_INTERRUPT)
7291 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7292
7293 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7294
7295 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7296 VMMRZCallRing3RemoveNotification(pVCpu);
7297 VMMRZCallRing3Enable(pVCpu);
7298
7299 return rc;
7300}
7301
7302
7303/**
7304 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7305 * longjump to ring-3 and possibly get preempted.
7306 *
7307 * @returns VBox status code.
7308 * @param pVCpu The cross context virtual CPU structure.
7309 * @param enmOperation The operation causing the ring-3 longjump.
7310 * @param pvUser Opaque pointer to the guest-CPU context. The data
7311 * may be out-of-sync. Make sure to update the required
7312 * fields before using them.
7313 */
7314static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7315{
7316 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7317 {
7318 /*
7319 * !!! IMPORTANT !!!
7320 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7321 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7322 */
7323 VMMRZCallRing3RemoveNotification(pVCpu);
7324 VMMRZCallRing3Disable(pVCpu);
7325 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7326 RTThreadPreemptDisable(&PreemptState);
7327
7328 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7329 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7330
7331#if HC_ARCH_BITS == 64
7332 /* Restore host-state bits that VT-x only restores partially. */
7333 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7334 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7335 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7336 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7337#endif
7338 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7339 if (pVCpu->hm.s.vmx.fLazyMsrs)
7340 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7341
7342 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7343 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7344 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7345 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7346 {
7347 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7348 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7349 }
7350
7351 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7352 VMMR0ThreadCtxHookDisable(pVCpu);
7353 HMR0LeaveCpu(pVCpu);
7354 RTThreadPreemptRestore(&PreemptState);
7355 return VINF_SUCCESS;
7356 }
7357
7358 Assert(pVCpu);
7359 Assert(pvUser);
7360 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7361 HMVMX_ASSERT_PREEMPT_SAFE();
7362
7363 VMMRZCallRing3Disable(pVCpu);
7364 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7365
7366 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7367 enmOperation));
7368
7369 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7370 AssertRCReturn(rc, rc);
7371
7372 VMMRZCallRing3Enable(pVCpu);
7373 return VINF_SUCCESS;
7374}
7375
7376
7377/**
7378 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7379 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7380 *
7381 * @param pVCpu The cross context virtual CPU structure.
7382 */
7383DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7384{
7385 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7386 {
7387 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7388 {
7389 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7390 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7391 AssertRC(rc);
7392 Log4(("Setup interrupt-window exiting\n"));
7393 }
7394 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7395}
7396
7397
7398/**
7399 * Clears the interrupt-window exiting control in the VMCS.
7400 *
7401 * @param pVCpu The cross context virtual CPU structure.
7402 */
7403DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7404{
7405 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7406 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7407 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7408 AssertRC(rc);
7409 Log4(("Cleared interrupt-window exiting\n"));
7410}
7411
7412
7413/**
7414 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7415 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7416 *
7417 * @param pVCpu The cross context virtual CPU structure.
7418 */
7419DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7420{
7421 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7422 {
7423 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7424 {
7425 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7426 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7427 AssertRC(rc);
7428 Log4(("Setup NMI-window exiting\n"));
7429 }
7430 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7431}
7432
7433
7434/**
7435 * Clears the NMI-window exiting control in the VMCS.
7436 *
7437 * @param pVCpu The cross context virtual CPU structure.
7438 */
7439DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7440{
7441 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7442 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7443 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7444 AssertRC(rc);
7445 Log4(("Cleared NMI-window exiting\n"));
7446}
7447
7448
7449/**
7450 * Evaluates the event to be delivered to the guest and sets it as the pending
7451 * event.
7452 *
7453 * @returns The VT-x guest-interruptibility state.
7454 * @param pVCpu The cross context virtual CPU structure.
7455 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7456 * out-of-sync. Make sure to update the required fields
7457 * before using them.
7458 */
7459static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7460{
7461 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7462 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7463 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7464 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7465 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7466
7467 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7468 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7469 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7470 Assert(!TRPMHasTrap(pVCpu));
7471
7472#ifdef VBOX_WITH_NEW_APIC
7473 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7474 APICUpdatePendingInterrupts(pVCpu);
7475#endif
7476
7477 /*
7478 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7479 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7480 */
7481 /** @todo SMI. SMIs take priority over NMIs. */
7482 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7483 {
7484 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7485 if ( !pVCpu->hm.s.Event.fPending
7486 && !fBlockNmi
7487 && !fBlockSti
7488 && !fBlockMovSS)
7489 {
7490 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7491 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7492 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7493
7494 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7495 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7496 }
7497 else
7498 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7499 }
7500 /*
7501 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7502 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7503 */
7504 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7505 && !pVCpu->hm.s.fSingleInstruction)
7506 {
7507 Assert(!DBGFIsStepping(pVCpu));
7508 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7509 AssertRC(rc);
7510 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7511 if ( !pVCpu->hm.s.Event.fPending
7512 && !fBlockInt
7513 && !fBlockSti
7514 && !fBlockMovSS)
7515 {
7516 uint8_t u8Interrupt;
7517 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7518 if (RT_SUCCESS(rc))
7519 {
7520 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7521 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7522 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7523
7524 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7525 }
7526 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7527 {
7528 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7529 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7530 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7531 }
7532 else
7533 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7534 }
7535 else
7536 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7537 }
7538
7539 return uIntrState;
7540}
7541
7542
7543/**
7544 * Sets a pending-debug exception to be delivered to the guest if the guest is
7545 * single-stepping in the VMCS.
7546 *
7547 * @param pVCpu The cross context virtual CPU structure.
7548 */
7549DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7550{
7551 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7552 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7553 AssertRC(rc);
7554}
7555
7556
7557/**
7558 * Injects any pending events into the guest if the guest is in a state to
7559 * receive them.
7560 *
7561 * @returns Strict VBox status code (i.e. informational status codes too).
7562 * @param pVCpu The cross context virtual CPU structure.
7563 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7564 * out-of-sync. Make sure to update the required fields
7565 * before using them.
7566 * @param uIntrState The VT-x guest-interruptibility state.
7567 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7568 * return VINF_EM_DBG_STEPPED if the event was
7569 * dispatched directly.
7570 */
7571static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7572{
7573 HMVMX_ASSERT_PREEMPT_SAFE();
7574 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7575
7576 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7577 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7578
7579 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7580 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7581 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7582 Assert(!TRPMHasTrap(pVCpu));
7583
7584 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7585 if (pVCpu->hm.s.Event.fPending)
7586 {
7587 /*
7588 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7589 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7590 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7591 *
7592 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7593 */
7594 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7595#ifdef VBOX_STRICT
7596 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7597 {
7598 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7599 Assert(!fBlockInt);
7600 Assert(!fBlockSti);
7601 Assert(!fBlockMovSS);
7602 }
7603 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7604 {
7605 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7606 Assert(!fBlockSti);
7607 Assert(!fBlockMovSS);
7608 Assert(!fBlockNmi);
7609 }
7610#endif
7611 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7612 (uint8_t)uIntType));
7613 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7614 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7615 fStepping, &uIntrState);
7616 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7617
7618 /* Update the interruptibility-state as it could have been changed by
7619 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7620 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7621 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7622
7623 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7624 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7625 else
7626 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7627 }
7628
7629 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7630 if ( fBlockSti
7631 || fBlockMovSS)
7632 {
7633 if (!pVCpu->hm.s.fSingleInstruction)
7634 {
7635 /*
7636 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7637 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7638 * See Intel spec. 27.3.4 "Saving Non-Register State".
7639 */
7640 Assert(!DBGFIsStepping(pVCpu));
7641 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7642 AssertRCReturn(rc2, rc2);
7643 if (pMixedCtx->eflags.Bits.u1TF)
7644 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7645 }
7646 else if (pMixedCtx->eflags.Bits.u1TF)
7647 {
7648 /*
7649 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7650 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7651 */
7652 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7653 uIntrState = 0;
7654 }
7655 }
7656
7657 /*
7658 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7659 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7660 */
7661 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7662 AssertRC(rc2);
7663
7664 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7665 NOREF(fBlockMovSS); NOREF(fBlockSti);
7666 return rcStrict;
7667}
7668
7669
7670/**
7671 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7672 *
7673 * @param pVCpu The cross context virtual CPU structure.
7674 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7675 * out-of-sync. Make sure to update the required fields
7676 * before using them.
7677 */
7678DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7679{
7680 NOREF(pMixedCtx);
7681 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7682 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7683}
7684
7685
7686/**
7687 * Injects a double-fault (\#DF) exception into the VM.
7688 *
7689 * @returns Strict VBox status code (i.e. informational status codes too).
7690 * @param pVCpu The cross context virtual CPU structure.
7691 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7692 * out-of-sync. Make sure to update the required fields
7693 * before using them.
7694 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7695 * and should return VINF_EM_DBG_STEPPED if the event
7696 * is injected directly (register modified by us, not
7697 * by hardware on VM-entry).
7698 * @param puIntrState Pointer to the current guest interruptibility-state.
7699 * This interruptibility-state will be updated if
7700 * necessary. This cannot not be NULL.
7701 */
7702DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7703{
7704 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7705 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7706 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7707 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7708 fStepping, puIntrState);
7709}
7710
7711
7712/**
7713 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7714 *
7715 * @param pVCpu The cross context virtual CPU structure.
7716 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7717 * out-of-sync. Make sure to update the required fields
7718 * before using them.
7719 */
7720DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7721{
7722 NOREF(pMixedCtx);
7723 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7724 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7725 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7726}
7727
7728
7729/**
7730 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7731 *
7732 * @param pVCpu The cross context virtual CPU structure.
7733 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7734 * out-of-sync. Make sure to update the required fields
7735 * before using them.
7736 * @param cbInstr The value of RIP that is to be pushed on the guest
7737 * stack.
7738 */
7739DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7740{
7741 NOREF(pMixedCtx);
7742 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7743 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7744 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7745}
7746
7747
7748/**
7749 * Injects a general-protection (\#GP) fault into the VM.
7750 *
7751 * @returns Strict VBox status code (i.e. informational status codes too).
7752 * @param pVCpu The cross context virtual CPU structure.
7753 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7754 * out-of-sync. Make sure to update the required fields
7755 * before using them.
7756 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7757 * mode, i.e. in real-mode it's not valid).
7758 * @param u32ErrorCode The error code associated with the \#GP.
7759 * @param fStepping Whether we're running in
7760 * hmR0VmxRunGuestCodeStep() and should return
7761 * VINF_EM_DBG_STEPPED if the event is injected
7762 * directly (register modified by us, not by
7763 * hardware on VM-entry).
7764 * @param puIntrState Pointer to the current guest interruptibility-state.
7765 * This interruptibility-state will be updated if
7766 * necessary. This cannot not be NULL.
7767 */
7768DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7769 bool fStepping, uint32_t *puIntrState)
7770{
7771 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7772 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7773 if (fErrorCodeValid)
7774 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7775 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7776 fStepping, puIntrState);
7777}
7778
7779
7780/**
7781 * Sets a general-protection (\#GP) exception as pending-for-injection into the
7782 * VM.
7783 *
7784 * @param pVCpu The cross context virtual CPU structure.
7785 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7786 * out-of-sync. Make sure to update the required fields
7787 * before using them.
7788 * @param u32ErrorCode The error code associated with the \#GP.
7789 */
7790DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7791{
7792 NOREF(pMixedCtx);
7793 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7794 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7795 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7796 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7797}
7798
7799
7800/**
7801 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7802 *
7803 * @param pVCpu The cross context virtual CPU structure.
7804 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7805 * out-of-sync. Make sure to update the required fields
7806 * before using them.
7807 * @param uVector The software interrupt vector number.
7808 * @param cbInstr The value of RIP that is to be pushed on the guest
7809 * stack.
7810 */
7811DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7812{
7813 NOREF(pMixedCtx);
7814 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7815 if ( uVector == X86_XCPT_BP
7816 || uVector == X86_XCPT_OF)
7817 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7818 else
7819 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7820 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7821}
7822
7823
7824/**
7825 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7826 * stack.
7827 *
7828 * @returns Strict VBox status code (i.e. informational status codes too).
7829 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7830 * @param pVM The cross context VM structure.
7831 * @param pMixedCtx Pointer to the guest-CPU context.
7832 * @param uValue The value to push to the guest stack.
7833 */
7834DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7835{
7836 /*
7837 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7838 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7839 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7840 */
7841 if (pMixedCtx->sp == 1)
7842 return VINF_EM_RESET;
7843 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7844 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7845 AssertRC(rc);
7846 return rc;
7847}
7848
7849
7850/**
7851 * Injects an event into the guest upon VM-entry by updating the relevant fields
7852 * in the VM-entry area in the VMCS.
7853 *
7854 * @returns Strict VBox status code (i.e. informational status codes too).
7855 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7856 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7857 *
7858 * @param pVCpu The cross context virtual CPU structure.
7859 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7860 * be out-of-sync. Make sure to update the required
7861 * fields before using them.
7862 * @param u64IntInfo The VM-entry interruption-information field.
7863 * @param cbInstr The VM-entry instruction length in bytes (for
7864 * software interrupts, exceptions and privileged
7865 * software exceptions).
7866 * @param u32ErrCode The VM-entry exception error code.
7867 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7868 * @param puIntrState Pointer to the current guest interruptibility-state.
7869 * This interruptibility-state will be updated if
7870 * necessary. This cannot not be NULL.
7871 * @param fStepping Whether we're running in
7872 * hmR0VmxRunGuestCodeStep() and should return
7873 * VINF_EM_DBG_STEPPED if the event is injected
7874 * directly (register modified by us, not by
7875 * hardware on VM-entry).
7876 *
7877 * @remarks Requires CR0!
7878 * @remarks No-long-jump zone!!!
7879 */
7880static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7881 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
7882 uint32_t *puIntrState)
7883{
7884 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7885 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7886 Assert(puIntrState);
7887 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7888
7889 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7890 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7891
7892#ifdef VBOX_STRICT
7893 /* Validate the error-code-valid bit for hardware exceptions. */
7894 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7895 {
7896 switch (uVector)
7897 {
7898 case X86_XCPT_PF:
7899 case X86_XCPT_DF:
7900 case X86_XCPT_TS:
7901 case X86_XCPT_NP:
7902 case X86_XCPT_SS:
7903 case X86_XCPT_GP:
7904 case X86_XCPT_AC:
7905 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7906 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7907 /* fallthru */
7908 default:
7909 break;
7910 }
7911 }
7912#endif
7913
7914 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7915 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7916 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7917
7918 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7919
7920 /* We require CR0 to check if the guest is in real-mode. */
7921 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7922 AssertRCReturn(rc, rc);
7923
7924 /*
7925 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7926 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7927 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7928 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7929 */
7930 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7931 {
7932 PVM pVM = pVCpu->CTX_SUFF(pVM);
7933 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7934 {
7935 Assert(PDMVmmDevHeapIsEnabled(pVM));
7936 Assert(pVM->hm.s.vmx.pRealModeTSS);
7937
7938 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7939 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7940 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7941 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7942 AssertRCReturn(rc, rc);
7943 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7944
7945 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7946 size_t const cbIdtEntry = sizeof(X86IDTR16);
7947 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7948 {
7949 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7950 if (uVector == X86_XCPT_DF)
7951 return VINF_EM_RESET;
7952
7953 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7954 if (uVector == X86_XCPT_GP)
7955 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
7956
7957 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7958 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7959 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
7960 fStepping, puIntrState);
7961 }
7962
7963 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7964 uint16_t uGuestIp = pMixedCtx->ip;
7965 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7966 {
7967 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7968 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7969 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7970 }
7971 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7972 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7973
7974 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7975 X86IDTR16 IdtEntry;
7976 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7977 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7978 AssertRCReturn(rc, rc);
7979
7980 /* Construct the stack frame for the interrupt/exception handler. */
7981 VBOXSTRICTRC rcStrict;
7982 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7983 if (rcStrict == VINF_SUCCESS)
7984 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7985 if (rcStrict == VINF_SUCCESS)
7986 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7987
7988 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7989 if (rcStrict == VINF_SUCCESS)
7990 {
7991 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7992 pMixedCtx->rip = IdtEntry.offSel;
7993 pMixedCtx->cs.Sel = IdtEntry.uSel;
7994 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7995 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7996 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7997 && uVector == X86_XCPT_PF)
7998 pMixedCtx->cr2 = GCPtrFaultAddress;
7999
8000 /* If any other guest-state bits are changed here, make sure to update
8001 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
8002 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
8003 | HM_CHANGED_GUEST_RIP
8004 | HM_CHANGED_GUEST_RFLAGS
8005 | HM_CHANGED_GUEST_RSP);
8006
8007 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8008 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8009 {
8010 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8011 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8012 Log4(("Clearing inhibition due to STI.\n"));
8013 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
8014 }
8015 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8016 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
8017
8018 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8019 it, if we are returning to ring-3 before executing guest code. */
8020 pVCpu->hm.s.Event.fPending = false;
8021
8022 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8023 if (fStepping)
8024 rcStrict = VINF_EM_DBG_STEPPED;
8025 }
8026 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8027 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8028 return rcStrict;
8029 }
8030
8031 /*
8032 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8033 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8034 */
8035 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8036 }
8037
8038 /* Validate. */
8039 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8040 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8041 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8042
8043 /* Inject. */
8044 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8045 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8046 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8047 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8048
8049 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8050 && uVector == X86_XCPT_PF)
8051 pMixedCtx->cr2 = GCPtrFaultAddress;
8052
8053 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8054 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8055
8056 AssertRCReturn(rc, rc);
8057 return VINF_SUCCESS;
8058}
8059
8060
8061/**
8062 * Clears the interrupt-window exiting control in the VMCS and if necessary
8063 * clears the current event in the VMCS as well.
8064 *
8065 * @returns VBox status code.
8066 * @param pVCpu The cross context virtual CPU structure.
8067 *
8068 * @remarks Use this function only to clear events that have not yet been
8069 * delivered to the guest but are injected in the VMCS!
8070 * @remarks No-long-jump zone!!!
8071 */
8072static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8073{
8074 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8075
8076 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8077 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8078
8079 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8080 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8081}
8082
8083
8084/**
8085 * Enters the VT-x session.
8086 *
8087 * @returns VBox status code.
8088 * @param pVM The cross context VM structure.
8089 * @param pVCpu The cross context virtual CPU structure.
8090 * @param pCpu Pointer to the CPU info struct.
8091 */
8092VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8093{
8094 AssertPtr(pVM);
8095 AssertPtr(pVCpu);
8096 Assert(pVM->hm.s.vmx.fSupported);
8097 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8098 NOREF(pCpu); NOREF(pVM);
8099
8100 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8101 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8102
8103#ifdef VBOX_STRICT
8104 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8105 RTCCUINTREG uHostCR4 = ASMGetCR4();
8106 if (!(uHostCR4 & X86_CR4_VMXE))
8107 {
8108 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8109 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8110 }
8111#endif
8112
8113 /*
8114 * Load the VCPU's VMCS as the current (and active) one.
8115 */
8116 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8117 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8118 if (RT_FAILURE(rc))
8119 return rc;
8120
8121 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8122 pVCpu->hm.s.fLeaveDone = false;
8123 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8124
8125 return VINF_SUCCESS;
8126}
8127
8128
8129/**
8130 * The thread-context callback (only on platforms which support it).
8131 *
8132 * @param enmEvent The thread-context event.
8133 * @param pVCpu The cross context virtual CPU structure.
8134 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8135 * @thread EMT(pVCpu)
8136 */
8137VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8138{
8139 NOREF(fGlobalInit);
8140
8141 switch (enmEvent)
8142 {
8143 case RTTHREADCTXEVENT_OUT:
8144 {
8145 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8146 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8147 VMCPU_ASSERT_EMT(pVCpu);
8148
8149 PVM pVM = pVCpu->CTX_SUFF(pVM);
8150 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8151
8152 /* No longjmps (logger flushes, locks) in this fragile context. */
8153 VMMRZCallRing3Disable(pVCpu);
8154 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8155
8156 /*
8157 * Restore host-state (FPU, debug etc.)
8158 */
8159 if (!pVCpu->hm.s.fLeaveDone)
8160 {
8161 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8162 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8163 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
8164 pVCpu->hm.s.fLeaveDone = true;
8165 }
8166
8167 /* Leave HM context, takes care of local init (term). */
8168 int rc = HMR0LeaveCpu(pVCpu);
8169 AssertRC(rc); NOREF(rc);
8170
8171 /* Restore longjmp state. */
8172 VMMRZCallRing3Enable(pVCpu);
8173 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8174 break;
8175 }
8176
8177 case RTTHREADCTXEVENT_IN:
8178 {
8179 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8180 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8181 VMCPU_ASSERT_EMT(pVCpu);
8182
8183 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8184 VMMRZCallRing3Disable(pVCpu);
8185 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8186
8187 /* Initialize the bare minimum state required for HM. This takes care of
8188 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8189 int rc = HMR0EnterCpu(pVCpu);
8190 AssertRC(rc);
8191 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8192
8193 /* Load the active VMCS as the current one. */
8194 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8195 {
8196 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8197 AssertRC(rc); NOREF(rc);
8198 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8199 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8200 }
8201 pVCpu->hm.s.fLeaveDone = false;
8202
8203 /* Restore longjmp state. */
8204 VMMRZCallRing3Enable(pVCpu);
8205 break;
8206 }
8207
8208 default:
8209 break;
8210 }
8211}
8212
8213
8214/**
8215 * Saves the host state in the VMCS host-state.
8216 * Sets up the VM-exit MSR-load area.
8217 *
8218 * The CPU state will be loaded from these fields on every successful VM-exit.
8219 *
8220 * @returns VBox status code.
8221 * @param pVM The cross context VM structure.
8222 * @param pVCpu The cross context virtual CPU structure.
8223 *
8224 * @remarks No-long-jump zone!!!
8225 */
8226static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8227{
8228 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8229
8230 int rc = VINF_SUCCESS;
8231 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8232 {
8233 rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8234 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8235
8236 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8237 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8238
8239 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8240 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8241
8242 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8243 }
8244 return rc;
8245}
8246
8247
8248/**
8249 * Saves the host state in the VMCS host-state.
8250 *
8251 * @returns VBox status code.
8252 * @param pVM The cross context VM structure.
8253 * @param pVCpu The cross context virtual CPU structure.
8254 *
8255 * @remarks No-long-jump zone!!!
8256 */
8257VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8258{
8259 AssertPtr(pVM);
8260 AssertPtr(pVCpu);
8261
8262 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8263
8264 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8265 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8266 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8267 return hmR0VmxSaveHostState(pVM, pVCpu);
8268}
8269
8270
8271/**
8272 * Loads the guest state into the VMCS guest-state area.
8273 *
8274 * The will typically be done before VM-entry when the guest-CPU state and the
8275 * VMCS state may potentially be out of sync.
8276 *
8277 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8278 * VM-entry controls.
8279 * Sets up the appropriate VMX non-root function to execute guest code based on
8280 * the guest CPU mode.
8281 *
8282 * @returns VBox strict status code.
8283 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8284 * without unrestricted guest access and the VMMDev is not presently
8285 * mapped (e.g. EFI32).
8286 *
8287 * @param pVM The cross context VM structure.
8288 * @param pVCpu The cross context virtual CPU structure.
8289 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8290 * out-of-sync. Make sure to update the required fields
8291 * before using them.
8292 *
8293 * @remarks No-long-jump zone!!! (Disables and enables long jmps for itself,
8294 * caller disables then again on successfull return. Confusing.)
8295 */
8296static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8297{
8298 AssertPtr(pVM);
8299 AssertPtr(pVCpu);
8300 AssertPtr(pMixedCtx);
8301 HMVMX_ASSERT_PREEMPT_SAFE();
8302
8303 VMMRZCallRing3Disable(pVCpu);
8304 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8305
8306 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8307
8308 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8309
8310 /* Determine real-on-v86 mode. */
8311 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8312 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8313 && CPUMIsGuestInRealModeEx(pMixedCtx))
8314 {
8315 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8316 }
8317
8318 /*
8319 * Load the guest-state into the VMCS.
8320 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8321 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8322 */
8323 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8324 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8325
8326 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8327 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8328 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8329
8330 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8331 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8332 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8333
8334 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8335 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8336
8337 VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8338 if (rcStrict == VINF_SUCCESS)
8339 { /* likely */ }
8340 else
8341 {
8342 VMMRZCallRing3Enable(pVCpu);
8343 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8344 return rcStrict;
8345 }
8346
8347 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8348 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8349 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8350
8351 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8352 determine we don't have to swap EFER after all. */
8353 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8354 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8355
8356 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8357 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8358
8359 rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8360 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8361
8362 /*
8363 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8364 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8365 */
8366 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8367 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8368
8369 /* Clear any unused and reserved bits. */
8370 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8371
8372 VMMRZCallRing3Enable(pVCpu);
8373
8374 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8375 return rc;
8376}
8377
8378
8379/**
8380 * Loads the state shared between the host and guest into the VMCS.
8381 *
8382 * @param pVM The cross context VM structure.
8383 * @param pVCpu The cross context virtual CPU structure.
8384 * @param pCtx Pointer to the guest-CPU context.
8385 *
8386 * @remarks No-long-jump zone!!!
8387 */
8388static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8389{
8390 NOREF(pVM);
8391
8392 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8393 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8394
8395 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8396 {
8397 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8398 AssertRC(rc);
8399 }
8400
8401 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8402 {
8403 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8404 AssertRC(rc);
8405
8406 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8407 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8408 {
8409 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8410 AssertRC(rc);
8411 }
8412 }
8413
8414 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8415 {
8416 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8417 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8418 }
8419
8420 /* Loading CR0, debug state might have changed intercepts, update VMCS. */
8421 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8422 {
8423 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8424 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8425 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8426 AssertRC(rc);
8427 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8428 }
8429
8430 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8431 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8432}
8433
8434
8435/**
8436 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8437 *
8438 * @returns Strict VBox status code (i.e. informational status codes too).
8439 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8440 * without unrestricted guest access and the VMMDev is not presently
8441 * mapped (e.g. EFI32).
8442 *
8443 * @param pVM The cross context VM structure.
8444 * @param pVCpu The cross context virtual CPU structure.
8445 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8446 * out-of-sync. Make sure to update the required fields
8447 * before using them.
8448 */
8449static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8450{
8451 HMVMX_ASSERT_PREEMPT_SAFE();
8452
8453 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8454#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8455 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8456#endif
8457
8458 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8459 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8460 {
8461 rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8462 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8463 { /* likely */}
8464 else
8465 {
8466 AssertMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8467 VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8468 }
8469 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8470 }
8471 else if (HMCPU_CF_VALUE(pVCpu))
8472 {
8473 rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8474 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8475 { /* likely */}
8476 else
8477 {
8478 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8479 ("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8480 return rcStrict;
8481 }
8482 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8483 }
8484
8485 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8486 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8487 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8488 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8489 return rcStrict;
8490}
8491
8492
8493/**
8494 * Does the preparations before executing guest code in VT-x.
8495 *
8496 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8497 * recompiler/IEM. We must be cautious what we do here regarding committing
8498 * guest-state information into the VMCS assuming we assuredly execute the
8499 * guest in VT-x mode.
8500 *
8501 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8502 * the common-state (TRPM/forceflags), we must undo those changes so that the
8503 * recompiler/IEM can (and should) use them when it resumes guest execution.
8504 * Otherwise such operations must be done when we can no longer exit to ring-3.
8505 *
8506 * @returns Strict VBox status code (i.e. informational status codes too).
8507 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8508 * have been disabled.
8509 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8510 * double-fault into the guest.
8511 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8512 * dispatched directly.
8513 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8514 *
8515 * @param pVM The cross context VM structure.
8516 * @param pVCpu The cross context virtual CPU structure.
8517 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8518 * out-of-sync. Make sure to update the required fields
8519 * before using them.
8520 * @param pVmxTransient Pointer to the VMX transient structure.
8521 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8522 * us ignore some of the reasons for returning to
8523 * ring-3, and return VINF_EM_DBG_STEPPED if event
8524 * dispatching took place.
8525 */
8526static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8527{
8528 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8529
8530#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8531 PGMRZDynMapFlushAutoSet(pVCpu);
8532#endif
8533
8534 /* Check force flag actions that might require us to go back to ring-3. */
8535 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8536 if (rcStrict == VINF_SUCCESS)
8537 { /* FFs doesn't get set all the time. */ }
8538 else
8539 return rcStrict;
8540
8541#ifndef IEM_VERIFICATION_MODE_FULL
8542 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8543 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8544 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8545 {
8546 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8547 RTGCPHYS GCPhysApicBase;
8548 GCPhysApicBase = pMixedCtx->msrApicBase;
8549 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8550
8551 /* Unalias any existing mapping. */
8552 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8553 AssertRCReturn(rc, rc);
8554
8555 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8556 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGp\n", GCPhysApicBase));
8557 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8558 AssertRCReturn(rc, rc);
8559
8560 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8561 }
8562#endif /* !IEM_VERIFICATION_MODE_FULL */
8563
8564 if (TRPMHasTrap(pVCpu))
8565 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8566 uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8567
8568 /*
8569 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8570 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8571 */
8572 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
8573 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8574 { /* likely */ }
8575 else
8576 {
8577 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8578 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8579 return rcStrict;
8580 }
8581
8582 /*
8583 * Load the guest state bits, we can handle longjmps/getting preempted here.
8584 *
8585 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8586 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8587 * Hence, this needs to be done -after- injection of events.
8588 */
8589 rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8590 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8591 { /* likely */ }
8592 else
8593 return rcStrict;
8594
8595 /*
8596 * No longjmps to ring-3 from this point on!!!
8597 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8598 * This also disables flushing of the R0-logger instance (if any).
8599 */
8600 VMMRZCallRing3Disable(pVCpu);
8601
8602 /*
8603 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8604 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8605 *
8606 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8607 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8608 *
8609 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8610 * executing guest code.
8611 */
8612 pVmxTransient->fEFlags = ASMIntDisableFlags();
8613
8614 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8615 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8616 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8617 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8618 {
8619 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8620 {
8621 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8622 pVCpu->hm.s.Event.fPending = false;
8623
8624 return VINF_SUCCESS;
8625 }
8626
8627 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8628 rcStrict = VINF_EM_RAW_INTERRUPT;
8629 }
8630 else
8631 {
8632 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8633 rcStrict = VINF_EM_RAW_TO_R3;
8634 }
8635
8636 ASMSetFlags(pVmxTransient->fEFlags);
8637 VMMRZCallRing3Enable(pVCpu);
8638
8639 return rcStrict;
8640}
8641
8642
8643/**
8644 * Prepares to run guest code in VT-x and we've committed to doing so. This
8645 * means there is no backing out to ring-3 or anywhere else at this
8646 * point.
8647 *
8648 * @param pVM The cross context VM structure.
8649 * @param pVCpu The cross context virtual CPU structure.
8650 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8651 * out-of-sync. Make sure to update the required fields
8652 * before using them.
8653 * @param pVmxTransient Pointer to the VMX transient structure.
8654 *
8655 * @remarks Called with preemption disabled.
8656 * @remarks No-long-jump zone!!!
8657 */
8658static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8659{
8660 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8661 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8662 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8663
8664 /*
8665 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8666 */
8667 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8668 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8669
8670#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8671 if (!CPUMIsGuestFPUStateActive(pVCpu))
8672 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8673 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8674 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8675#endif
8676
8677 if ( pVCpu->hm.s.fPreloadGuestFpu
8678 && !CPUMIsGuestFPUStateActive(pVCpu))
8679 {
8680 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8681 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8682 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8683 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8684 }
8685
8686 /*
8687 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8688 */
8689 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8690 && pVCpu->hm.s.vmx.cMsrs > 0)
8691 {
8692 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8693 }
8694
8695 /*
8696 * Load the host state bits as we may've been preempted (only happens when
8697 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8698 */
8699 /** @todo Why should hmR0VmxSetupVMRunHandler() changing pfnStartVM have
8700 * any effect to the host state needing to be saved? */
8701 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8702 {
8703 /* This ASSUMES that pfnStartVM has been set up already. */
8704 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8705 AssertRC(rc);
8706 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
8707 }
8708 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8709
8710 /*
8711 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8712 */
8713 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8714 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8715 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8716
8717 /* Store status of the shared guest-host state at the time of VM-entry. */
8718#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8719 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8720 {
8721 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8722 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8723 }
8724 else
8725#endif
8726 {
8727 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8728 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8729 }
8730 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8731
8732 /*
8733 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8734 */
8735 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8736 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8737
8738 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8739 RTCPUID idCurrentCpu = pCpu->idCpu;
8740 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8741 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8742 {
8743 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8744 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8745 }
8746
8747 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8748 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8749 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8750 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8751
8752 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8753
8754 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8755 to start executing. */
8756
8757 /*
8758 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8759 */
8760 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8761 {
8762 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8763 {
8764 bool fMsrUpdated;
8765 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8766 AssertRC(rc2);
8767 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8768
8769 rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8770 &fMsrUpdated);
8771 AssertRC(rc2);
8772 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8773
8774 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8775 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8776 }
8777 else
8778 {
8779 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8780 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8781 }
8782 }
8783
8784#ifdef VBOX_STRICT
8785 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8786 hmR0VmxCheckHostEferMsr(pVCpu);
8787 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8788#endif
8789#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8790 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8791 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8792 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8793#endif
8794}
8795
8796
8797/**
8798 * Performs some essential restoration of state after running guest code in
8799 * VT-x.
8800 *
8801 * @param pVM The cross context VM structure.
8802 * @param pVCpu The cross context virtual CPU structure.
8803 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8804 * out-of-sync. Make sure to update the required fields
8805 * before using them.
8806 * @param pVmxTransient Pointer to the VMX transient structure.
8807 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8808 *
8809 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8810 *
8811 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8812 * unconditionally when it is safe to do so.
8813 */
8814static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8815{
8816 NOREF(pVM);
8817
8818 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8819
8820 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8821 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8822 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8823 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8824 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8825 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8826
8827 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8828 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
8829
8830 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8831 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8832 Assert(!ASMIntAreEnabled());
8833 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8834
8835#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8836 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVM, pVCpu))
8837 {
8838 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8839 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8840 }
8841#endif
8842
8843#if HC_ARCH_BITS == 64
8844 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8845#endif
8846 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8847#ifdef VBOX_STRICT
8848 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8849#endif
8850 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8851 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8852
8853 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8854 uint32_t uExitReason;
8855 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8856 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8857 AssertRC(rc);
8858 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8859 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8860
8861 /* Update the VM-exit history array. */
8862 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8863
8864 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8865 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8866 {
8867 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8868 pVmxTransient->fVMEntryFailed));
8869 return;
8870 }
8871
8872 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8873 {
8874 /** @todo We can optimize this by only syncing with our force-flags when
8875 * really needed and keeping the VMCS state as it is for most
8876 * VM-exits. */
8877 /* Update the guest interruptibility-state from the VMCS. */
8878 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8879
8880#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8881 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8882 AssertRC(rc);
8883#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8884 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8885 AssertRC(rc);
8886#endif
8887
8888 /*
8889 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8890 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8891 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8892 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8893 */
8894 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8895 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8896 {
8897 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8898 AssertRC(rc);
8899 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8900 }
8901 }
8902}
8903
8904
8905/**
8906 * Runs the guest code using VT-x the normal way.
8907 *
8908 * @returns VBox status code.
8909 * @param pVM The cross context VM structure.
8910 * @param pVCpu The cross context virtual CPU structure.
8911 * @param pCtx Pointer to the guest-CPU context.
8912 *
8913 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8914 */
8915static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8916{
8917 VMXTRANSIENT VmxTransient;
8918 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8919 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8920 uint32_t cLoops = 0;
8921
8922 for (;; cLoops++)
8923 {
8924 Assert(!HMR0SuspendPending());
8925 HMVMX_ASSERT_CPU_SAFE();
8926
8927 /* Preparatory work for running guest code, this may force us to return
8928 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8929 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8930 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8931 if (rcStrict != VINF_SUCCESS)
8932 break;
8933
8934 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8935 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8936 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8937
8938 /* Restore any residual host-state and save any bits shared between host
8939 and guest into the guest-CPU state. Re-enables interrupts! */
8940 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
8941
8942 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8943 if (RT_SUCCESS(rcRun))
8944 { /* very likely */ }
8945 else
8946 {
8947 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8948 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
8949 return rcRun;
8950 }
8951
8952 /* Profile the VM-exit. */
8953 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8954 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8955 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8956 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8957 HMVMX_START_EXIT_DISPATCH_PROF();
8958
8959 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8960
8961 /* Handle the VM-exit. */
8962#ifdef HMVMX_USE_FUNCTION_TABLE
8963 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8964#else
8965 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8966#endif
8967 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8968 if (rcStrict == VINF_SUCCESS)
8969 {
8970 if (cLoops <= pVM->hm.s.cMaxResumeLoops)
8971 continue; /* likely */
8972 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8973 rcStrict = VINF_EM_RAW_INTERRUPT;
8974 }
8975 break;
8976 }
8977
8978 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8979 return rcStrict;
8980}
8981
8982
8983
8984/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8985 * probes.
8986 *
8987 * The following few functions and associated structure contains the bloat
8988 * necessary for providing detailed debug events and dtrace probes as well as
8989 * reliable host side single stepping. This works on the principle of
8990 * "subclassing" the normal execution loop and workers. We replace the loop
8991 * method completely and override selected helpers to add necessary adjustments
8992 * to their core operation.
8993 *
8994 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8995 * any performance for debug and analysis features.
8996 *
8997 * @{
8998 */
8999
9000typedef struct VMXRUNDBGSTATE
9001{
9002 /** The RIP we started executing at. This is for detecting that we stepped. */
9003 uint64_t uRipStart;
9004 /** The CS we started executing with. */
9005 uint16_t uCsStart;
9006
9007 /** Whether we've actually modified the 1st execution control field. */
9008 bool fModifiedProcCtls : 1;
9009 /** Whether we've actually modified the 2nd execution control field. */
9010 bool fModifiedProcCtls2 : 1;
9011 /** Whether we've actually modified the exception bitmap. */
9012 bool fModifiedXcptBitmap : 1;
9013
9014 /** We desire the modified the CR0 mask to be cleared. */
9015 bool fClearCr0Mask : 1;
9016 /** We desire the modified the CR4 mask to be cleared. */
9017 bool fClearCr4Mask : 1;
9018 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9019 uint32_t fCpe1Extra;
9020 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9021 uint32_t fCpe1Unwanted;
9022 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9023 uint32_t fCpe2Extra;
9024 /** Extra stuff we need in */
9025 uint32_t bmXcptExtra;
9026 /** The sequence number of the Dtrace provider settings the state was
9027 * configured against. */
9028 uint32_t uDtraceSettingsSeqNo;
9029 /** Exits to check (one bit per exit). */
9030 uint32_t bmExitsToCheck[3];
9031
9032 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9033 uint32_t fProcCtlsInitial;
9034 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9035 uint32_t fProcCtls2Initial;
9036 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9037 uint32_t bmXcptInitial;
9038} VMXRUNDBGSTATE;
9039AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9040typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9041
9042
9043/**
9044 * Initializes the VMXRUNDBGSTATE structure.
9045 *
9046 * @param pVCpu The cross context virtual CPU structure of the
9047 * calling EMT.
9048 * @param pCtx The CPU register context to go with @a pVCpu.
9049 * @param pDbgState The structure to initialize.
9050 */
9051DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9052{
9053 pDbgState->uRipStart = pCtx->rip;
9054 pDbgState->uCsStart = pCtx->cs.Sel;
9055
9056 pDbgState->fModifiedProcCtls = false;
9057 pDbgState->fModifiedProcCtls2 = false;
9058 pDbgState->fModifiedXcptBitmap = false;
9059 pDbgState->fClearCr0Mask = false;
9060 pDbgState->fClearCr4Mask = false;
9061 pDbgState->fCpe1Extra = 0;
9062 pDbgState->fCpe1Unwanted = 0;
9063 pDbgState->fCpe2Extra = 0;
9064 pDbgState->bmXcptExtra = 0;
9065 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9066 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9067 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9068}
9069
9070
9071/**
9072 * Updates the VMSC fields with changes requested by @a pDbgState.
9073 *
9074 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9075 * immediately before executing guest code, i.e. when interrupts are disabled.
9076 * We don't check status codes here as we cannot easily assert or return in the
9077 * latter case.
9078 *
9079 * @param pVCpu The cross context virtual CPU structure.
9080 * @param pDbgState The debug state.
9081 */
9082DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9083{
9084 /*
9085 * Ensure desired flags in VMCS control fields are set.
9086 * (Ignoring write failure here, as we're committed and it's just debug extras.)
9087 *
9088 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9089 * there should be no stale data in pCtx at this point.
9090 */
9091 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9092 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9093 {
9094 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
9095 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9096 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9097 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9098 pDbgState->fModifiedProcCtls = true;
9099 }
9100
9101 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9102 {
9103 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
9104 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9105 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9106 pDbgState->fModifiedProcCtls2 = true;
9107 }
9108
9109 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9110 {
9111 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
9112 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9113 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9114 pDbgState->fModifiedXcptBitmap = true;
9115 }
9116
9117 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9118 {
9119 pVCpu->hm.s.vmx.u32CR0Mask = 0;
9120 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9121 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9122 }
9123
9124 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9125 {
9126 pVCpu->hm.s.vmx.u32CR4Mask = 0;
9127 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9128 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9129 }
9130}
9131
9132
9133DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9134{
9135 /*
9136 * Restore exit control settings as we may not reenter this function the
9137 * next time around.
9138 */
9139 /* We reload the initial value, trigger what we can of recalculations the
9140 next time around. From the looks of things, that's all that's required atm. */
9141 if (pDbgState->fModifiedProcCtls)
9142 {
9143 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9144 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9145 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9146 AssertRCReturn(rc2, rc2);
9147 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9148 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0 | HM_CHANGED_GUEST_DEBUG);
9149 }
9150
9151 /* We're currently the only ones messing with this one, so just restore the
9152 cached value and reload the field. */
9153 if ( pDbgState->fModifiedProcCtls2
9154 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9155 {
9156 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9157 AssertRCReturn(rc2, rc2);
9158 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9159 }
9160
9161 /* If we've modified the exception bitmap, we restore it and trigger
9162 reloading and partial recalculation the next time around. */
9163 if (pDbgState->fModifiedXcptBitmap)
9164 {
9165 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9166 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS | HM_CHANGED_GUEST_CR0);
9167 }
9168
9169 /* We assume hmR0VmxLoadSharedCR0 will recalculate and load the CR0 mask. */
9170 if (pDbgState->fClearCr0Mask)
9171 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9172
9173 /* We assume hmR0VmxLoadGuestCR3AndCR4 will recalculate and load the CR4 mask. */
9174 if (pDbgState->fClearCr4Mask)
9175 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9176
9177 return rcStrict;
9178}
9179
9180
9181/**
9182 * Configures VM-exit controls for current DBGF and DTrace settings.
9183 *
9184 * This updates @a pDbgState and the VMCS execution control fields to reflect
9185 * the necessary exits demanded by DBGF and DTrace.
9186 *
9187 * @param pVM The cross context VM structure.
9188 * @param pVCpu The cross context virtual CPU structure.
9189 * @param pCtx Pointer to the guest-CPU context.
9190 * @param pDbgState The debug state.
9191 * @param pVmxTransient Pointer to the VMX transient structure. May update
9192 * fUpdateTscOffsettingAndPreemptTimer.
9193 */
9194static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9195 PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9196{
9197 /*
9198 * Take down the dtrace serial number so we can spot changes.
9199 */
9200 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9201 ASMCompilerBarrier();
9202
9203 /*
9204 * We'll rebuild most of the middle block of data members (holding the
9205 * current settings) as we go along here, so start by clearing it all.
9206 */
9207 pDbgState->bmXcptExtra = 0;
9208 pDbgState->fCpe1Extra = 0;
9209 pDbgState->fCpe1Unwanted = 0;
9210 pDbgState->fCpe2Extra = 0;
9211 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9212 pDbgState->bmExitsToCheck[i] = 0;
9213
9214 /*
9215 * Software interrupts (INT XXh) - no idea how to trigger these...
9216 */
9217 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9218 || VBOXVMM_INT_SOFTWARE_ENABLED())
9219 {
9220 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9221 }
9222
9223 /*
9224 * Exception bitmap and XCPT events+probes.
9225 */
9226 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9227 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9228 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9229
9230 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9231 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9232 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9233 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9234 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9235 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9236 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9237 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9238 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9239 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9240 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9241 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9242 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9243 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9244 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9245 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9246 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9247 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9248
9249 if (pDbgState->bmXcptExtra)
9250 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9251
9252 /*
9253 * Process events and probes for VM exits, making sure we get the wanted exits.
9254 *
9255 * Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9256 * So, when adding/changing/removing please don't forget to update it.
9257 *
9258 * Some of the macros are picking up local variables to save horizontal space,
9259 * (being able to see it in a table is the lesser evil here).
9260 */
9261#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9262 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9263 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9264#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9265 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9266 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9267 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9268 } else do { } while (0)
9269#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9270 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9271 { \
9272 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9273 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9274 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9275 } else do { } while (0)
9276#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9277 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9278 { \
9279 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9280 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9281 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9282 } else do { } while (0)
9283#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9284 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9285 { \
9286 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9287 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9288 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9289 } else do { } while (0)
9290
9291 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9292 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9293 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9294 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9295 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9296
9297 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9298 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9299 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9300 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9301 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9302 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9303 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9304 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9305 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9306 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9307 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9308 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9309 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9310 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9311 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9312 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9313 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9314 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9315 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9316 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9317 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9318 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9319 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9320 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9321 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9322 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9323 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9324 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9325 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9326 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9327 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9328 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9329 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9330 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9331 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9332 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9333
9334 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9335 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9336 {
9337 int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9338 rc2 |= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9339 rc2 |= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9340 AssertRC(rc2);
9341
9342#if 0 /** @todo fix me */
9343 pDbgState->fClearCr0Mask = true;
9344 pDbgState->fClearCr4Mask = true;
9345#endif
9346 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9347 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9348 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9349 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9350 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9351 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9352 require clearing here and in the loop if we start using it. */
9353 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9354 }
9355 else
9356 {
9357 if (pDbgState->fClearCr0Mask)
9358 {
9359 pDbgState->fClearCr0Mask = false;
9360 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9361 }
9362 if (pDbgState->fClearCr4Mask)
9363 {
9364 pDbgState->fClearCr4Mask = false;
9365 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9366 }
9367 }
9368 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9369 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9370
9371 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9372 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9373 {
9374 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9375 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9376 }
9377 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9378 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9379
9380 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9381 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9382 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9383 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9384 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9385 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9386 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9387 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9388#if 0 /** @todo too slow, fix handler. */
9389 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9390#endif
9391 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9392
9393 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9394 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9395 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9396 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9397 {
9398 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9399 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9400 }
9401 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9402 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9403 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9404 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9405
9406 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9407 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9408 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9409 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9410 {
9411 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9412 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9413 }
9414 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9415 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9416 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9417 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9418
9419 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9420 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9421 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9422 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9423 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9424 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9425 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9426 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9427 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9428 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9429 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9430 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9431 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9432 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9433 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9434 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9435 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9436 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9437 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9438 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9439 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9440 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9441
9442#undef IS_EITHER_ENABLED
9443#undef SET_ONLY_XBM_IF_EITHER_EN
9444#undef SET_CPE1_XBM_IF_EITHER_EN
9445#undef SET_CPEU_XBM_IF_EITHER_EN
9446#undef SET_CPE2_XBM_IF_EITHER_EN
9447
9448 /*
9449 * Sanitize the control stuff.
9450 */
9451 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9452 if (pDbgState->fCpe2Extra)
9453 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9454 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9455 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9456 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9457 {
9458 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9459 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9460 }
9461
9462 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9463 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9464 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9465 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9466}
9467
9468
9469/**
9470 * Fires off DBGF events and dtrace probes for an exit, when it's appropriate.
9471 *
9472 * The caller has checked exit against the VMXRUNDBGSTATE::bmExitsToCheck
9473 * bitmap. The caller has checked for NMIs already, so we don't have to do that
9474 * either.
9475 *
9476 * @returns Strict VBox status code (i.e. informational status codes too).
9477 * @param pVM The cross context VM structure.
9478 * @param pVCpu The cross context virtual CPU structure.
9479 * @param pMixedCtx Pointer to the guest-CPU context.
9480 * @param pVmxTransient Pointer to the VMX-transient structure.
9481 * @param uExitReason The VM-exit reason.
9482 *
9483 * @remarks The name of this function is displayed by dtrace, so keep it short
9484 * and to the point. No longer than 33 chars long, please.
9485 */
9486static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9487 PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9488{
9489 /*
9490 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9491 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9492 *
9493 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9494 * does. Must add/change/remove both places. Same ordering, please.
9495 *
9496 * Added/removed events must also be reflected in the next section
9497 * where we dispatch dtrace events.
9498 */
9499 bool fDtrace1 = false;
9500 bool fDtrace2 = false;
9501 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9502 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9503 uint32_t uEventArg = 0;
9504#define SET_EXIT(a_EventSubName) \
9505 do { \
9506 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9507 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9508 } while (0)
9509#define SET_BOTH(a_EventSubName) \
9510 do { \
9511 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9512 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9513 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9514 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9515 } while (0)
9516 switch (uExitReason)
9517 {
9518 case VMX_EXIT_MTF:
9519 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9520
9521 case VMX_EXIT_XCPT_OR_NMI:
9522 {
9523 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9524 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9525 {
9526 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9527 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9528 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9529 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9530 {
9531 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9532 {
9533 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9534 uEventArg = pVmxTransient->uExitIntErrorCode;
9535 }
9536 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9537 switch (enmEvent1)
9538 {
9539 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9540 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9541 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9542 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9543 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9544 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9545 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9546 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9547 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9548 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9549 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9550 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9551 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9552 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9553 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9554 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9555 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9556 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9557 default: break;
9558 }
9559 }
9560 else
9561 AssertFailed();
9562 break;
9563
9564 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9565 uEventArg = idxVector;
9566 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9567 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9568 break;
9569 }
9570 break;
9571 }
9572
9573 case VMX_EXIT_TRIPLE_FAULT:
9574 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9575 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9576 break;
9577 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9578 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9579 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9580 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9581 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9582
9583 /* Instruction specific VM-exits: */
9584 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9585 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9586 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9587 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9588 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9589 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9590 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9591 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9592 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9593 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9594 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9595 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9596 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9597 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9598 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9599 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9600 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9601 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9602 case VMX_EXIT_MOV_CRX:
9603 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9604/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
9605* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
9606 if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
9607 == VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
9608 SET_BOTH(CRX_READ);
9609 else
9610 SET_BOTH(CRX_WRITE);
9611 uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
9612 break;
9613 case VMX_EXIT_MOV_DRX:
9614 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9615 if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
9616 == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
9617 SET_BOTH(DRX_READ);
9618 else
9619 SET_BOTH(DRX_WRITE);
9620 uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
9621 break;
9622 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9623 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9624 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9625 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9626 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9627 case VMX_EXIT_XDTR_ACCESS:
9628 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9629 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9630 {
9631 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9632 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9633 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9634 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9635 }
9636 break;
9637
9638 case VMX_EXIT_TR_ACCESS:
9639 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9640 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9641 {
9642 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9643 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9644 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9645 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9646 }
9647 break;
9648
9649 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9650 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9651 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9652 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9653 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9654 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9655 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9656 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9657 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9658 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9659 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9660
9661 /* Events that aren't relevant at this point. */
9662 case VMX_EXIT_EXT_INT:
9663 case VMX_EXIT_INT_WINDOW:
9664 case VMX_EXIT_NMI_WINDOW:
9665 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9666 case VMX_EXIT_PREEMPT_TIMER:
9667 case VMX_EXIT_IO_INSTR:
9668 break;
9669
9670 /* Errors and unexpected events. */
9671 case VMX_EXIT_INIT_SIGNAL:
9672 case VMX_EXIT_SIPI:
9673 case VMX_EXIT_IO_SMI:
9674 case VMX_EXIT_SMI:
9675 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9676 case VMX_EXIT_ERR_MSR_LOAD:
9677 case VMX_EXIT_ERR_MACHINE_CHECK:
9678 break;
9679
9680 default:
9681 AssertMsgFailed(("Unexpected exit=%#x\n", uExitReason));
9682 break;
9683 }
9684#undef SET_BOTH
9685#undef SET_EXIT
9686
9687 /*
9688 * Dtrace tracepoints go first. We do them here at once so we don't
9689 * have to copy the guest state saving and stuff a few dozen times.
9690 * Down side is that we've got to repeat the switch, though this time
9691 * we use enmEvent since the probes are a subset of what DBGF does.
9692 */
9693 if (fDtrace1 || fDtrace2)
9694 {
9695 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9696 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9697 switch (enmEvent1)
9698 {
9699 /** @todo consider which extra parameters would be helpful for each probe. */
9700 case DBGFEVENT_END: break;
9701 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9702 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9703 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9704 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9705 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9706 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9707 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9708 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9709 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9710 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9711 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9712 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9713 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9714 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9715 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9716 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9717 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9718 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9719 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9720 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9721 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9722 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9723 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9724 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9725 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9726 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9727 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9728 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9729 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9730 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9731 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9732 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9733 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9734 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9735 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9736 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9737 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9738 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9739 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9740 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9741 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9742 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9743 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9744 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9745 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9746 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9747 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9748 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9749 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9750 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9751 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9752 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9753 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9754 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9755 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9756 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9757 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9758 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9759 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9760 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9761 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9762 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9763 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9764 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9765 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9766 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9767 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9768 }
9769 switch (enmEvent2)
9770 {
9771 /** @todo consider which extra parameters would be helpful for each probe. */
9772 case DBGFEVENT_END: break;
9773 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9774 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9775 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9776 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9777 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9778 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9779 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9780 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9781 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9782 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9783 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9784 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9785 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9786 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9787 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9788 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9789 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9790 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9791 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9792 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9793 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9794 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9795 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9796 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9797 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9798 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9799 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9800 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9801 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9802 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9803 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9804 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9805 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9806 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9807 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9808 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9809 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9810 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9811 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9812 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9813 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9814 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9815 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9816 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9817 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9818 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9819 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9820 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9821 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9822 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9823 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9824 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9825 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9826 }
9827 }
9828
9829 /*
9830 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9831 * the DBGF call will do a full check).
9832 *
9833 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9834 * Note! If we have to events, we prioritize the first, i.e. the instruction
9835 * one, in order to avoid event nesting.
9836 */
9837 if ( enmEvent1 != DBGFEVENT_END
9838 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9839 {
9840 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9841 if (rcStrict != VINF_SUCCESS)
9842 return rcStrict;
9843 }
9844 else if ( enmEvent2 != DBGFEVENT_END
9845 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9846 {
9847 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9848 if (rcStrict != VINF_SUCCESS)
9849 return rcStrict;
9850 }
9851
9852 return VINF_SUCCESS;
9853}
9854
9855
9856/**
9857 * Single-stepping VM-exit filtering.
9858 *
9859 * This is preprocessing the exits and deciding whether we've gotten far enough
9860 * to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit handling is
9861 * performed.
9862 *
9863 * @returns Strict VBox status code (i.e. informational status codes too).
9864 * @param pVM The cross context VM structure.
9865 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9866 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9867 * out-of-sync. Make sure to update the required
9868 * fields before using them.
9869 * @param pVmxTransient Pointer to the VMX-transient structure.
9870 * @param uExitReason The VM-exit reason.
9871 * @param pDbgState The debug state.
9872 */
9873DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9874 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9875{
9876 /*
9877 * Expensive (saves context) generic dtrace exit probe.
9878 */
9879 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9880 { /* more likely */ }
9881 else
9882 {
9883 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9884 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9885 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9886 }
9887
9888 /*
9889 * Check for host NMI, just to get that out of the way.
9890 */
9891 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9892 { /* normally likely */ }
9893 else
9894 {
9895 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9896 AssertRCReturn(rc2, rc2);
9897 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9898 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9899 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9900 }
9901
9902 /*
9903 * Check for single stepping event if we're stepping.
9904 */
9905 if (pVCpu->hm.s.fSingleInstruction)
9906 {
9907 switch (uExitReason)
9908 {
9909 case VMX_EXIT_MTF:
9910 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9911
9912 /* Various events: */
9913 case VMX_EXIT_XCPT_OR_NMI:
9914 case VMX_EXIT_EXT_INT:
9915 case VMX_EXIT_TRIPLE_FAULT:
9916 case VMX_EXIT_INT_WINDOW:
9917 case VMX_EXIT_NMI_WINDOW:
9918 case VMX_EXIT_TASK_SWITCH:
9919 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9920 case VMX_EXIT_APIC_ACCESS:
9921 case VMX_EXIT_EPT_VIOLATION:
9922 case VMX_EXIT_EPT_MISCONFIG:
9923 case VMX_EXIT_PREEMPT_TIMER:
9924
9925 /* Instruction specific VM-exits: */
9926 case VMX_EXIT_CPUID:
9927 case VMX_EXIT_GETSEC:
9928 case VMX_EXIT_HLT:
9929 case VMX_EXIT_INVD:
9930 case VMX_EXIT_INVLPG:
9931 case VMX_EXIT_RDPMC:
9932 case VMX_EXIT_RDTSC:
9933 case VMX_EXIT_RSM:
9934 case VMX_EXIT_VMCALL:
9935 case VMX_EXIT_VMCLEAR:
9936 case VMX_EXIT_VMLAUNCH:
9937 case VMX_EXIT_VMPTRLD:
9938 case VMX_EXIT_VMPTRST:
9939 case VMX_EXIT_VMREAD:
9940 case VMX_EXIT_VMRESUME:
9941 case VMX_EXIT_VMWRITE:
9942 case VMX_EXIT_VMXOFF:
9943 case VMX_EXIT_VMXON:
9944 case VMX_EXIT_MOV_CRX:
9945 case VMX_EXIT_MOV_DRX:
9946 case VMX_EXIT_IO_INSTR:
9947 case VMX_EXIT_RDMSR:
9948 case VMX_EXIT_WRMSR:
9949 case VMX_EXIT_MWAIT:
9950 case VMX_EXIT_MONITOR:
9951 case VMX_EXIT_PAUSE:
9952 case VMX_EXIT_XDTR_ACCESS:
9953 case VMX_EXIT_TR_ACCESS:
9954 case VMX_EXIT_INVEPT:
9955 case VMX_EXIT_RDTSCP:
9956 case VMX_EXIT_INVVPID:
9957 case VMX_EXIT_WBINVD:
9958 case VMX_EXIT_XSETBV:
9959 case VMX_EXIT_RDRAND:
9960 case VMX_EXIT_INVPCID:
9961 case VMX_EXIT_VMFUNC:
9962 case VMX_EXIT_RDSEED:
9963 case VMX_EXIT_XSAVES:
9964 case VMX_EXIT_XRSTORS:
9965 {
9966 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9967 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9968 AssertRCReturn(rc2, rc2);
9969 if ( pMixedCtx->rip != pDbgState->uRipStart
9970 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
9971 return VINF_EM_DBG_STEPPED;
9972 break;
9973 }
9974
9975 /* Errors and unexpected events: */
9976 case VMX_EXIT_INIT_SIGNAL:
9977 case VMX_EXIT_SIPI:
9978 case VMX_EXIT_IO_SMI:
9979 case VMX_EXIT_SMI:
9980 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9981 case VMX_EXIT_ERR_MSR_LOAD:
9982 case VMX_EXIT_ERR_MACHINE_CHECK:
9983 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9984 break;
9985
9986 default:
9987 AssertMsgFailed(("Unexpected exit=%#x\n", uExitReason));
9988 break;
9989 }
9990 }
9991
9992 /*
9993 * Check for debugger event breakpoints and dtrace probes.
9994 */
9995 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9996 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9997 {
9998 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9999 if (rcStrict != VINF_SUCCESS)
10000 return rcStrict;
10001 }
10002
10003 /*
10004 * Normal processing.
10005 */
10006#ifdef HMVMX_USE_FUNCTION_TABLE
10007 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
10008#else
10009 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10010#endif
10011}
10012
10013
10014/**
10015 * Single steps guest code using VT-x.
10016 *
10017 * @returns Strict VBox status code (i.e. informational status codes too).
10018 * @param pVM The cross context VM structure.
10019 * @param pVCpu The cross context virtual CPU structure.
10020 * @param pCtx Pointer to the guest-CPU context.
10021 *
10022 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10023 */
10024static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10025{
10026 VMXTRANSIENT VmxTransient;
10027 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10028
10029 /* Set HMCPU indicators. */
10030 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10031 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
10032 pVCpu->hm.s.fDebugWantRdTscExit = false;
10033 pVCpu->hm.s.fUsingDebugLoop = true;
10034
10035 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10036 VMXRUNDBGSTATE DbgState;
10037 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10038 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10039
10040 /*
10041 * The loop.
10042 */
10043 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10044 for (uint32_t cLoops = 0; ; cLoops++)
10045 {
10046 Assert(!HMR0SuspendPending());
10047 HMVMX_ASSERT_CPU_SAFE();
10048 bool fStepping = pVCpu->hm.s.fSingleInstruction;
10049
10050 /*
10051 * Preparatory work for running guest code, this may force us to return
10052 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10053 */
10054 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10055 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10056 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10057 if (rcStrict != VINF_SUCCESS)
10058 break;
10059
10060 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10061 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10062
10063 /*
10064 * Now we can run the guest code.
10065 */
10066 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10067
10068 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10069
10070 /*
10071 * Restore any residual host-state and save any bits shared between host
10072 * and guest into the guest-CPU state. Re-enables interrupts!
10073 */
10074 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
10075
10076 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10077 if (RT_SUCCESS(rcRun))
10078 { /* very likely */ }
10079 else
10080 {
10081 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10082 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10083 return rcRun;
10084 }
10085
10086 /* Profile the VM-exit. */
10087 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10088 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10089 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10090 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10091 HMVMX_START_EXIT_DISPATCH_PROF();
10092
10093 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10094
10095 /*
10096 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10097 */
10098 rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10099 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10100 if (rcStrict != VINF_SUCCESS)
10101 break;
10102 if (cLoops > pVM->hm.s.cMaxResumeLoops)
10103 {
10104 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10105 rcStrict = VINF_EM_RAW_INTERRUPT;
10106 break;
10107 }
10108
10109 /*
10110 * Stepping: Did the RIP change, if so, consider it a single step.
10111 * Otherwise, make sure one of the TFs gets set.
10112 */
10113 if (fStepping)
10114 {
10115 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10116 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10117 AssertRCReturn(rc2, rc2);
10118 if ( pCtx->rip != DbgState.uRipStart
10119 || pCtx->cs.Sel != DbgState.uCsStart)
10120 {
10121 rcStrict = VINF_EM_DBG_STEPPED;
10122 break;
10123 }
10124 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10125 }
10126
10127 /*
10128 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
10129 */
10130 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10131 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10132 }
10133
10134 /*
10135 * Clear the X86_EFL_TF if necessary.
10136 */
10137 if (pVCpu->hm.s.fClearTrapFlag)
10138 {
10139 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10140 AssertRCReturn(rc2, rc2);
10141 pVCpu->hm.s.fClearTrapFlag = false;
10142 pCtx->eflags.Bits.u1TF = 0;
10143 }
10144 /** @todo there seems to be issues with the resume flag when the monitor trap
10145 * flag is pending without being used. Seen early in bios init when
10146 * accessing APIC page in protected mode. */
10147
10148 /*
10149 * Restore VM-exit control settings as we may not reenter this function the
10150 * next time around.
10151 */
10152 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10153
10154 /* Restore HMCPU indicators. */
10155 pVCpu->hm.s.fUsingDebugLoop = false;
10156 pVCpu->hm.s.fDebugWantRdTscExit = false;
10157 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10158
10159 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10160 return rcStrict;
10161}
10162
10163
10164/** @} */
10165
10166
10167/**
10168 * Checks if any expensive dtrace probes are enabled and we should go to the
10169 * debug loop.
10170 *
10171 * @returns true if we should use debug loop, false if not.
10172 */
10173static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10174{
10175 /* It's probably faster to OR the raw 32-bit counter variables together.
10176 Since the variables are in an array and the probes are next to one
10177 another (more or less), we have good locality. So, better read
10178 eight-nine cache lines ever time and only have one conditional, than
10179 128+ conditionals, right? */
10180 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10181 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10182 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10183 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10184 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10185 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10186 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10187 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10188 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10189 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10190 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10191 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10192 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10193 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10194 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10195 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10196 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10197 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10198 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10199 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10200 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10201 ) != 0
10202 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10203 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10204 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10205 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10206 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10207 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10208 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10209 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10210 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10211 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10212 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10213 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10214 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10215 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10216 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10217 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10218 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10219 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10220 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10221 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10222 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10223 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10224 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10225 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10226 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10227 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10228 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10229 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10230 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10231 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10232 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10233 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10234 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10235 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10236 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10237 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10238 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10239 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10240 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10241 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10242 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10243 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10244 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10245 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10246 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10247 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10248 ) != 0
10249 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10250 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10251 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10252 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10253 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10254 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10255 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10256 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10257 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10258 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10259 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10260 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10261 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10262 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10263 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10264 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10265 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10266 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10267 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10268 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10269 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10270 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10271 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10272 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10273 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10274 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10275 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10276 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10277 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10278 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10279 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10280 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10281 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10282 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10283 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10284 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10285 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10286 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10287 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10288 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10289 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10290 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10291 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10292 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10293 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10294 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10295 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10296 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10297 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10298 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10299 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10300 ) != 0;
10301}
10302
10303
10304/**
10305 * Runs the guest code using VT-x.
10306 *
10307 * @returns Strict VBox status code (i.e. informational status codes too).
10308 * @param pVM The cross context VM structure.
10309 * @param pVCpu The cross context virtual CPU structure.
10310 * @param pCtx Pointer to the guest-CPU context.
10311 */
10312VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10313{
10314 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10315 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10316 HMVMX_ASSERT_PREEMPT_SAFE();
10317
10318 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10319
10320 VBOXSTRICTRC rcStrict;
10321 if ( !pVCpu->hm.s.fUseDebugLoop
10322 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10323 && !DBGFIsStepping(pVCpu) )
10324 rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10325 else
10326 rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10327
10328 if (rcStrict == VERR_EM_INTERPRETER)
10329 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10330 else if (rcStrict == VINF_EM_RESET)
10331 rcStrict = VINF_EM_TRIPLE_FAULT;
10332
10333 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10334 if (RT_FAILURE(rc2))
10335 {
10336 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10337 rcStrict = rc2;
10338 }
10339 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10340 return rcStrict;
10341}
10342
10343
10344#ifndef HMVMX_USE_FUNCTION_TABLE
10345DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10346{
10347# ifdef DEBUG_ramshankar
10348# define RETURN_EXIT_CALL(a_CallExpr) \
10349 do { \
10350 int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10351 VBOXSTRICTRC rcStrict = a_CallExpr; \
10352 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10353 return rcStrict; \
10354 } while (0)
10355# else
10356# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10357# endif
10358 switch (rcReason)
10359 {
10360 case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10361 case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10362 case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10363 case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10364 case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10365 case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10366 case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10367 case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10368 case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10369 case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10370 case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10371 case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10372 case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10373 case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10374 case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10375 case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10376 case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10377 case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10378 case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10379 case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10380 case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10381 case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10382 case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10383 case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10384 case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10385 case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10386 case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10387 case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10388 case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10389 case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10390 case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10391 case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10392 case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10393 case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10394
10395 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10396 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10397 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10398 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10399 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10400 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10401 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10402 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10403 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10404
10405 case VMX_EXIT_VMCLEAR:
10406 case VMX_EXIT_VMLAUNCH:
10407 case VMX_EXIT_VMPTRLD:
10408 case VMX_EXIT_VMPTRST:
10409 case VMX_EXIT_VMREAD:
10410 case VMX_EXIT_VMRESUME:
10411 case VMX_EXIT_VMWRITE:
10412 case VMX_EXIT_VMXOFF:
10413 case VMX_EXIT_VMXON:
10414 case VMX_EXIT_INVEPT:
10415 case VMX_EXIT_INVVPID:
10416 case VMX_EXIT_VMFUNC:
10417 case VMX_EXIT_XSAVES:
10418 case VMX_EXIT_XRSTORS:
10419 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10420 case VMX_EXIT_RESERVED_60:
10421 case VMX_EXIT_RDSEED: /* only spurious exits, so undefined */
10422 case VMX_EXIT_RESERVED_62:
10423 default:
10424 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10425 }
10426#undef RETURN_EXIT_CALL
10427}
10428#endif /* !HMVMX_USE_FUNCTION_TABLE */
10429
10430
10431#ifdef VBOX_STRICT
10432/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10433# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10434 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10435
10436# define HMVMX_ASSERT_PREEMPT_CPUID() \
10437 do { \
10438 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10439 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10440 } while (0)
10441
10442# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10443 do { \
10444 AssertPtr(pVCpu); \
10445 AssertPtr(pMixedCtx); \
10446 AssertPtr(pVmxTransient); \
10447 Assert(pVmxTransient->fVMEntryFailed == false); \
10448 Assert(ASMIntAreEnabled()); \
10449 HMVMX_ASSERT_PREEMPT_SAFE(); \
10450 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10451 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10452 HMVMX_ASSERT_PREEMPT_SAFE(); \
10453 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10454 HMVMX_ASSERT_PREEMPT_CPUID(); \
10455 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10456 } while (0)
10457
10458# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10459 do { \
10460 Log4Func(("\n")); \
10461 } while (0)
10462#else /* nonstrict builds: */
10463# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10464 do { \
10465 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10466 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10467 } while (0)
10468# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10469#endif
10470
10471
10472/**
10473 * Advances the guest RIP by the specified number of bytes.
10474 *
10475 * @param pVCpu The cross context virtual CPU structure.
10476 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10477 * out-of-sync. Make sure to update the required fields
10478 * before using them.
10479 * @param cbInstr Number of bytes to advance the RIP by.
10480 *
10481 * @remarks No-long-jump zone!!!
10482 */
10483DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10484{
10485 /* Advance the RIP. */
10486 pMixedCtx->rip += cbInstr;
10487 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10488
10489 /* Update interrupt inhibition. */
10490 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10491 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10492 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10493}
10494
10495
10496/**
10497 * Advances the guest RIP after reading it from the VMCS.
10498 *
10499 * @returns VBox status code, no informational status codes.
10500 * @param pVCpu The cross context virtual CPU structure.
10501 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10502 * out-of-sync. Make sure to update the required fields
10503 * before using them.
10504 * @param pVmxTransient Pointer to the VMX transient structure.
10505 *
10506 * @remarks No-long-jump zone!!!
10507 */
10508static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10509{
10510 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10511 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10512 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10513 AssertRCReturn(rc, rc);
10514
10515 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10516
10517 /*
10518 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10519 * pending debug exception field as it takes care of priority of events.
10520 *
10521 * See Intel spec. 32.2.1 "Debug Exceptions".
10522 */
10523 if ( !pVCpu->hm.s.fSingleInstruction
10524 && pMixedCtx->eflags.Bits.u1TF)
10525 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10526
10527 return VINF_SUCCESS;
10528}
10529
10530
10531/**
10532 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10533 * and update error record fields accordingly.
10534 *
10535 * @return VMX_IGS_* return codes.
10536 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10537 * wrong with the guest state.
10538 *
10539 * @param pVM The cross context VM structure.
10540 * @param pVCpu The cross context virtual CPU structure.
10541 * @param pCtx Pointer to the guest-CPU state.
10542 *
10543 * @remarks This function assumes our cache of the VMCS controls
10544 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10545 */
10546static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10547{
10548#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10549#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10550 uError = (err); \
10551 break; \
10552 } else do { } while (0)
10553
10554 int rc;
10555 uint32_t uError = VMX_IGS_ERROR;
10556 uint32_t u32Val;
10557 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10558
10559 do
10560 {
10561 /*
10562 * CR0.
10563 */
10564 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10565 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10566 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10567 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10568 if (fUnrestrictedGuest)
10569 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
10570
10571 uint32_t u32GuestCR0;
10572 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
10573 AssertRCBreak(rc);
10574 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
10575 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
10576 if ( !fUnrestrictedGuest
10577 && (u32GuestCR0 & X86_CR0_PG)
10578 && !(u32GuestCR0 & X86_CR0_PE))
10579 {
10580 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10581 }
10582
10583 /*
10584 * CR4.
10585 */
10586 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10587 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10588
10589 uint32_t u32GuestCR4;
10590 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
10591 AssertRCBreak(rc);
10592 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
10593 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
10594
10595 /*
10596 * IA32_DEBUGCTL MSR.
10597 */
10598 uint64_t u64Val;
10599 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10600 AssertRCBreak(rc);
10601 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10602 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10603 {
10604 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10605 }
10606 uint64_t u64DebugCtlMsr = u64Val;
10607
10608#ifdef VBOX_STRICT
10609 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10610 AssertRCBreak(rc);
10611 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10612#endif
10613 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10614
10615 /*
10616 * RIP and RFLAGS.
10617 */
10618 uint32_t u32Eflags;
10619#if HC_ARCH_BITS == 64
10620 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10621 AssertRCBreak(rc);
10622 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10623 if ( !fLongModeGuest
10624 || !pCtx->cs.Attr.n.u1Long)
10625 {
10626 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10627 }
10628 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10629 * must be identical if the "IA-32e mode guest" VM-entry
10630 * control is 1 and CS.L is 1. No check applies if the
10631 * CPU supports 64 linear-address bits. */
10632
10633 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10634 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10635 AssertRCBreak(rc);
10636 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10637 VMX_IGS_RFLAGS_RESERVED);
10638 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10639 u32Eflags = u64Val;
10640#else
10641 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10642 AssertRCBreak(rc);
10643 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10644 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10645#endif
10646
10647 if ( fLongModeGuest
10648 || ( fUnrestrictedGuest
10649 && !(u32GuestCR0 & X86_CR0_PE)))
10650 {
10651 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10652 }
10653
10654 uint32_t u32EntryInfo;
10655 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10656 AssertRCBreak(rc);
10657 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10658 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10659 {
10660 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10661 }
10662
10663 /*
10664 * 64-bit checks.
10665 */
10666#if HC_ARCH_BITS == 64
10667 if (fLongModeGuest)
10668 {
10669 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10670 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10671 }
10672
10673 if ( !fLongModeGuest
10674 && (u32GuestCR4 & X86_CR4_PCIDE))
10675 {
10676 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10677 }
10678
10679 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10680 * 51:32 beyond the processor's physical-address width are 0. */
10681
10682 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10683 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10684 {
10685 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10686 }
10687
10688 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10689 AssertRCBreak(rc);
10690 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10691
10692 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10693 AssertRCBreak(rc);
10694 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10695#endif
10696
10697 /*
10698 * PERF_GLOBAL MSR.
10699 */
10700 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10701 {
10702 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10703 AssertRCBreak(rc);
10704 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10705 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10706 }
10707
10708 /*
10709 * PAT MSR.
10710 */
10711 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10712 {
10713 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10714 AssertRCBreak(rc);
10715 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10716 for (unsigned i = 0; i < 8; i++)
10717 {
10718 uint8_t u8Val = (u64Val & 0xff);
10719 if ( u8Val != 0 /* UC */
10720 && u8Val != 1 /* WC */
10721 && u8Val != 4 /* WT */
10722 && u8Val != 5 /* WP */
10723 && u8Val != 6 /* WB */
10724 && u8Val != 7 /* UC- */)
10725 {
10726 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10727 }
10728 u64Val >>= 8;
10729 }
10730 }
10731
10732 /*
10733 * EFER MSR.
10734 */
10735 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10736 {
10737 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10738 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10739 AssertRCBreak(rc);
10740 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10741 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10742 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10743 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10744 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10745 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10746 || !(u32GuestCR0 & X86_CR0_PG)
10747 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10748 VMX_IGS_EFER_LMA_LME_MISMATCH);
10749 }
10750
10751 /*
10752 * Segment registers.
10753 */
10754 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10755 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10756 if (!(u32Eflags & X86_EFL_VM))
10757 {
10758 /* CS */
10759 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10760 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10761 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10762 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10763 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10764 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10765 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10766 /* CS cannot be loaded with NULL in protected mode. */
10767 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10768 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10769 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10770 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10771 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10772 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10773 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10774 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10775 else
10776 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10777
10778 /* SS */
10779 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10780 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10781 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10782 if ( !(pCtx->cr0 & X86_CR0_PE)
10783 || pCtx->cs.Attr.n.u4Type == 3)
10784 {
10785 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10786 }
10787 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10788 {
10789 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10790 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10791 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10792 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10793 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10794 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10795 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10796 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10797 }
10798
10799 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
10800 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10801 {
10802 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10803 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10804 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10805 || pCtx->ds.Attr.n.u4Type > 11
10806 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10807 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10808 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10809 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10810 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10811 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10812 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10813 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10814 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10815 }
10816 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10817 {
10818 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10819 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10820 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10821 || pCtx->es.Attr.n.u4Type > 11
10822 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10823 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10824 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10825 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10826 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10827 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10828 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10829 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10830 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10831 }
10832 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10833 {
10834 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10835 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10836 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10837 || pCtx->fs.Attr.n.u4Type > 11
10838 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10839 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10840 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10841 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10842 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10843 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10844 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10845 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10846 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10847 }
10848 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10849 {
10850 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10851 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10852 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10853 || pCtx->gs.Attr.n.u4Type > 11
10854 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10855 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10856 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10857 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10858 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10859 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10860 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10861 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10862 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10863 }
10864 /* 64-bit capable CPUs. */
10865#if HC_ARCH_BITS == 64
10866 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10867 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10868 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10869 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10870 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10871 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10872 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10873 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10874 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10875 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10876 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10877#endif
10878 }
10879 else
10880 {
10881 /* V86 mode checks. */
10882 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10883 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10884 {
10885 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10886 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10887 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10888 }
10889 else
10890 {
10891 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10892 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10893 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10894 }
10895
10896 /* CS */
10897 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10898 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10899 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10900 /* SS */
10901 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10902 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10903 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10904 /* DS */
10905 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10906 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10907 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10908 /* ES */
10909 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10910 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10911 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10912 /* FS */
10913 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10914 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10915 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10916 /* GS */
10917 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10918 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10919 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10920 /* 64-bit capable CPUs. */
10921#if HC_ARCH_BITS == 64
10922 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10923 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10924 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10925 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10926 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10927 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10928 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10929 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10930 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10931 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10932 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10933#endif
10934 }
10935
10936 /*
10937 * TR.
10938 */
10939 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10940 /* 64-bit capable CPUs. */
10941#if HC_ARCH_BITS == 64
10942 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10943#endif
10944 if (fLongModeGuest)
10945 {
10946 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10947 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10948 }
10949 else
10950 {
10951 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10952 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10953 VMX_IGS_TR_ATTR_TYPE_INVALID);
10954 }
10955 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10956 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10957 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10958 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10959 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10960 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10961 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10962 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10963
10964 /*
10965 * GDTR and IDTR.
10966 */
10967#if HC_ARCH_BITS == 64
10968 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10969 AssertRCBreak(rc);
10970 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10971
10972 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10973 AssertRCBreak(rc);
10974 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10975#endif
10976
10977 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10978 AssertRCBreak(rc);
10979 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10980
10981 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10982 AssertRCBreak(rc);
10983 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10984
10985 /*
10986 * Guest Non-Register State.
10987 */
10988 /* Activity State. */
10989 uint32_t u32ActivityState;
10990 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10991 AssertRCBreak(rc);
10992 HMVMX_CHECK_BREAK( !u32ActivityState
10993 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
10994 VMX_IGS_ACTIVITY_STATE_INVALID);
10995 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10996 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10997 uint32_t u32IntrState;
10998 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
10999 AssertRCBreak(rc);
11000 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
11001 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11002 {
11003 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11004 }
11005
11006 /** @todo Activity state and injecting interrupts. Left as a todo since we
11007 * currently don't use activity states but ACTIVE. */
11008
11009 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11010 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11011
11012 /* Guest interruptibility-state. */
11013 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11014 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11015 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
11016 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11017 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11018 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11019 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11020 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11021 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11022 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
11023 {
11024 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11025 {
11026 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11027 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11028 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11029 }
11030 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11031 {
11032 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11033 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11034 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11035 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11036 }
11037 }
11038 /** @todo Assumes the processor is not in SMM. */
11039 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11040 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11041 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11042 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11043 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11044 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11045 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11046 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11047 {
11048 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11049 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11050 }
11051
11052 /* Pending debug exceptions. */
11053#if HC_ARCH_BITS == 64
11054 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11055 AssertRCBreak(rc);
11056 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11057 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11058 u32Val = u64Val; /* For pending debug exceptions checks below. */
11059#else
11060 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11061 AssertRCBreak(rc);
11062 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11063 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11064#endif
11065
11066 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11067 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11068 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11069 {
11070 if ( (u32Eflags & X86_EFL_TF)
11071 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11072 {
11073 /* Bit 14 is PendingDebug.BS. */
11074 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11075 }
11076 if ( !(u32Eflags & X86_EFL_TF)
11077 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11078 {
11079 /* Bit 14 is PendingDebug.BS. */
11080 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11081 }
11082 }
11083
11084 /* VMCS link pointer. */
11085 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11086 AssertRCBreak(rc);
11087 if (u64Val != UINT64_C(0xffffffffffffffff))
11088 {
11089 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11090 /** @todo Bits beyond the processor's physical-address width MBZ. */
11091 /** @todo 32-bit located in memory referenced by value of this field (as a
11092 * physical address) must contain the processor's VMCS revision ID. */
11093 /** @todo SMM checks. */
11094 }
11095
11096 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11097 * not using Nested Paging? */
11098 if ( pVM->hm.s.fNestedPaging
11099 && !fLongModeGuest
11100 && CPUMIsGuestInPAEModeEx(pCtx))
11101 {
11102 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11103 AssertRCBreak(rc);
11104 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11105
11106 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11107 AssertRCBreak(rc);
11108 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11109
11110 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11111 AssertRCBreak(rc);
11112 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11113
11114 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11115 AssertRCBreak(rc);
11116 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11117 }
11118
11119 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11120 if (uError == VMX_IGS_ERROR)
11121 uError = VMX_IGS_REASON_NOT_FOUND;
11122 } while (0);
11123
11124 pVCpu->hm.s.u32HMError = uError;
11125 return uError;
11126
11127#undef HMVMX_ERROR_BREAK
11128#undef HMVMX_CHECK_BREAK
11129}
11130
11131/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11132/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11133/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11134
11135/** @name VM-exit handlers.
11136 * @{
11137 */
11138
11139/**
11140 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11141 */
11142HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11143{
11144 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11145 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11146 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11147 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11148 return VINF_SUCCESS;
11149 return VINF_EM_RAW_INTERRUPT;
11150}
11151
11152
11153/**
11154 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11155 */
11156HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11157{
11158 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11159 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11160
11161 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11162 AssertRCReturn(rc, rc);
11163
11164 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11165 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11166 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11167 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11168
11169 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11170 {
11171 /*
11172 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11173 * anything we inject is not going to cause a VM-exit directly for the event being injected.
11174 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11175 *
11176 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11177 */
11178 VMXDispatchHostNmi();
11179 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11180 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11181 return VINF_SUCCESS;
11182 }
11183
11184 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11185 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11186 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11187 { /* likely */ }
11188 else
11189 {
11190 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11191 rcStrictRc1 = VINF_SUCCESS;
11192 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11193 return rcStrictRc1;
11194 }
11195
11196 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11197 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11198 switch (uIntType)
11199 {
11200 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11201 Assert(uVector == X86_XCPT_DB);
11202 /* no break */
11203 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11204 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11205 /* no break */
11206 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11207 {
11208 /*
11209 * If there's any exception caused as a result of event injection, go back to
11210 * the interpreter. The page-fault case is complicated and we manually handle
11211 * any currently pending event in hmR0VmxExitXcptPF. Nested #ACs are already
11212 * handled in hmR0VmxCheckExitDueToEventDelivery.
11213 */
11214 if (!pVCpu->hm.s.Event.fPending)
11215 { /* likely */ }
11216 else if ( uVector != X86_XCPT_PF
11217 && uVector != X86_XCPT_AC)
11218 {
11219 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
11220 rc = VERR_EM_INTERPRETER;
11221 break;
11222 }
11223
11224 switch (uVector)
11225 {
11226 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11227 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11228 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11229 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11230 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11231 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11232 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11233
11234 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11235 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11236 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11237 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11238 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11239 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11240 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11241 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11242 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11243 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11244 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11245 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11246 default:
11247 {
11248 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11249 AssertRCReturn(rc, rc);
11250
11251 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11252 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11253 {
11254 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11255 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11256 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11257
11258 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11259 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11260 AssertRCReturn(rc, rc);
11261 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11262 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11263 0 /* GCPtrFaultAddress */);
11264 AssertRCReturn(rc, rc);
11265 }
11266 else
11267 {
11268 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11269 pVCpu->hm.s.u32HMError = uVector;
11270 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11271 }
11272 break;
11273 }
11274 }
11275 break;
11276 }
11277
11278 default:
11279 {
11280 pVCpu->hm.s.u32HMError = uExitIntInfo;
11281 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11282 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11283 break;
11284 }
11285 }
11286 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11287 return rc;
11288}
11289
11290
11291/**
11292 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11293 */
11294HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11295{
11296 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11297
11298 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11299 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11300
11301 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11302 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11303 return VINF_SUCCESS;
11304}
11305
11306
11307/**
11308 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11309 */
11310HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11311{
11312 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11313 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11314 {
11315 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11316 HMVMX_RETURN_UNEXPECTED_EXIT();
11317 }
11318
11319 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11320
11321 /*
11322 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11323 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11324 */
11325 uint32_t uIntrState = 0;
11326 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11327 AssertRCReturn(rc, rc);
11328
11329 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11330 if ( fBlockSti
11331 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11332 {
11333 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11334 }
11335
11336 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11337 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11338
11339 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11340 return VINF_SUCCESS;
11341}
11342
11343
11344/**
11345 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11346 */
11347HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11348{
11349 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11350 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11351 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11352}
11353
11354
11355/**
11356 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11357 */
11358HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11359{
11360 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11361 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11362 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11363}
11364
11365
11366/**
11367 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11368 */
11369HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11370{
11371 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11372 PVM pVM = pVCpu->CTX_SUFF(pVM);
11373 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11374 if (RT_LIKELY(rc == VINF_SUCCESS))
11375 {
11376 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11377 Assert(pVmxTransient->cbInstr == 2);
11378 }
11379 else
11380 {
11381 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11382 rc = VERR_EM_INTERPRETER;
11383 }
11384 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11385 return rc;
11386}
11387
11388
11389/**
11390 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11391 */
11392HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11393{
11394 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11395 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11396 AssertRCReturn(rc, rc);
11397
11398 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11399 return VINF_EM_RAW_EMULATE_INSTR;
11400
11401 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11402 HMVMX_RETURN_UNEXPECTED_EXIT();
11403}
11404
11405
11406/**
11407 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11408 */
11409HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11410{
11411 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11412 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11413 AssertRCReturn(rc, rc);
11414
11415 PVM pVM = pVCpu->CTX_SUFF(pVM);
11416 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11417 if (RT_LIKELY(rc == VINF_SUCCESS))
11418 {
11419 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11420 Assert(pVmxTransient->cbInstr == 2);
11421 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11422 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11423 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11424 }
11425 else
11426 rc = VERR_EM_INTERPRETER;
11427 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11428 return rc;
11429}
11430
11431
11432/**
11433 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11434 */
11435HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11436{
11437 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11438 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11439 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11440 AssertRCReturn(rc, rc);
11441
11442 PVM pVM = pVCpu->CTX_SUFF(pVM);
11443 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11444 if (RT_SUCCESS(rc))
11445 {
11446 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11447 Assert(pVmxTransient->cbInstr == 3);
11448 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11449 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11450 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11451 }
11452 else
11453 {
11454 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11455 rc = VERR_EM_INTERPRETER;
11456 }
11457 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11458 return rc;
11459}
11460
11461
11462/**
11463 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11464 */
11465HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11466{
11467 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11468 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11469 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11470 AssertRCReturn(rc, rc);
11471
11472 PVM pVM = pVCpu->CTX_SUFF(pVM);
11473 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11474 if (RT_LIKELY(rc == VINF_SUCCESS))
11475 {
11476 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11477 Assert(pVmxTransient->cbInstr == 2);
11478 }
11479 else
11480 {
11481 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11482 rc = VERR_EM_INTERPRETER;
11483 }
11484 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11485 return rc;
11486}
11487
11488
11489/**
11490 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11491 */
11492HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11493{
11494 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11495 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11496
11497 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11498 if (pVCpu->hm.s.fHypercallsEnabled)
11499 {
11500#if 0
11501 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11502#else
11503 /* Aggressive state sync. for now. */
11504 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11505 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11506 AssertRCReturn(rc, rc);
11507#endif
11508
11509 /* Perform the hypercall. */
11510 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11511 if (rcStrict == VINF_SUCCESS)
11512 {
11513 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11514 AssertRCReturn(rc, rc);
11515 }
11516 else
11517 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11518 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11519 || RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
11520
11521 /* If the hypercall changes anything other than guest's general-purpose registers,
11522 we would need to reload the guest changed bits here before VM-entry. */
11523 }
11524 else
11525 Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11526
11527 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11528 if (RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)))
11529 {
11530 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11531 rcStrict = VINF_SUCCESS;
11532 }
11533
11534 return rcStrict;
11535}
11536
11537
11538/**
11539 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11540 */
11541HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11542{
11543 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11544 PVM pVM = pVCpu->CTX_SUFF(pVM);
11545 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11546
11547 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11548 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11549 AssertRCReturn(rc, rc);
11550
11551 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11552 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11553 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11554 else
11555 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11556 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11557 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
11558 return rcStrict;
11559}
11560
11561
11562/**
11563 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11564 */
11565HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11566{
11567 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11568 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11569 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11570 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11571 AssertRCReturn(rc, rc);
11572
11573 PVM pVM = pVCpu->CTX_SUFF(pVM);
11574 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11575 if (RT_LIKELY(rc == VINF_SUCCESS))
11576 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11577 else
11578 {
11579 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11580 rc = VERR_EM_INTERPRETER;
11581 }
11582 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11583 return rc;
11584}
11585
11586
11587/**
11588 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11589 */
11590HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11591{
11592 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11593 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11594 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11595 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11596 AssertRCReturn(rc, rc);
11597
11598 PVM pVM = pVCpu->CTX_SUFF(pVM);
11599 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11600 rc = VBOXSTRICTRC_VAL(rc2);
11601 if (RT_LIKELY( rc == VINF_SUCCESS
11602 || rc == VINF_EM_HALT))
11603 {
11604 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11605 AssertRCReturn(rc3, rc3);
11606
11607 if ( rc == VINF_EM_HALT
11608 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11609 {
11610 rc = VINF_SUCCESS;
11611 }
11612 }
11613 else
11614 {
11615 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11616 rc = VERR_EM_INTERPRETER;
11617 }
11618 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11619 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11620 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11621 return rc;
11622}
11623
11624
11625/**
11626 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11627 */
11628HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11629{
11630 /*
11631 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
11632 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
11633 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
11634 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
11635 */
11636 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11637 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11638 HMVMX_RETURN_UNEXPECTED_EXIT();
11639}
11640
11641
11642/**
11643 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11644 */
11645HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11646{
11647 /*
11648 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
11649 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
11650 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
11651 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
11652 */
11653 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11654 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11655 HMVMX_RETURN_UNEXPECTED_EXIT();
11656}
11657
11658
11659/**
11660 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11661 */
11662HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11663{
11664 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11665 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11666 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11667 HMVMX_RETURN_UNEXPECTED_EXIT();
11668}
11669
11670
11671/**
11672 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11673 */
11674HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11675{
11676 /*
11677 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
11678 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
11679 * See Intel spec. 25.3 "Other Causes of VM-exits".
11680 */
11681 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11682 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11683 HMVMX_RETURN_UNEXPECTED_EXIT();
11684}
11685
11686
11687/**
11688 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11689 * VM-exit.
11690 */
11691HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11692{
11693 /*
11694 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11695 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11696 *
11697 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11698 * See Intel spec. "23.8 Restrictions on VMX operation".
11699 */
11700 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11701 return VINF_SUCCESS;
11702}
11703
11704
11705/**
11706 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11707 * VM-exit.
11708 */
11709HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11710{
11711 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11712 return VINF_EM_RESET;
11713}
11714
11715
11716/**
11717 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11718 */
11719HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11720{
11721 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11722 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11723
11724 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11725 AssertRCReturn(rc, rc);
11726
11727 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11728 rc = VINF_SUCCESS;
11729 else
11730 rc = VINF_EM_HALT;
11731
11732 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11733 if (rc != VINF_SUCCESS)
11734 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11735 return rc;
11736}
11737
11738
11739/**
11740 * VM-exit handler for instructions that result in a \#UD exception delivered to
11741 * the guest.
11742 */
11743HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11744{
11745 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11746 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11747 return VINF_SUCCESS;
11748}
11749
11750
11751/**
11752 * VM-exit handler for expiry of the VMX preemption timer.
11753 */
11754HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11755{
11756 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11757
11758 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11759 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11760
11761 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11762 PVM pVM = pVCpu->CTX_SUFF(pVM);
11763 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11764 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11765 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11766}
11767
11768
11769/**
11770 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11771 */
11772HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11773{
11774 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11775
11776 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11777 rc |= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, false /*fNeedRsp*/);
11778 rc |= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11779 AssertRCReturn(rc, rc);
11780
11781 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11782 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
11783
11784 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11785
11786 return rcStrict;
11787}
11788
11789
11790/**
11791 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11792 */
11793HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11794{
11795 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11796
11797 /* The guest should not invalidate the host CPU's TLBs, fallback to interpreter. */
11798 /** @todo implement EMInterpretInvpcid() */
11799 return VERR_EM_INTERPRETER;
11800}
11801
11802
11803/**
11804 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11805 * Error VM-exit.
11806 */
11807HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11808{
11809 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11810 AssertRCReturn(rc, rc);
11811
11812 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11813 AssertRCReturn(rc, rc);
11814
11815 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11816 NOREF(uInvalidReason);
11817
11818#ifdef VBOX_STRICT
11819 uint32_t uIntrState;
11820 RTHCUINTREG uHCReg;
11821 uint64_t u64Val;
11822 uint32_t u32Val;
11823
11824 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11825 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11826 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11827 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11828 AssertRCReturn(rc, rc);
11829
11830 Log4(("uInvalidReason %u\n", uInvalidReason));
11831 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11832 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11833 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11834 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
11835
11836 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11837 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11838 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11839 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11840 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11841 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11842 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11843 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11844 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11845 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11846 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11847 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11848#else
11849 NOREF(pVmxTransient);
11850#endif
11851
11852 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11853 return VERR_VMX_INVALID_GUEST_STATE;
11854}
11855
11856
11857/**
11858 * VM-exit handler for VM-entry failure due to an MSR-load
11859 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11860 */
11861HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11862{
11863 NOREF(pVmxTransient);
11864 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11865 HMVMX_RETURN_UNEXPECTED_EXIT();
11866}
11867
11868
11869/**
11870 * VM-exit handler for VM-entry failure due to a machine-check event
11871 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11872 */
11873HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11874{
11875 NOREF(pVmxTransient);
11876 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11877 HMVMX_RETURN_UNEXPECTED_EXIT();
11878}
11879
11880
11881/**
11882 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11883 * theory.
11884 */
11885HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11886{
11887 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
11888 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
11889 return VERR_VMX_UNDEFINED_EXIT_CODE;
11890}
11891
11892
11893/**
11894 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11895 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11896 * Conditional VM-exit.
11897 */
11898HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11899{
11900 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11901
11902 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
11903 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11904 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
11905 return VERR_EM_INTERPRETER;
11906 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11907 HMVMX_RETURN_UNEXPECTED_EXIT();
11908}
11909
11910
11911/**
11912 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11913 */
11914HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11915{
11916 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11917
11918 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
11919 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
11920 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
11921 return VERR_EM_INTERPRETER;
11922 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11923 HMVMX_RETURN_UNEXPECTED_EXIT();
11924}
11925
11926
11927/**
11928 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11929 */
11930HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11931{
11932 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11933
11934 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
11935 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11936 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11937 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11938 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11939 {
11940 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
11941 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
11942 }
11943 AssertRCReturn(rc, rc);
11944 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
11945
11946#ifdef VBOX_STRICT
11947 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
11948 {
11949 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
11950 && pMixedCtx->ecx != MSR_K6_EFER)
11951 {
11952 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11953 pMixedCtx->ecx));
11954 HMVMX_RETURN_UNEXPECTED_EXIT();
11955 }
11956 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11957 {
11958 VMXMSREXITREAD enmRead;
11959 VMXMSREXITWRITE enmWrite;
11960 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
11961 AssertRCReturn(rc2, rc2);
11962 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11963 {
11964 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11965 HMVMX_RETURN_UNEXPECTED_EXIT();
11966 }
11967 }
11968 }
11969#endif
11970
11971 PVM pVM = pVCpu->CTX_SUFF(pVM);
11972 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11973 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
11974 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
11975 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11976 if (RT_SUCCESS(rc))
11977 {
11978 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11979 Assert(pVmxTransient->cbInstr == 2);
11980 }
11981 return rc;
11982}
11983
11984
11985/**
11986 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11987 */
11988HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11989{
11990 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11991 PVM pVM = pVCpu->CTX_SUFF(pVM);
11992 int rc = VINF_SUCCESS;
11993
11994 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
11995 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11996 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11997 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11998 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11999 {
12000 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12001 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12002 }
12003 AssertRCReturn(rc, rc);
12004 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
12005
12006 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12007 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
12008 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12009
12010 if (RT_SUCCESS(rc))
12011 {
12012 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12013
12014 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
12015 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
12016 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
12017 {
12018 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12019 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
12020 EMInterpretWrmsr() changes it. */
12021 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12022 }
12023 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12024 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12025 else if (pMixedCtx->ecx == MSR_K6_EFER)
12026 {
12027 /*
12028 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12029 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12030 * the other bits as well, SCE and NXE. See @bugref{7368}.
12031 */
12032 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
12033 }
12034
12035 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12036 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12037 {
12038 switch (pMixedCtx->ecx)
12039 {
12040 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
12041 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
12042 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
12043 case MSR_K8_FS_BASE: /* no break */
12044 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
12045 case MSR_K6_EFER: /* already handled above */ break;
12046 default:
12047 {
12048 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12049 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12050 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12051 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
12052 break;
12053 }
12054 }
12055 }
12056#ifdef VBOX_STRICT
12057 else
12058 {
12059 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12060 switch (pMixedCtx->ecx)
12061 {
12062 case MSR_IA32_SYSENTER_CS:
12063 case MSR_IA32_SYSENTER_EIP:
12064 case MSR_IA32_SYSENTER_ESP:
12065 case MSR_K8_FS_BASE:
12066 case MSR_K8_GS_BASE:
12067 {
12068 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12069 HMVMX_RETURN_UNEXPECTED_EXIT();
12070 }
12071
12072 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12073 default:
12074 {
12075 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12076 {
12077 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12078 if (pMixedCtx->ecx != MSR_K6_EFER)
12079 {
12080 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12081 pMixedCtx->ecx));
12082 HMVMX_RETURN_UNEXPECTED_EXIT();
12083 }
12084 }
12085
12086 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12087 {
12088 VMXMSREXITREAD enmRead;
12089 VMXMSREXITWRITE enmWrite;
12090 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12091 AssertRCReturn(rc2, rc2);
12092 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12093 {
12094 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12095 HMVMX_RETURN_UNEXPECTED_EXIT();
12096 }
12097 }
12098 break;
12099 }
12100 }
12101 }
12102#endif /* VBOX_STRICT */
12103 }
12104 return rc;
12105}
12106
12107
12108/**
12109 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12110 */
12111HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12112{
12113 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12114
12115 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12116 return VINF_EM_RAW_INTERRUPT;
12117}
12118
12119
12120/**
12121 * VM-exit handler for when the TPR value is lowered below the specified
12122 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12123 */
12124HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12125{
12126 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12127 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12128
12129 /*
12130 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
12131 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
12132 * resume guest execution.
12133 */
12134 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12135 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12136 return VINF_SUCCESS;
12137}
12138
12139
12140/**
12141 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12142 * VM-exit.
12143 *
12144 * @retval VINF_SUCCESS when guest execution can continue.
12145 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12146 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12147 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12148 * interpreter.
12149 */
12150HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12151{
12152 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12153 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12154 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12155 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12156 AssertRCReturn(rc, rc);
12157
12158 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12159 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12160 PVM pVM = pVCpu->CTX_SUFF(pVM);
12161 VBOXSTRICTRC rcStrict;
12162 rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, true /*fNeedRsp*/);
12163 switch (uAccessType)
12164 {
12165 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12166 {
12167 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12168 AssertRCReturn(rc, rc);
12169
12170 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12171 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12172 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12173 AssertMsg( rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE
12174 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12175 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12176 {
12177 case 0: /* CR0 */
12178 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12179 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12180 break;
12181 case 2: /* CR2 */
12182 /* Nothing to do here, CR2 it's not part of the VMCS. */
12183 break;
12184 case 3: /* CR3 */
12185 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
12186 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12187 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12188 break;
12189 case 4: /* CR4 */
12190 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12191 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12192 VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12193 break;
12194 case 8: /* CR8 */
12195 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12196 /* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12197 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12198 break;
12199 default:
12200 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12201 break;
12202 }
12203
12204 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12205 break;
12206 }
12207
12208 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12209 {
12210 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12211 AssertRCReturn(rc, rc);
12212
12213 Assert( !pVM->hm.s.fNestedPaging
12214 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12215 || pVCpu->hm.s.fUsingDebugLoop
12216 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12217
12218 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12219 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12220 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12221
12222 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12223 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12224 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12225 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12226 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12227 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12228 VBOXSTRICTRC_VAL(rcStrict)));
12229 break;
12230 }
12231
12232 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12233 {
12234 AssertRCReturn(rc, rc);
12235 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12236 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12237 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12238 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12239 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12240 break;
12241 }
12242
12243 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12244 {
12245 AssertRCReturn(rc, rc);
12246 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12247 VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12248 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE,
12249 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12250 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12251 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12252 break;
12253 }
12254
12255 default:
12256 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12257 VERR_VMX_UNEXPECTED_EXCEPTION);
12258 }
12259
12260 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12261 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12262 NOREF(pVM);
12263 return rcStrict;
12264}
12265
12266
12267/**
12268 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12269 * VM-exit.
12270 */
12271HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12272{
12273 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12274 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12275
12276 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12277 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12278 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12279 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12280 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12281 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12282 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12283 AssertRCReturn(rc2, rc2);
12284
12285 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12286 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12287 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12288 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12289 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12290 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12291 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12292 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12293 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12294
12295 /* I/O operation lookup arrays. */
12296 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12297 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12298
12299 VBOXSTRICTRC rcStrict;
12300 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12301 uint32_t const cbInstr = pVmxTransient->cbInstr;
12302 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12303 PVM pVM = pVCpu->CTX_SUFF(pVM);
12304 if (fIOString)
12305 {
12306#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12307 See @bugref{5752#c158}. Should work now. */
12308 /*
12309 * INS/OUTS - I/O String instruction.
12310 *
12311 * Use instruction-information if available, otherwise fall back on
12312 * interpreting the instruction.
12313 */
12314 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12315 fIOWrite ? 'w' : 'r'));
12316 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12317 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12318 {
12319 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12320 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12321 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12322 AssertRCReturn(rc2, rc2);
12323 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12324 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12325 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12326 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12327 if (fIOWrite)
12328 {
12329 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12330 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12331 }
12332 else
12333 {
12334 /*
12335 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12336 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12337 * See Intel Instruction spec. for "INS".
12338 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12339 */
12340 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12341 }
12342 }
12343 else
12344 {
12345 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12346 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12347 AssertRCReturn(rc2, rc2);
12348 rcStrict = IEMExecOne(pVCpu);
12349 }
12350 /** @todo IEM needs to be setting these flags somehow. */
12351 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12352 fUpdateRipAlready = true;
12353#else
12354 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12355 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12356 if (RT_SUCCESS(rcStrict))
12357 {
12358 if (fIOWrite)
12359 {
12360 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12361 (DISCPUMODE)pDis->uAddrMode, cbValue);
12362 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12363 }
12364 else
12365 {
12366 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12367 (DISCPUMODE)pDis->uAddrMode, cbValue);
12368 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12369 }
12370 }
12371 else
12372 {
12373 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12374 pMixedCtx->rip));
12375 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12376 }
12377#endif
12378 }
12379 else
12380 {
12381 /*
12382 * IN/OUT - I/O instruction.
12383 */
12384 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12385 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12386 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12387 if (fIOWrite)
12388 {
12389 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12390 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12391 }
12392 else
12393 {
12394 uint32_t u32Result = 0;
12395 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12396 if (IOM_SUCCESS(rcStrict))
12397 {
12398 /* Save result of I/O IN instr. in AL/AX/EAX. */
12399 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12400 }
12401 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12402 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12403 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12404 }
12405 }
12406
12407 if (IOM_SUCCESS(rcStrict))
12408 {
12409 if (!fUpdateRipAlready)
12410 {
12411 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12412 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12413 }
12414
12415 /*
12416 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12417 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12418 */
12419 if (fIOString)
12420 {
12421 /** @todo Single-step for INS/OUTS with REP prefix? */
12422 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12423 }
12424 else if ( !fDbgStepping
12425 && fGstStepping)
12426 {
12427 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12428 }
12429
12430 /*
12431 * If any I/O breakpoints are armed, we need to check if one triggered
12432 * and take appropriate action.
12433 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12434 */
12435 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12436 AssertRCReturn(rc2, rc2);
12437
12438 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12439 * execution engines about whether hyper BPs and such are pending. */
12440 uint32_t const uDr7 = pMixedCtx->dr[7];
12441 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12442 && X86_DR7_ANY_RW_IO(uDr7)
12443 && (pMixedCtx->cr4 & X86_CR4_DE))
12444 || DBGFBpIsHwIoArmed(pVM)))
12445 {
12446 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12447
12448 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12449 VMMRZCallRing3Disable(pVCpu);
12450 HM_DISABLE_PREEMPT();
12451
12452 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12453
12454 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12455 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12456 {
12457 /* Raise #DB. */
12458 if (fIsGuestDbgActive)
12459 ASMSetDR6(pMixedCtx->dr[6]);
12460 if (pMixedCtx->dr[7] != uDr7)
12461 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12462
12463 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12464 }
12465 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12466 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12467 else if ( rcStrict2 != VINF_SUCCESS
12468 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12469 rcStrict = rcStrict2;
12470 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12471
12472 HM_RESTORE_PREEMPT();
12473 VMMRZCallRing3Enable(pVCpu);
12474 }
12475 }
12476
12477#ifdef VBOX_STRICT
12478 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12479 Assert(!fIOWrite);
12480 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12481 Assert(fIOWrite);
12482 else
12483 {
12484#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12485 * statuses, that the VMM device and some others may return. See
12486 * IOM_SUCCESS() for guidance. */
12487 AssertMsg( RT_FAILURE(rcStrict)
12488 || rcStrict == VINF_SUCCESS
12489 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12490 || rcStrict == VINF_EM_DBG_BREAKPOINT
12491 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12492 || rcStrict == VINF_EM_RAW_TO_R3
12493 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12494#endif
12495 }
12496#endif
12497
12498 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12499 return rcStrict;
12500}
12501
12502
12503/**
12504 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12505 * VM-exit.
12506 */
12507HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12508{
12509 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12510
12511 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12512 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12513 AssertRCReturn(rc, rc);
12514 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12515 {
12516 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12517 AssertRCReturn(rc, rc);
12518 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12519 {
12520 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12521
12522 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12523 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12524
12525 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
12526 Assert(!pVCpu->hm.s.Event.fPending);
12527 pVCpu->hm.s.Event.fPending = true;
12528 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
12529 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12530 AssertRCReturn(rc, rc);
12531 if (fErrorCodeValid)
12532 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12533 else
12534 pVCpu->hm.s.Event.u32ErrCode = 0;
12535 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12536 && uVector == X86_XCPT_PF)
12537 {
12538 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
12539 }
12540
12541 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12542 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12543 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12544 }
12545 }
12546
12547 /* Fall back to the interpreter to emulate the task-switch. */
12548 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12549 return VERR_EM_INTERPRETER;
12550}
12551
12552
12553/**
12554 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12555 */
12556HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12557{
12558 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12559 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12560 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12561 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12562 AssertRCReturn(rc, rc);
12563 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12564 return VINF_EM_DBG_STEPPED;
12565}
12566
12567
12568/**
12569 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12570 */
12571HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12572{
12573 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12574
12575 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12576
12577 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12578 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12579 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12580 {
12581 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12582 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12583 {
12584 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12585 return VERR_EM_INTERPRETER;
12586 }
12587 }
12588 else
12589 {
12590 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12591 rcStrict1 = VINF_SUCCESS;
12592 return rcStrict1;
12593 }
12594
12595#if 0
12596 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
12597 * just sync the whole thing. */
12598 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12599#else
12600 /* Aggressive state sync. for now. */
12601 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12602 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12603 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12604#endif
12605 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12606 AssertRCReturn(rc, rc);
12607
12608 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12609 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12610 VBOXSTRICTRC rcStrict2;
12611 switch (uAccessType)
12612 {
12613 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12614 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12615 {
12616 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12617 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != 0x80,
12618 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12619
12620 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
12621 GCPhys &= PAGE_BASE_GC_MASK;
12622 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12623 PVM pVM = pVCpu->CTX_SUFF(pVM);
12624 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12625 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12626
12627 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12628 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12629 CPUMCTX2CORE(pMixedCtx), GCPhys);
12630 Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
12631 if ( rcStrict2 == VINF_SUCCESS
12632 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12633 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12634 {
12635 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12636 | HM_CHANGED_GUEST_RSP
12637 | HM_CHANGED_GUEST_RFLAGS
12638 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12639 rcStrict2 = VINF_SUCCESS;
12640 }
12641 break;
12642 }
12643
12644 default:
12645 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
12646 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12647 break;
12648 }
12649
12650 if (rcStrict2 != VINF_SUCCESS)
12651 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12652 return rcStrict2;
12653}
12654
12655
12656/**
12657 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12658 * VM-exit.
12659 */
12660HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12661{
12662 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12663
12664 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12665 if (pVmxTransient->fWasGuestDebugStateActive)
12666 {
12667 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12668 HMVMX_RETURN_UNEXPECTED_EXIT();
12669 }
12670
12671 if ( !pVCpu->hm.s.fSingleInstruction
12672 && !pVmxTransient->fWasHyperDebugStateActive)
12673 {
12674 Assert(!DBGFIsStepping(pVCpu));
12675 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12676
12677 /* Don't intercept MOV DRx any more. */
12678 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12679 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12680 AssertRCReturn(rc, rc);
12681
12682 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12683 VMMRZCallRing3Disable(pVCpu);
12684 HM_DISABLE_PREEMPT();
12685
12686 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12687 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12688 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12689
12690 HM_RESTORE_PREEMPT();
12691 VMMRZCallRing3Enable(pVCpu);
12692
12693#ifdef VBOX_WITH_STATISTICS
12694 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12695 AssertRCReturn(rc, rc);
12696 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12697 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12698 else
12699 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12700#endif
12701 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12702 return VINF_SUCCESS;
12703 }
12704
12705 /*
12706 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12707 * Update the segment registers and DR7 from the CPU.
12708 */
12709 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12710 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12711 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12712 AssertRCReturn(rc, rc);
12713 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12714
12715 PVM pVM = pVCpu->CTX_SUFF(pVM);
12716 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12717 {
12718 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12719 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
12720 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
12721 if (RT_SUCCESS(rc))
12722 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12723 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12724 }
12725 else
12726 {
12727 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12728 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
12729 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
12730 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12731 }
12732
12733 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12734 if (RT_SUCCESS(rc))
12735 {
12736 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12737 AssertRCReturn(rc2, rc2);
12738 return VINF_SUCCESS;
12739 }
12740 return rc;
12741}
12742
12743
12744/**
12745 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12746 * Conditional VM-exit.
12747 */
12748HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12749{
12750 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12751 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12752
12753 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12754 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12755 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12756 {
12757 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12758 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12759 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12760 {
12761 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12762 return VERR_EM_INTERPRETER;
12763 }
12764 }
12765 else
12766 {
12767 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12768 rcStrict1 = VINF_SUCCESS;
12769 return rcStrict1;
12770 }
12771
12772 RTGCPHYS GCPhys = 0;
12773 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12774
12775#if 0
12776 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12777#else
12778 /* Aggressive state sync. for now. */
12779 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12780 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12781 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12782#endif
12783 AssertRCReturn(rc, rc);
12784
12785 /*
12786 * If we succeed, resume guest execution.
12787 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12788 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12789 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12790 * weird case. See @bugref{6043}.
12791 */
12792 PVM pVM = pVCpu->CTX_SUFF(pVM);
12793 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12794 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
12795 if ( rcStrict2 == VINF_SUCCESS
12796 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12797 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12798 {
12799 /* Successfully handled MMIO operation. */
12800 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12801 | HM_CHANGED_GUEST_RSP
12802 | HM_CHANGED_GUEST_RFLAGS
12803 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12804 return VINF_SUCCESS;
12805 }
12806 return rcStrict2;
12807}
12808
12809
12810/**
12811 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12812 * VM-exit.
12813 */
12814HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12815{
12816 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12817 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12818
12819 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12820 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12821 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12822 {
12823 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12824 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12825 Log4(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12826 }
12827 else
12828 {
12829 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12830 rcStrict1 = VINF_SUCCESS;
12831 return rcStrict1;
12832 }
12833
12834 RTGCPHYS GCPhys = 0;
12835 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12836 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12837#if 0
12838 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12839#else
12840 /* Aggressive state sync. for now. */
12841 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12842 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12843 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12844#endif
12845 AssertRCReturn(rc, rc);
12846
12847 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12848 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12849
12850 RTGCUINT uErrorCode = 0;
12851 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
12852 uErrorCode |= X86_TRAP_PF_ID;
12853 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
12854 uErrorCode |= X86_TRAP_PF_RW;
12855 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
12856 uErrorCode |= X86_TRAP_PF_P;
12857
12858 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12859
12860 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12861 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12862
12863 /* Handle the pagefault trap for the nested shadow table. */
12864 PVM pVM = pVCpu->CTX_SUFF(pVM);
12865 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12866 TRPMResetTrap(pVCpu);
12867
12868 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12869 if ( rcStrict2 == VINF_SUCCESS
12870 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12871 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12872 {
12873 /* Successfully synced our nested page tables. */
12874 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12875 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12876 | HM_CHANGED_GUEST_RSP
12877 | HM_CHANGED_GUEST_RFLAGS);
12878 return VINF_SUCCESS;
12879 }
12880
12881 Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12882 return rcStrict2;
12883}
12884
12885/** @} */
12886
12887/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12888/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
12889/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12890
12891/** @name VM-exit exception handlers.
12892 * @{
12893 */
12894
12895/**
12896 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12897 */
12898static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12899{
12900 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12901 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12902
12903 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12904 AssertRCReturn(rc, rc);
12905
12906 if (!(pMixedCtx->cr0 & X86_CR0_NE))
12907 {
12908 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12909 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12910
12911 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12912 * provides VM-exit instruction length. If this causes problem later,
12913 * disassemble the instruction like it's done on AMD-V. */
12914 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12915 AssertRCReturn(rc2, rc2);
12916 return rc;
12917 }
12918
12919 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12920 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12921 return rc;
12922}
12923
12924
12925/**
12926 * VM-exit exception handler for \#BP (Breakpoint exception).
12927 */
12928static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12929{
12930 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12931 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12932
12933 /** @todo Try optimize this by not saving the entire guest state unless
12934 * really needed. */
12935 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12936 AssertRCReturn(rc, rc);
12937
12938 PVM pVM = pVCpu->CTX_SUFF(pVM);
12939 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12940 if (rc == VINF_EM_RAW_GUEST_TRAP)
12941 {
12942 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12943 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12944 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12945 AssertRCReturn(rc, rc);
12946
12947 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12948 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12949 }
12950
12951 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
12952 return rc;
12953}
12954
12955
12956/**
12957 * VM-exit exception handler for \#AC (alignment check exception).
12958 */
12959static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12960{
12961 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12962
12963 /*
12964 * Re-inject it. We'll detect any nesting before getting here.
12965 */
12966 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12967 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12968 AssertRCReturn(rc, rc);
12969 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
12970
12971 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12972 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12973 return VINF_SUCCESS;
12974}
12975
12976
12977/**
12978 * VM-exit exception handler for \#DB (Debug exception).
12979 */
12980static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12981{
12982 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12983 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
12984 Log6(("XcptDB\n"));
12985
12986 /*
12987 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
12988 * for processing.
12989 */
12990 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12991 AssertRCReturn(rc, rc);
12992
12993 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
12994 uint64_t uDR6 = X86_DR6_INIT_VAL;
12995 uDR6 |= ( pVmxTransient->uExitQualification
12996 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
12997
12998 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
12999 if (rc == VINF_EM_RAW_GUEST_TRAP)
13000 {
13001 /*
13002 * The exception was for the guest. Update DR6, DR7.GD and
13003 * IA32_DEBUGCTL.LBR before forwarding it.
13004 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13005 */
13006 VMMRZCallRing3Disable(pVCpu);
13007 HM_DISABLE_PREEMPT();
13008
13009 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
13010 pMixedCtx->dr[6] |= uDR6;
13011 if (CPUMIsGuestDebugStateActive(pVCpu))
13012 ASMSetDR6(pMixedCtx->dr[6]);
13013
13014 HM_RESTORE_PREEMPT();
13015 VMMRZCallRing3Enable(pVCpu);
13016
13017 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13018 AssertRCReturn(rc, rc);
13019
13020 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13021 pMixedCtx->dr[7] &= ~X86_DR7_GD;
13022
13023 /* Paranoia. */
13024 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13025 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
13026
13027 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
13028 AssertRCReturn(rc, rc);
13029
13030 /*
13031 * Raise #DB in the guest.
13032 *
13033 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
13034 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
13035 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
13036 *
13037 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
13038 */
13039 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13040 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13041 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13042 AssertRCReturn(rc, rc);
13043 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13044 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13045 return VINF_SUCCESS;
13046 }
13047
13048 /*
13049 * Not a guest trap, must be a hypervisor related debug event then.
13050 * Update DR6 in case someone is interested in it.
13051 */
13052 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13053 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13054 CPUMSetHyperDR6(pVCpu, uDR6);
13055
13056 return rc;
13057}
13058
13059
13060/**
13061 * VM-exit exception handler for \#NM (Device-not-available exception: floating
13062 * point exception).
13063 */
13064static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13065{
13066 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13067
13068 /* We require CR0 and EFER. EFER is always up-to-date. */
13069 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13070 AssertRCReturn(rc, rc);
13071
13072 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
13073 VMMRZCallRing3Disable(pVCpu);
13074 HM_DISABLE_PREEMPT();
13075
13076 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
13077 if (pVmxTransient->fWasGuestFPUStateActive)
13078 {
13079 rc = VINF_EM_RAW_GUEST_TRAP;
13080 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
13081 }
13082 else
13083 {
13084#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13085 Assert(!pVmxTransient->fWasGuestFPUStateActive || pVCpu->hm.s.fUsingDebugLoop);
13086#endif
13087 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu);
13088 Assert( rc == VINF_EM_RAW_GUEST_TRAP
13089 || ((rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
13090 if (rc == VINF_CPUM_HOST_CR0_MODIFIED)
13091 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
13092 }
13093
13094 HM_RESTORE_PREEMPT();
13095 VMMRZCallRing3Enable(pVCpu);
13096
13097 if (rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED)
13098 {
13099 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
13100 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13101 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
13102 pVCpu->hm.s.fPreloadGuestFpu = true;
13103 }
13104 else
13105 {
13106 /* Forward #NM to the guest. */
13107 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
13108 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13109 AssertRCReturn(rc, rc);
13110 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13111 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
13112 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13113 }
13114
13115 return VINF_SUCCESS;
13116}
13117
13118
13119/**
13120 * VM-exit exception handler for \#GP (General-protection exception).
13121 *
13122 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13123 */
13124static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13125{
13126 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13127 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13128
13129 int rc;
13130 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13131 { /* likely */ }
13132 else
13133 {
13134#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13135 Assert(pVCpu->hm.s.fUsingDebugLoop);
13136#endif
13137 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13138 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13139 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13140 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13141 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13142 AssertRCReturn(rc, rc);
13143 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13144 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13145 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13146 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13147 return rc;
13148 }
13149
13150 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13151 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13152
13153 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13154 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13155 AssertRCReturn(rc, rc);
13156
13157 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13158 uint32_t cbOp = 0;
13159 PVM pVM = pVCpu->CTX_SUFF(pVM);
13160 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13161 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13162 if (RT_SUCCESS(rc))
13163 {
13164 rc = VINF_SUCCESS;
13165 Assert(cbOp == pDis->cbInstr);
13166 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13167 switch (pDis->pCurInstr->uOpcode)
13168 {
13169 case OP_CLI:
13170 {
13171 pMixedCtx->eflags.Bits.u1IF = 0;
13172 pMixedCtx->eflags.Bits.u1RF = 0;
13173 pMixedCtx->rip += pDis->cbInstr;
13174 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13175 if ( !fDbgStepping
13176 && pMixedCtx->eflags.Bits.u1TF)
13177 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13178 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13179 break;
13180 }
13181
13182 case OP_STI:
13183 {
13184 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13185 pMixedCtx->eflags.Bits.u1IF = 1;
13186 pMixedCtx->eflags.Bits.u1RF = 0;
13187 pMixedCtx->rip += pDis->cbInstr;
13188 if (!fOldIF)
13189 {
13190 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13191 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13192 }
13193 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13194 if ( !fDbgStepping
13195 && pMixedCtx->eflags.Bits.u1TF)
13196 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13197 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13198 break;
13199 }
13200
13201 case OP_HLT:
13202 {
13203 rc = VINF_EM_HALT;
13204 pMixedCtx->rip += pDis->cbInstr;
13205 pMixedCtx->eflags.Bits.u1RF = 0;
13206 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13207 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13208 break;
13209 }
13210
13211 case OP_POPF:
13212 {
13213 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13214 uint32_t cbParm;
13215 uint32_t uMask;
13216 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13217 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13218 {
13219 cbParm = 4;
13220 uMask = 0xffffffff;
13221 }
13222 else
13223 {
13224 cbParm = 2;
13225 uMask = 0xffff;
13226 }
13227
13228 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13229 RTGCPTR GCPtrStack = 0;
13230 X86EFLAGS Eflags;
13231 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13232 &GCPtrStack);
13233 if (RT_SUCCESS(rc))
13234 {
13235 Assert(sizeof(Eflags.u32) >= cbParm);
13236 Eflags.u32 = 0;
13237 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13238 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13239 }
13240 if (RT_FAILURE(rc))
13241 {
13242 rc = VERR_EM_INTERPRETER;
13243 break;
13244 }
13245 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13246 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13247 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13248 pMixedCtx->esp += cbParm;
13249 pMixedCtx->esp &= uMask;
13250 pMixedCtx->rip += pDis->cbInstr;
13251 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13252 | HM_CHANGED_GUEST_RSP
13253 | HM_CHANGED_GUEST_RFLAGS);
13254 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13255 POPF restores EFLAGS.TF. */
13256 if ( !fDbgStepping
13257 && fGstStepping)
13258 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13259 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13260 break;
13261 }
13262
13263 case OP_PUSHF:
13264 {
13265 uint32_t cbParm;
13266 uint32_t uMask;
13267 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13268 {
13269 cbParm = 4;
13270 uMask = 0xffffffff;
13271 }
13272 else
13273 {
13274 cbParm = 2;
13275 uMask = 0xffff;
13276 }
13277
13278 /* Get the stack pointer & push the contents of eflags onto the stack. */
13279 RTGCPTR GCPtrStack = 0;
13280 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13281 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13282 if (RT_FAILURE(rc))
13283 {
13284 rc = VERR_EM_INTERPRETER;
13285 break;
13286 }
13287 X86EFLAGS Eflags = pMixedCtx->eflags;
13288 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13289 Eflags.Bits.u1RF = 0;
13290 Eflags.Bits.u1VM = 0;
13291
13292 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13293 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13294 {
13295 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13296 rc = VERR_EM_INTERPRETER;
13297 break;
13298 }
13299 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13300 pMixedCtx->esp -= cbParm;
13301 pMixedCtx->esp &= uMask;
13302 pMixedCtx->rip += pDis->cbInstr;
13303 pMixedCtx->eflags.Bits.u1RF = 0;
13304 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13305 | HM_CHANGED_GUEST_RSP
13306 | HM_CHANGED_GUEST_RFLAGS);
13307 if ( !fDbgStepping
13308 && pMixedCtx->eflags.Bits.u1TF)
13309 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13310 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13311 break;
13312 }
13313
13314 case OP_IRET:
13315 {
13316 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13317 * instruction reference. */
13318 RTGCPTR GCPtrStack = 0;
13319 uint32_t uMask = 0xffff;
13320 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13321 uint16_t aIretFrame[3];
13322 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13323 {
13324 rc = VERR_EM_INTERPRETER;
13325 break;
13326 }
13327 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13328 &GCPtrStack);
13329 if (RT_SUCCESS(rc))
13330 {
13331 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13332 PGMACCESSORIGIN_HM));
13333 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13334 }
13335 if (RT_FAILURE(rc))
13336 {
13337 rc = VERR_EM_INTERPRETER;
13338 break;
13339 }
13340 pMixedCtx->eip = 0;
13341 pMixedCtx->ip = aIretFrame[0];
13342 pMixedCtx->cs.Sel = aIretFrame[1];
13343 pMixedCtx->cs.ValidSel = aIretFrame[1];
13344 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13345 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13346 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13347 pMixedCtx->sp += sizeof(aIretFrame);
13348 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13349 | HM_CHANGED_GUEST_SEGMENT_REGS
13350 | HM_CHANGED_GUEST_RSP
13351 | HM_CHANGED_GUEST_RFLAGS);
13352 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13353 if ( !fDbgStepping
13354 && fGstStepping)
13355 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13356 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13357 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13358 break;
13359 }
13360
13361 case OP_INT:
13362 {
13363 uint16_t uVector = pDis->Param1.uValue & 0xff;
13364 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13365 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13366 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13367 break;
13368 }
13369
13370 case OP_INTO:
13371 {
13372 if (pMixedCtx->eflags.Bits.u1OF)
13373 {
13374 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13375 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13376 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13377 }
13378 else
13379 {
13380 pMixedCtx->eflags.Bits.u1RF = 0;
13381 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13382 }
13383 break;
13384 }
13385
13386 default:
13387 {
13388 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13389 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13390 EMCODETYPE_SUPERVISOR);
13391 rc = VBOXSTRICTRC_VAL(rc2);
13392 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13393 /** @todo We have to set pending-debug exceptions here when the guest is
13394 * single-stepping depending on the instruction that was interpreted. */
13395 Log4(("#GP rc=%Rrc\n", rc));
13396 break;
13397 }
13398 }
13399 }
13400 else
13401 rc = VERR_EM_INTERPRETER;
13402
13403 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13404 ("#GP Unexpected rc=%Rrc\n", rc));
13405 return rc;
13406}
13407
13408
13409/**
13410 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13411 * the exception reported in the VMX transient structure back into the VM.
13412 *
13413 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13414 * up-to-date.
13415 */
13416static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13417{
13418 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13419#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13420 Assert(pVCpu->hm.s.fUsingDebugLoop);
13421#endif
13422
13423 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13424 hmR0VmxCheckExitDueToEventDelivery(). */
13425 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13426 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13427 AssertRCReturn(rc, rc);
13428 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13429
13430#ifdef DEBUG_ramshankar
13431 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13432 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13433 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13434#endif
13435
13436 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13437 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13438 return VINF_SUCCESS;
13439}
13440
13441
13442/**
13443 * VM-exit exception handler for \#PF (Page-fault exception).
13444 */
13445static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13446{
13447 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13448 PVM pVM = pVCpu->CTX_SUFF(pVM);
13449 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13450 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13451 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13452 AssertRCReturn(rc, rc);
13453
13454 if (!pVM->hm.s.fNestedPaging)
13455 { /* likely */ }
13456 else
13457 {
13458#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13459 Assert(pVCpu->hm.s.fUsingDebugLoop);
13460#endif
13461 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13462 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13463 {
13464 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13465 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13466 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13467 }
13468 else
13469 {
13470 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13471 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13472 Log4(("Pending #DF due to vectoring #PF. NP\n"));
13473 }
13474 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13475 return rc;
13476 }
13477
13478 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13479 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13480 if (pVmxTransient->fVectoringPF)
13481 {
13482 Assert(pVCpu->hm.s.Event.fPending);
13483 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13484 }
13485
13486 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13487 AssertRCReturn(rc, rc);
13488
13489 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13490 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13491
13492 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13493 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13494 (RTGCPTR)pVmxTransient->uExitQualification);
13495
13496 Log4(("#PF: rc=%Rrc\n", rc));
13497 if (rc == VINF_SUCCESS)
13498 {
13499#if 0
13500 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13501 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
13502 * memory? We don't update the whole state here... */
13503 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13504 | HM_CHANGED_GUEST_RSP
13505 | HM_CHANGED_GUEST_RFLAGS
13506 | HM_CHANGED_VMX_GUEST_APIC_STATE);
13507#else
13508 /*
13509 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13510 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13511 */
13512 /** @todo take advantage of CPUM changed flags instead of brute forcing. */
13513 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13514#endif
13515 TRPMResetTrap(pVCpu);
13516 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13517 return rc;
13518 }
13519
13520 if (rc == VINF_EM_RAW_GUEST_TRAP)
13521 {
13522 if (!pVmxTransient->fVectoringDoublePF)
13523 {
13524 /* It's a guest page fault and needs to be reflected to the guest. */
13525 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13526 TRPMResetTrap(pVCpu);
13527 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13528 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13529 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13530 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13531 }
13532 else
13533 {
13534 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13535 TRPMResetTrap(pVCpu);
13536 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13537 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13538 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
13539 }
13540
13541 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13542 return VINF_SUCCESS;
13543 }
13544
13545 TRPMResetTrap(pVCpu);
13546 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13547 return rc;
13548}
13549
13550/** @} */
13551
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette