VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 74251

Last change on this file since 74251 was 74225, checked in by vboxsync, 6 years ago

VMX: Custom debug build warning fix for Windows hosts.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 575.3 KB
Line 
1/* $Id: HMVMXR0.cpp 74225 2018-09-12 15:35:39Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/x86.h>
25#include <iprt/asm-amd64-x86.h>
26#include <iprt/thread.h>
27
28#include <VBox/vmm/pdmapi.h>
29#include <VBox/vmm/dbgf.h>
30#include <VBox/vmm/iem.h>
31#include <VBox/vmm/iom.h>
32#include <VBox/vmm/selm.h>
33#include <VBox/vmm/tm.h>
34#include <VBox/vmm/em.h>
35#include <VBox/vmm/gim.h>
36#include <VBox/vmm/apic.h>
37#ifdef VBOX_WITH_REM
38# include <VBox/vmm/rem.h>
39#endif
40#include "HMInternal.h"
41#include <VBox/vmm/vm.h>
42#include "HMVMXR0.h"
43#include "dtrace/VBoxVMM.h"
44
45#ifdef DEBUG_ramshankar
46# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
47# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
49# define HMVMX_ALWAYS_CHECK_GUEST_STATE
50# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
51# define HMVMX_ALWAYS_TRAP_PF
52# define HMVMX_ALWAYS_FLUSH_TLB
53# define HMVMX_ALWAYS_SWAP_EFER
54#endif
55
56
57/*********************************************************************************************************************************
58* Defined Constants And Macros *
59*********************************************************************************************************************************/
60/** Use the function table. */
61#define HMVMX_USE_FUNCTION_TABLE
62
63/** Determine which tagged-TLB flush handler to use. */
64#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
65#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
66#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
67#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
68
69/** @name HMVMX_READ_XXX
70 * Flags to skip redundant reads of some common VMCS fields that are not part of
71 * the guest-CPU or VCPU state but are needed while handling VM-exits.
72 */
73#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
74#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
75#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
76#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
77#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
78#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
79#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
80/** @} */
81
82/**
83 * States of the VMCS.
84 *
85 * This does not reflect all possible VMCS states but currently only those
86 * needed for maintaining the VMCS consistently even when thread-context hooks
87 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
88 */
89#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
90#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
91#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
92
93/**
94 * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
95 * guest using hardware-assisted VMX.
96 *
97 * This excludes state like GPRs (other than RSP) which are always are
98 * swapped and restored across the world-switch and also registers like EFER,
99 * MSR which cannot be modified by the guest without causing a VM-exit.
100 */
101#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
102 | CPUMCTX_EXTRN_RFLAGS \
103 | CPUMCTX_EXTRN_RSP \
104 | CPUMCTX_EXTRN_SREG_MASK \
105 | CPUMCTX_EXTRN_TABLE_MASK \
106 | CPUMCTX_EXTRN_KERNEL_GS_BASE \
107 | CPUMCTX_EXTRN_SYSCALL_MSRS \
108 | CPUMCTX_EXTRN_SYSENTER_MSRS \
109 | CPUMCTX_EXTRN_TSC_AUX \
110 | CPUMCTX_EXTRN_OTHER_MSRS \
111 | CPUMCTX_EXTRN_CR0 \
112 | CPUMCTX_EXTRN_CR3 \
113 | CPUMCTX_EXTRN_CR4 \
114 | CPUMCTX_EXTRN_DR7 \
115 | CPUMCTX_EXTRN_HM_VMX_MASK)
116
117/**
118 * Exception bitmap mask for real-mode guests (real-on-v86).
119 *
120 * We need to intercept all exceptions manually except:
121 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
122 * due to bugs in Intel CPUs.
123 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
124 * support.
125 */
126#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
127 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
128 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
129 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
130 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
131 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
132 | RT_BIT(X86_XCPT_XF))
133
134/** Maximum VM-instruction error number. */
135#define HMVMX_INSTR_ERROR_MAX 28
136
137/** Profiling macro. */
138#ifdef HM_PROFILE_EXIT_DISPATCH
139# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
140# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
141#else
142# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
143# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
144#endif
145
146/** Assert that preemption is disabled or covered by thread-context hooks. */
147#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
148 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
149
150/** Assert that we haven't migrated CPUs when thread-context hooks are not
151 * used. */
152#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
153 || (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
154 ("Illegal migration! Entered on CPU %u Current %u\n", \
155 (a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
156
157/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
158 * context. */
159#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
160 ("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
161 (a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
162
163/** Helper macro for VM-exit handlers called unexpectedly. */
164#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
165 do { \
166 (a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
167 return VERR_VMX_UNEXPECTED_EXIT; \
168 } while (0)
169
170/** Macro for importing segment registers to the VMCS from the guest-CPU context. */
171#ifdef VMX_USE_CACHED_VMCS_ACCESSES
172# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
173 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
174 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
175#else
176# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
177 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
178 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
179#endif
180
181/** Macro for exporting segment registers to the VMCS from the guest-CPU context. */
182#define HMVMX_EXPORT_SREG(Sel, a_pCtxSelReg) \
183 hmR0VmxExportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
184 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
185
186#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
187/** Macro that does the necessary privilege checks and intercepted VM-exits for
188 * guests that attempted to execute a VMX instruction. */
189# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
190 do \
191 { \
192 VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
193 if (rcStrictTmp == VINF_SUCCESS) \
194 { /* likely */ } \
195 else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
196 { \
197 Assert((a_pVCpu)->hm.s.Event.fPending); \
198 Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
199 return VINF_SUCCESS; \
200 } \
201 else \
202 { \
203 int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
204 AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
205 } \
206 } while (0)
207
208/** Macro that decodes a memory operand for an instruction VM-exit. */
209# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
210 do \
211 { \
212 VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
213 (a_pGCPtrEffAddr)); \
214 if (rcStrictTmp == VINF_SUCCESS) \
215 { /* likely */ } \
216 else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
217 { \
218 uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
219 Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
220 return VINF_SUCCESS; \
221 } \
222 else \
223 { \
224 Log4Func(("hmR0VmxCheckExitDueToVmxInstr failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
225 return rcStrictTmp; \
226 } \
227 } while (0)
228
229#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
230
231
232/*********************************************************************************************************************************
233* Structures and Typedefs *
234*********************************************************************************************************************************/
235/**
236 * VMX transient state.
237 *
238 * A state structure for holding miscellaneous information across
239 * VMX non-root operation and restored after the transition.
240 */
241typedef struct VMXTRANSIENT
242{
243 /** The host's rflags/eflags. */
244 RTCCUINTREG fEFlags;
245#if HC_ARCH_BITS == 32
246 uint32_t u32Alignment0;
247#endif
248 /** The guest's TPR value used for TPR shadowing. */
249 uint8_t u8GuestTpr;
250 /** Alignment. */
251 uint8_t abAlignment0[7];
252
253 /** The basic VM-exit reason. */
254 uint16_t uExitReason;
255 /** Alignment. */
256 uint16_t u16Alignment0;
257 /** The VM-exit interruption error code. */
258 uint32_t uExitIntErrorCode;
259 /** The VM-exit exit code qualification. */
260 uint64_t uExitQual;
261
262 /** The VM-exit interruption-information field. */
263 uint32_t uExitIntInfo;
264 /** The VM-exit instruction-length field. */
265 uint32_t cbInstr;
266 /** The VM-exit instruction-information field. */
267 VMXEXITINSTRINFO ExitInstrInfo;
268 /** Whether the VM-entry failed or not. */
269 bool fVMEntryFailed;
270 /** Alignment. */
271 uint8_t abAlignment1[3];
272
273 /** The VM-entry interruption-information field. */
274 uint32_t uEntryIntInfo;
275 /** The VM-entry exception error code field. */
276 uint32_t uEntryXcptErrorCode;
277 /** The VM-entry instruction length field. */
278 uint32_t cbEntryInstr;
279
280 /** IDT-vectoring information field. */
281 uint32_t uIdtVectoringInfo;
282 /** IDT-vectoring error code. */
283 uint32_t uIdtVectoringErrorCode;
284
285 /** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
286 uint32_t fVmcsFieldsRead;
287
288 /** Whether the guest debug state was active at the time of VM-exit. */
289 bool fWasGuestDebugStateActive;
290 /** Whether the hyper debug state was active at the time of VM-exit. */
291 bool fWasHyperDebugStateActive;
292 /** Whether TSC-offsetting should be setup before VM-entry. */
293 bool fUpdateTscOffsettingAndPreemptTimer;
294 /** Whether the VM-exit was caused by a page-fault during delivery of a
295 * contributory exception or a page-fault. */
296 bool fVectoringDoublePF;
297 /** Whether the VM-exit was caused by a page-fault during delivery of an
298 * external interrupt or NMI. */
299 bool fVectoringPF;
300} VMXTRANSIENT;
301AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
302AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
303AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
304AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
305AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
306/** Pointer to VMX transient state. */
307typedef VMXTRANSIENT *PVMXTRANSIENT;
308
309/**
310 * MSR-bitmap read permissions.
311 */
312typedef enum VMXMSREXITREAD
313{
314 /** Reading this MSR causes a VM-exit. */
315 VMXMSREXIT_INTERCEPT_READ = 0xb,
316 /** Reading this MSR does not cause a VM-exit. */
317 VMXMSREXIT_PASSTHRU_READ
318} VMXMSREXITREAD;
319/** Pointer to MSR-bitmap read permissions. */
320typedef VMXMSREXITREAD* PVMXMSREXITREAD;
321
322/**
323 * MSR-bitmap write permissions.
324 */
325typedef enum VMXMSREXITWRITE
326{
327 /** Writing to this MSR causes a VM-exit. */
328 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
329 /** Writing to this MSR does not cause a VM-exit. */
330 VMXMSREXIT_PASSTHRU_WRITE
331} VMXMSREXITWRITE;
332/** Pointer to MSR-bitmap write permissions. */
333typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
334
335/**
336 * Memory operand read or write access.
337 */
338typedef enum VMXMEMACCESS
339{
340 VMXMEMACCESS_READ = 0,
341 VMXMEMACCESS_WRITE = 1
342} VMXMEMACCESS;
343
344/**
345 * VMX VM-exit handler.
346 *
347 * @returns Strict VBox status code (i.e. informational status codes too).
348 * @param pVCpu The cross context virtual CPU structure.
349 * @param pVmxTransient Pointer to the VMX-transient structure.
350 */
351#ifndef HMVMX_USE_FUNCTION_TABLE
352typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
353#else
354typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
355/** Pointer to VM-exit handler. */
356typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
357#endif
358
359/**
360 * VMX VM-exit handler, non-strict status code.
361 *
362 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
363 *
364 * @returns VBox status code, no informational status code returned.
365 * @param pVCpu The cross context virtual CPU structure.
366 * @param pVmxTransient Pointer to the VMX-transient structure.
367 *
368 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
369 * use of that status code will be replaced with VINF_EM_SOMETHING
370 * later when switching over to IEM.
371 */
372#ifndef HMVMX_USE_FUNCTION_TABLE
373typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
374#else
375typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
376#endif
377
378
379/*********************************************************************************************************************************
380* Internal Functions *
381*********************************************************************************************************************************/
382static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush);
383static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr);
384static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
385static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat);
386static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
387 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState);
388#if HC_ARCH_BITS == 32
389static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
390#endif
391#ifndef HMVMX_USE_FUNCTION_TABLE
392DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
393# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
394# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
395#else
396# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
397# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
398#endif
399
400/** @name VM-exit handlers.
401 * @{
402 */
403static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
404static FNVMXEXITHANDLER hmR0VmxExitExtInt;
405static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
406static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
407static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
408static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
409static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
410static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
411static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
412static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
413static FNVMXEXITHANDLER hmR0VmxExitCpuid;
414static FNVMXEXITHANDLER hmR0VmxExitGetsec;
415static FNVMXEXITHANDLER hmR0VmxExitHlt;
416static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
417static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
418static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
419static FNVMXEXITHANDLER hmR0VmxExitVmcall;
420#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
421static FNVMXEXITHANDLER hmR0VmxExitVmclear;
422static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
423static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
424static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
425static FNVMXEXITHANDLER hmR0VmxExitVmread;
426static FNVMXEXITHANDLER hmR0VmxExitVmresume;
427static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
428static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
429static FNVMXEXITHANDLER hmR0VmxExitVmxon;
430#endif
431static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
432static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
433static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
434static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
435static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
436static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
437static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
438static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
439static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
440static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
441static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
442static FNVMXEXITHANDLER hmR0VmxExitMwait;
443static FNVMXEXITHANDLER hmR0VmxExitMtf;
444static FNVMXEXITHANDLER hmR0VmxExitMonitor;
445static FNVMXEXITHANDLER hmR0VmxExitPause;
446static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
447static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
448static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
449static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
450static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
451static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
452static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
453static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
454static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
455static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
456static FNVMXEXITHANDLER hmR0VmxExitRdrand;
457static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
458/** @} */
459
460static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
461static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
462static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
463static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
464static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
465static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
466static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
467static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu);
468
469
470/*********************************************************************************************************************************
471* Global Variables *
472*********************************************************************************************************************************/
473#ifdef HMVMX_USE_FUNCTION_TABLE
474
475/**
476 * VMX_EXIT dispatch table.
477 */
478static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
479{
480 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
481 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
482 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
483 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
484 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
485 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
486 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
487 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
488 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
489 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
490 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
491 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
492 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
493 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
494 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
495 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
496 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
497 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
498 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
499#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
500 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
501 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
502 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
503 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
504 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
505 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
506 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
507 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
508 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
509#else
510 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
511 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
512 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
513 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
514 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
515 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
516 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
517 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
518 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
519#endif
520 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
521 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
522 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
523 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
524 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
525 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
526 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
527 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
528 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
529 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
530 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
531 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
532 /* 40 UNDEFINED */ hmR0VmxExitPause,
533 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
534 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
535 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
536 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
537 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
538 /* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitXdtrAccess,
539 /* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitXdtrAccess,
540 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
541 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
542 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
543 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
544 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
545 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
546 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
547 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
548 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
549 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
550 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
551 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
552 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
553 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
554 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
555 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
556 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
557};
558#endif /* HMVMX_USE_FUNCTION_TABLE */
559
560#ifdef VBOX_STRICT
561static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
562{
563 /* 0 */ "(Not Used)",
564 /* 1 */ "VMCALL executed in VMX root operation.",
565 /* 2 */ "VMCLEAR with invalid physical address.",
566 /* 3 */ "VMCLEAR with VMXON pointer.",
567 /* 4 */ "VMLAUNCH with non-clear VMCS.",
568 /* 5 */ "VMRESUME with non-launched VMCS.",
569 /* 6 */ "VMRESUME after VMXOFF",
570 /* 7 */ "VM-entry with invalid control fields.",
571 /* 8 */ "VM-entry with invalid host state fields.",
572 /* 9 */ "VMPTRLD with invalid physical address.",
573 /* 10 */ "VMPTRLD with VMXON pointer.",
574 /* 11 */ "VMPTRLD with incorrect revision identifier.",
575 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
576 /* 13 */ "VMWRITE to read-only VMCS component.",
577 /* 14 */ "(Not Used)",
578 /* 15 */ "VMXON executed in VMX root operation.",
579 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
580 /* 17 */ "VM-entry with non-launched executing VMCS.",
581 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
582 /* 19 */ "VMCALL with non-clear VMCS.",
583 /* 20 */ "VMCALL with invalid VM-exit control fields.",
584 /* 21 */ "(Not Used)",
585 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
586 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
587 /* 24 */ "VMCALL with invalid SMM-monitor features.",
588 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
589 /* 26 */ "VM-entry with events blocked by MOV SS.",
590 /* 27 */ "(Not Used)",
591 /* 28 */ "Invalid operand to INVEPT/INVVPID."
592};
593#endif /* VBOX_STRICT */
594
595
596/**
597 * Updates the VM's last error record.
598 *
599 * If there was a VMX instruction error, reads the error data from the VMCS and
600 * updates VCPU's last error record as well.
601 *
602 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
603 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
604 * VERR_VMX_INVALID_VMCS_FIELD.
605 * @param rc The error code.
606 */
607static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
608{
609 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
610 || rc == VERR_VMX_UNABLE_TO_START_VM)
611 {
612 AssertPtrReturnVoid(pVCpu);
613 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
614 }
615 pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
616}
617
618
619/**
620 * Reads the VM-entry interruption-information field from the VMCS into the VMX
621 * transient structure.
622 *
623 * @returns VBox status code.
624 * @param pVmxTransient Pointer to the VMX transient structure.
625 *
626 * @remarks No-long-jump zone!!!
627 */
628DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
629{
630 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
631 AssertRCReturn(rc, rc);
632 return VINF_SUCCESS;
633}
634
635#ifdef VBOX_STRICT
636/**
637 * Reads the VM-entry exception error code field from the VMCS into
638 * the VMX transient structure.
639 *
640 * @returns VBox status code.
641 * @param pVmxTransient Pointer to the VMX transient structure.
642 *
643 * @remarks No-long-jump zone!!!
644 */
645DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
646{
647 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
648 AssertRCReturn(rc, rc);
649 return VINF_SUCCESS;
650}
651
652
653/**
654 * Reads the VM-entry exception error code field from the VMCS into
655 * the VMX transient structure.
656 *
657 * @returns VBox status code.
658 * @param pVmxTransient Pointer to the VMX transient structure.
659 *
660 * @remarks No-long-jump zone!!!
661 */
662DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
663{
664 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
665 AssertRCReturn(rc, rc);
666 return VINF_SUCCESS;
667}
668#endif /* VBOX_STRICT */
669
670
671/**
672 * Reads the VM-exit interruption-information field from the VMCS into the VMX
673 * transient structure.
674 *
675 * @returns VBox status code.
676 * @param pVmxTransient Pointer to the VMX transient structure.
677 */
678DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
679{
680 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
681 {
682 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
683 AssertRCReturn(rc,rc);
684 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_INFO;
685 }
686 return VINF_SUCCESS;
687}
688
689
690/**
691 * Reads the VM-exit interruption error code from the VMCS into the VMX
692 * transient structure.
693 *
694 * @returns VBox status code.
695 * @param pVmxTransient Pointer to the VMX transient structure.
696 */
697DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
698{
699 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
700 {
701 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
702 AssertRCReturn(rc, rc);
703 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
704 }
705 return VINF_SUCCESS;
706}
707
708
709/**
710 * Reads the VM-exit instruction length field from the VMCS into the VMX
711 * transient structure.
712 *
713 * @returns VBox status code.
714 * @param pVmxTransient Pointer to the VMX transient structure.
715 */
716DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
717{
718 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
719 {
720 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
721 AssertRCReturn(rc, rc);
722 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_LEN;
723 }
724 return VINF_SUCCESS;
725}
726
727
728/**
729 * Reads the VM-exit instruction-information field from the VMCS into
730 * the VMX transient structure.
731 *
732 * @returns VBox status code.
733 * @param pVmxTransient Pointer to the VMX transient structure.
734 */
735DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
736{
737 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
738 {
739 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
740 AssertRCReturn(rc, rc);
741 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_INFO;
742 }
743 return VINF_SUCCESS;
744}
745
746
747/**
748 * Reads the exit code qualification from the VMCS into the VMX transient
749 * structure.
750 *
751 * @returns VBox status code.
752 * @param pVCpu The cross context virtual CPU structure of the
753 * calling EMT. (Required for the VMCS cache case.)
754 * @param pVmxTransient Pointer to the VMX transient structure.
755 */
756DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
757{
758 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
759 {
760 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
761 AssertRCReturn(rc, rc);
762 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION;
763 }
764 return VINF_SUCCESS;
765}
766
767
768/**
769 * Reads the IDT-vectoring information field from the VMCS into the VMX
770 * transient structure.
771 *
772 * @returns VBox status code.
773 * @param pVmxTransient Pointer to the VMX transient structure.
774 *
775 * @remarks No-long-jump zone!!!
776 */
777DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
778{
779 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
780 {
781 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
782 AssertRCReturn(rc, rc);
783 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_INFO;
784 }
785 return VINF_SUCCESS;
786}
787
788
789/**
790 * Reads the IDT-vectoring error code from the VMCS into the VMX
791 * transient structure.
792 *
793 * @returns VBox status code.
794 * @param pVmxTransient Pointer to the VMX transient structure.
795 */
796DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
797{
798 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
799 {
800 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
801 AssertRCReturn(rc, rc);
802 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
803 }
804 return VINF_SUCCESS;
805}
806
807
808/**
809 * Enters VMX root mode operation on the current CPU.
810 *
811 * @returns VBox status code.
812 * @param pVM The cross context VM structure. Can be
813 * NULL, after a resume.
814 * @param HCPhysCpuPage Physical address of the VMXON region.
815 * @param pvCpuPage Pointer to the VMXON region.
816 */
817static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
818{
819 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
820 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
821 Assert(pvCpuPage);
822 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
823
824 if (pVM)
825 {
826 /* Write the VMCS revision dword to the VMXON region. */
827 *(uint32_t *)pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
828 }
829
830 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
831 RTCCUINTREG fEFlags = ASMIntDisableFlags();
832
833 /* Enable the VMX bit in CR4 if necessary. */
834 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
835
836 /* Enter VMX root mode. */
837 int rc = VMXEnable(HCPhysCpuPage);
838 if (RT_FAILURE(rc))
839 {
840 if (!(uOldCr4 & X86_CR4_VMXE))
841 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
842
843 if (pVM)
844 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
845 }
846
847 /* Restore interrupts. */
848 ASMSetFlags(fEFlags);
849 return rc;
850}
851
852
853/**
854 * Exits VMX root mode operation on the current CPU.
855 *
856 * @returns VBox status code.
857 */
858static int hmR0VmxLeaveRootMode(void)
859{
860 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
861
862 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
863 RTCCUINTREG fEFlags = ASMIntDisableFlags();
864
865 /* If we're for some reason not in VMX root mode, then don't leave it. */
866 RTCCUINTREG uHostCR4 = ASMGetCR4();
867
868 int rc;
869 if (uHostCR4 & X86_CR4_VMXE)
870 {
871 /* Exit VMX root mode and clear the VMX bit in CR4. */
872 VMXDisable();
873 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
874 rc = VINF_SUCCESS;
875 }
876 else
877 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
878
879 /* Restore interrupts. */
880 ASMSetFlags(fEFlags);
881 return rc;
882}
883
884
885/**
886 * Allocates and maps one physically contiguous page. The allocated page is
887 * zero'd out. (Used by various VT-x structures).
888 *
889 * @returns IPRT status code.
890 * @param pMemObj Pointer to the ring-0 memory object.
891 * @param ppVirt Where to store the virtual address of the
892 * allocation.
893 * @param pHCPhys Where to store the physical address of the
894 * allocation.
895 */
896static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
897{
898 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
899 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
900 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
901
902 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
903 if (RT_FAILURE(rc))
904 return rc;
905 *ppVirt = RTR0MemObjAddress(*pMemObj);
906 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
907 ASMMemZero32(*ppVirt, PAGE_SIZE);
908 return VINF_SUCCESS;
909}
910
911
912/**
913 * Frees and unmaps an allocated physical page.
914 *
915 * @param pMemObj Pointer to the ring-0 memory object.
916 * @param ppVirt Where to re-initialize the virtual address of
917 * allocation as 0.
918 * @param pHCPhys Where to re-initialize the physical address of the
919 * allocation as 0.
920 */
921static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
922{
923 AssertPtr(pMemObj);
924 AssertPtr(ppVirt);
925 AssertPtr(pHCPhys);
926 if (*pMemObj != NIL_RTR0MEMOBJ)
927 {
928 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
929 AssertRC(rc);
930 *pMemObj = NIL_RTR0MEMOBJ;
931 *ppVirt = 0;
932 *pHCPhys = 0;
933 }
934}
935
936
937/**
938 * Worker function to free VT-x related structures.
939 *
940 * @returns IPRT status code.
941 * @param pVM The cross context VM structure.
942 */
943static void hmR0VmxStructsFree(PVM pVM)
944{
945 for (VMCPUID i = 0; i < pVM->cCpus; i++)
946 {
947 PVMCPU pVCpu = &pVM->aCpus[i];
948 AssertPtr(pVCpu);
949
950 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
951 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
952
953 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
954 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
955
956 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
957 }
958
959 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
960#ifdef VBOX_WITH_CRASHDUMP_MAGIC
961 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
962#endif
963}
964
965
966/**
967 * Worker function to allocate VT-x related VM structures.
968 *
969 * @returns IPRT status code.
970 * @param pVM The cross context VM structure.
971 */
972static int hmR0VmxStructsAlloc(PVM pVM)
973{
974 /*
975 * Initialize members up-front so we can cleanup properly on allocation failure.
976 */
977#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
978 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
979 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
980 pVM->hm.s.vmx.HCPhys##a_Name = 0;
981
982#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
983 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
984 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
985 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
986
987#ifdef VBOX_WITH_CRASHDUMP_MAGIC
988 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
989#endif
990 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
991
992 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
993 for (VMCPUID i = 0; i < pVM->cCpus; i++)
994 {
995 PVMCPU pVCpu = &pVM->aCpus[i];
996 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
997 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
998 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
999 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
1000 }
1001#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
1002#undef VMXLOCAL_INIT_VM_MEMOBJ
1003
1004 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
1005 AssertReturnStmt(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE) <= PAGE_SIZE,
1006 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
1007 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
1008
1009 /*
1010 * Allocate all the VT-x structures.
1011 */
1012 int rc = VINF_SUCCESS;
1013#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1014 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1015 if (RT_FAILURE(rc))
1016 goto cleanup;
1017 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1018 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1019#endif
1020
1021 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1022 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1023 {
1024 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1025 &pVM->hm.s.vmx.HCPhysApicAccess);
1026 if (RT_FAILURE(rc))
1027 goto cleanup;
1028 }
1029
1030 /*
1031 * Initialize per-VCPU VT-x structures.
1032 */
1033 for (VMCPUID i = 0; i < pVM->cCpus; i++)
1034 {
1035 PVMCPU pVCpu = &pVM->aCpus[i];
1036 AssertPtr(pVCpu);
1037
1038 /* Allocate the VM control structure (VMCS). */
1039 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1040 if (RT_FAILURE(rc))
1041 goto cleanup;
1042
1043 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1044 if ( PDMHasApic(pVM)
1045 && (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1046 {
1047 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1048 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1049 if (RT_FAILURE(rc))
1050 goto cleanup;
1051 }
1052
1053 /*
1054 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1055 * transparent accesses of specific MSRs.
1056 *
1057 * If the condition for enabling MSR bitmaps changes here, don't forget to
1058 * update HMAreMsrBitmapsAvailable().
1059 */
1060 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1061 {
1062 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1063 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1064 if (RT_FAILURE(rc))
1065 goto cleanup;
1066 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1067 }
1068
1069 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1070 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1071 if (RT_FAILURE(rc))
1072 goto cleanup;
1073
1074 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1075 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1076 if (RT_FAILURE(rc))
1077 goto cleanup;
1078 }
1079
1080 return VINF_SUCCESS;
1081
1082cleanup:
1083 hmR0VmxStructsFree(pVM);
1084 return rc;
1085}
1086
1087
1088/**
1089 * Does global VT-x initialization (called during module initialization).
1090 *
1091 * @returns VBox status code.
1092 */
1093VMMR0DECL(int) VMXR0GlobalInit(void)
1094{
1095#ifdef HMVMX_USE_FUNCTION_TABLE
1096 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1097# ifdef VBOX_STRICT
1098 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1099 Assert(g_apfnVMExitHandlers[i]);
1100# endif
1101#endif
1102 return VINF_SUCCESS;
1103}
1104
1105
1106/**
1107 * Does global VT-x termination (called during module termination).
1108 */
1109VMMR0DECL(void) VMXR0GlobalTerm()
1110{
1111 /* Nothing to do currently. */
1112}
1113
1114
1115/**
1116 * Sets up and activates VT-x on the current CPU.
1117 *
1118 * @returns VBox status code.
1119 * @param pHostCpu Pointer to the global CPU info struct.
1120 * @param pVM The cross context VM structure. Can be
1121 * NULL after a host resume operation.
1122 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1123 * fEnabledByHost is @c true).
1124 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1125 * @a fEnabledByHost is @c true).
1126 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1127 * enable VT-x on the host.
1128 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1129 */
1130VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1131 void *pvMsrs)
1132{
1133 Assert(pHostCpu);
1134 Assert(pvMsrs);
1135 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1136
1137 /* Enable VT-x if it's not already enabled by the host. */
1138 if (!fEnabledByHost)
1139 {
1140 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1141 if (RT_FAILURE(rc))
1142 return rc;
1143 }
1144
1145 /*
1146 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
1147 * using EPTPs) so we don't retain any stale guest-physical mappings which won't get
1148 * invalidated when flushing by VPID.
1149 */
1150 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1151 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1152 {
1153 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
1154 pHostCpu->fFlushAsidBeforeUse = false;
1155 }
1156 else
1157 pHostCpu->fFlushAsidBeforeUse = true;
1158
1159 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1160 ++pHostCpu->cTlbFlushes;
1161
1162 return VINF_SUCCESS;
1163}
1164
1165
1166/**
1167 * Deactivates VT-x on the current CPU.
1168 *
1169 * @returns VBox status code.
1170 * @param pHostCpu Pointer to the global CPU info struct.
1171 * @param pvCpuPage Pointer to the VMXON region.
1172 * @param HCPhysCpuPage Physical address of the VMXON region.
1173 *
1174 * @remarks This function should never be called when SUPR0EnableVTx() or
1175 * similar was used to enable VT-x on the host.
1176 */
1177VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1178{
1179 RT_NOREF3(pHostCpu, pvCpuPage, HCPhysCpuPage);
1180
1181 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1182 return hmR0VmxLeaveRootMode();
1183}
1184
1185
1186/**
1187 * Sets the permission bits for the specified MSR in the MSR bitmap.
1188 *
1189 * @param pVCpu The cross context virtual CPU structure.
1190 * @param uMsr The MSR value.
1191 * @param enmRead Whether reading this MSR causes a VM-exit.
1192 * @param enmWrite Whether writing this MSR causes a VM-exit.
1193 */
1194static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1195{
1196 int32_t iBit;
1197 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1198
1199 /*
1200 * Layout:
1201 * 0x000 - 0x3ff - Low MSR read bits
1202 * 0x400 - 0x7ff - High MSR read bits
1203 * 0x800 - 0xbff - Low MSR write bits
1204 * 0xc00 - 0xfff - High MSR write bits
1205 */
1206 if (uMsr <= 0x00001fff)
1207 iBit = uMsr;
1208 else if (uMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1209 {
1210 iBit = uMsr - UINT32_C(0xc0000000);
1211 pbMsrBitmap += 0x400;
1212 }
1213 else
1214 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1215
1216 Assert(iBit <= 0x1fff);
1217 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1218 ASMBitSet(pbMsrBitmap, iBit);
1219 else
1220 ASMBitClear(pbMsrBitmap, iBit);
1221
1222 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1223 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1224 else
1225 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1226}
1227
1228
1229#ifdef VBOX_STRICT
1230/**
1231 * Gets the permission bits for the specified MSR in the MSR bitmap.
1232 *
1233 * @returns VBox status code.
1234 * @retval VINF_SUCCESS if the specified MSR is found.
1235 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1236 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1237 *
1238 * @param pVCpu The cross context virtual CPU structure.
1239 * @param uMsr The MSR.
1240 * @param penmRead Where to store the read permissions.
1241 * @param penmWrite Where to store the write permissions.
1242 */
1243static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1244{
1245 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1246 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1247 int32_t iBit;
1248 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1249
1250 /* See hmR0VmxSetMsrPermission() for the layout. */
1251 if (uMsr <= 0x00001fff)
1252 iBit = uMsr;
1253 else if ( uMsr >= 0xc0000000
1254 && uMsr <= 0xc0001fff)
1255 {
1256 iBit = (uMsr - 0xc0000000);
1257 pbMsrBitmap += 0x400;
1258 }
1259 else
1260 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1261
1262 Assert(iBit <= 0x1fff);
1263 if (ASMBitTest(pbMsrBitmap, iBit))
1264 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1265 else
1266 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1267
1268 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1269 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1270 else
1271 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1272 return VINF_SUCCESS;
1273}
1274#endif /* VBOX_STRICT */
1275
1276
1277/**
1278 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1279 * area.
1280 *
1281 * @returns VBox status code.
1282 * @param pVCpu The cross context virtual CPU structure.
1283 * @param cMsrs The number of MSRs.
1284 */
1285static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1286{
1287 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1288 uint64_t const uVmxMiscMsr = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc;
1289 uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(uVmxMiscMsr);
1290 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1291 {
1292 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1293 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1294 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1295 }
1296
1297 /* Update number of guest MSRs to load/store across the world-switch. */
1298 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1299 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1300
1301 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1302 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1303 AssertRCReturn(rc, rc);
1304
1305 /* Update the VCPU's copy of the MSR count. */
1306 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1307
1308 return VINF_SUCCESS;
1309}
1310
1311
1312/**
1313 * Adds a new (or updates the value of an existing) guest/host MSR
1314 * pair to be swapped during the world-switch as part of the
1315 * auto-load/store MSR area in the VMCS.
1316 *
1317 * @returns VBox status code.
1318 * @param pVCpu The cross context virtual CPU structure.
1319 * @param uMsr The MSR.
1320 * @param uGuestMsrValue Value of the guest MSR.
1321 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1322 * necessary.
1323 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1324 * its value was updated. Optional, can be NULL.
1325 */
1326static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1327 bool *pfAddedAndUpdated)
1328{
1329 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1330 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1331 uint32_t i;
1332 for (i = 0; i < cMsrs; i++)
1333 {
1334 if (pGuestMsr->u32Msr == uMsr)
1335 break;
1336 pGuestMsr++;
1337 }
1338
1339 bool fAdded = false;
1340 if (i == cMsrs)
1341 {
1342 ++cMsrs;
1343 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1344 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1345
1346 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1347 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1348 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1349
1350 fAdded = true;
1351 }
1352
1353 /* Update the MSR values in the auto-load/store MSR area. */
1354 pGuestMsr->u32Msr = uMsr;
1355 pGuestMsr->u64Value = uGuestMsrValue;
1356
1357 /* Create/update the MSR slot in the host MSR area. */
1358 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1359 pHostMsr += i;
1360 pHostMsr->u32Msr = uMsr;
1361
1362 /*
1363 * Update the host MSR only when requested by the caller AND when we're
1364 * adding it to the auto-load/store area. Otherwise, it would have been
1365 * updated by hmR0VmxExportHostMsrs(). We do this for performance reasons.
1366 */
1367 bool fUpdatedMsrValue = false;
1368 if ( fAdded
1369 && fUpdateHostMsr)
1370 {
1371 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1372 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1373 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1374 fUpdatedMsrValue = true;
1375 }
1376
1377 if (pfAddedAndUpdated)
1378 *pfAddedAndUpdated = fUpdatedMsrValue;
1379 return VINF_SUCCESS;
1380}
1381
1382
1383/**
1384 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1385 * auto-load/store MSR area in the VMCS.
1386 *
1387 * @returns VBox status code.
1388 * @param pVCpu The cross context virtual CPU structure.
1389 * @param uMsr The MSR.
1390 */
1391static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1392{
1393 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1394 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1395 for (uint32_t i = 0; i < cMsrs; i++)
1396 {
1397 /* Find the MSR. */
1398 if (pGuestMsr->u32Msr == uMsr)
1399 {
1400 /* If it's the last MSR, simply reduce the count. */
1401 if (i == cMsrs - 1)
1402 {
1403 --cMsrs;
1404 break;
1405 }
1406
1407 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1408 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1409 pLastGuestMsr += cMsrs - 1;
1410 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1411 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1412
1413 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1414 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1415 pLastHostMsr += cMsrs - 1;
1416 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1417 pHostMsr->u64Value = pLastHostMsr->u64Value;
1418 --cMsrs;
1419 break;
1420 }
1421 pGuestMsr++;
1422 }
1423
1424 /* Update the VMCS if the count changed (meaning the MSR was found). */
1425 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1426 {
1427 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1428 AssertRCReturn(rc, rc);
1429
1430 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1431 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1432 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1433
1434 Log4Func(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1435 return VINF_SUCCESS;
1436 }
1437
1438 return VERR_NOT_FOUND;
1439}
1440
1441
1442/**
1443 * Checks if the specified guest MSR is part of the auto-load/store area in
1444 * the VMCS.
1445 *
1446 * @returns true if found, false otherwise.
1447 * @param pVCpu The cross context virtual CPU structure.
1448 * @param uMsr The MSR to find.
1449 */
1450static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1451{
1452 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1453 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1454
1455 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1456 {
1457 if (pGuestMsr->u32Msr == uMsr)
1458 return true;
1459 }
1460 return false;
1461}
1462
1463
1464/**
1465 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1466 *
1467 * @param pVCpu The cross context virtual CPU structure.
1468 *
1469 * @remarks No-long-jump zone!!!
1470 */
1471static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1472{
1473 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1474 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1475 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1476 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1477
1478 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1479 {
1480 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1481
1482 /*
1483 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1484 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1485 */
1486 if (pHostMsr->u32Msr == MSR_K6_EFER)
1487 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1488 else
1489 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1490 }
1491
1492 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1493}
1494
1495
1496/**
1497 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1498 * perform lazy restoration of the host MSRs while leaving VT-x.
1499 *
1500 * @param pVCpu The cross context virtual CPU structure.
1501 *
1502 * @remarks No-long-jump zone!!!
1503 */
1504static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1505{
1506 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1507
1508 /*
1509 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1510 */
1511 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1512 {
1513 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1514#if HC_ARCH_BITS == 64
1515 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1516 {
1517 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1518 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1519 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1520 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1521 }
1522#endif
1523 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1524 }
1525}
1526
1527
1528/**
1529 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1530 * lazily while leaving VT-x.
1531 *
1532 * @returns true if it does, false otherwise.
1533 * @param pVCpu The cross context virtual CPU structure.
1534 * @param uMsr The MSR to check.
1535 */
1536static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1537{
1538 NOREF(pVCpu);
1539#if HC_ARCH_BITS == 64
1540 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1541 {
1542 switch (uMsr)
1543 {
1544 case MSR_K8_LSTAR:
1545 case MSR_K6_STAR:
1546 case MSR_K8_SF_MASK:
1547 case MSR_K8_KERNEL_GS_BASE:
1548 return true;
1549 }
1550 }
1551#else
1552 RT_NOREF(pVCpu, uMsr);
1553#endif
1554 return false;
1555}
1556
1557
1558/**
1559 * Loads a set of guests MSRs to allow read/passthru to the guest.
1560 *
1561 * The name of this function is slightly confusing. This function does NOT
1562 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1563 * common prefix for functions dealing with "lazy restoration" of the shared
1564 * MSRs.
1565 *
1566 * @param pVCpu The cross context virtual CPU structure.
1567 *
1568 * @remarks No-long-jump zone!!!
1569 */
1570static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1571{
1572 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1573 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1574
1575 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1576#if HC_ARCH_BITS == 64
1577 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1578 {
1579 /*
1580 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1581 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1582 * we can skip a few MSR writes.
1583 *
1584 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1585 * guest MSR values in the guest-CPU context might be different to what's currently
1586 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1587 * CPU, see @bugref{8728}.
1588 */
1589 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1590 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1591 && pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1592 && pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1593 && pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1594 && pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1595 {
1596#ifdef VBOX_STRICT
1597 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1598 Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1599 Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
1600 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
1601#endif
1602 }
1603 else
1604 {
1605 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1606 ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
1607 ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
1608 ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
1609 }
1610 }
1611#endif
1612 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1613}
1614
1615
1616/**
1617 * Performs lazy restoration of the set of host MSRs if they were previously
1618 * loaded with guest MSR values.
1619 *
1620 * @param pVCpu The cross context virtual CPU structure.
1621 *
1622 * @remarks No-long-jump zone!!!
1623 * @remarks The guest MSRs should have been saved back into the guest-CPU
1624 * context by hmR0VmxImportGuestState()!!!
1625 */
1626static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1627{
1628 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1629 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1630
1631 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1632 {
1633 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1634#if HC_ARCH_BITS == 64
1635 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1636 {
1637 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1638 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1639 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1640 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1641 }
1642#endif
1643 }
1644 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1645}
1646
1647
1648/**
1649 * Verifies that our cached values of the VMCS fields are all consistent with
1650 * what's actually present in the VMCS.
1651 *
1652 * @returns VBox status code.
1653 * @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
1654 * @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
1655 * VMCS content. HMCPU error-field is
1656 * updated, see VMX_VCI_XXX.
1657 * @param pVCpu The cross context virtual CPU structure.
1658 */
1659static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1660{
1661 uint32_t u32Val;
1662 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1663 AssertRCReturn(rc, rc);
1664 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32EntryCtls == u32Val,
1665 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1666 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
1667 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1668
1669 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1670 AssertRCReturn(rc, rc);
1671 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ExitCtls == u32Val,
1672 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1673 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
1674 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1675
1676 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1677 AssertRCReturn(rc, rc);
1678 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32PinCtls == u32Val,
1679 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1680 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
1681 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1682
1683 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1684 AssertRCReturn(rc, rc);
1685 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls == u32Val,
1686 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1687 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
1688 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1689
1690 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
1691 {
1692 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1693 AssertRCReturn(rc, rc);
1694 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1695 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1696 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
1697 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1698 }
1699
1700 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
1701 AssertRCReturn(rc, rc);
1702 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32XcptBitmap == u32Val,
1703 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap, u32Val),
1704 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
1705 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1706
1707 uint64_t u64Val;
1708 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
1709 AssertRCReturn(rc, rc);
1710 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u64TscOffset == u64Val,
1711 ("Cache=%#RX64 VMCS=%#RX64\n", pVCpu->hm.s.vmx.u64TscOffset, u64Val),
1712 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
1713 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1714
1715 return VINF_SUCCESS;
1716}
1717
1718
1719#ifdef VBOX_STRICT
1720/**
1721 * Verifies that our cached host EFER value has not changed
1722 * since we cached it.
1723 *
1724 * @param pVCpu The cross context virtual CPU structure.
1725 */
1726static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1727{
1728 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1729
1730 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
1731 {
1732 uint64_t u64Val;
1733 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1734 AssertRC(rc);
1735
1736 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1737 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1738 }
1739}
1740
1741
1742/**
1743 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1744 * VMCS are correct.
1745 *
1746 * @param pVCpu The cross context virtual CPU structure.
1747 */
1748static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1749{
1750 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1751
1752 /* Verify MSR counts in the VMCS are what we think it should be. */
1753 uint32_t cMsrs;
1754 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1755 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1756
1757 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1758 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1759
1760 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1761 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1762
1763 PCVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1764 PCVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1765 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1766 {
1767 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1768 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1769 pGuestMsr->u32Msr, cMsrs));
1770
1771 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1772 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1773 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1774
1775 /* Verify that the permissions are as expected in the MSR bitmap. */
1776 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1777 {
1778 VMXMSREXITREAD enmRead;
1779 VMXMSREXITWRITE enmWrite;
1780 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1781 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1782 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1783 {
1784 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1785 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1786 }
1787 else
1788 {
1789 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1790 pGuestMsr->u32Msr, cMsrs));
1791 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1792 pGuestMsr->u32Msr, cMsrs));
1793 }
1794 }
1795 }
1796}
1797#endif /* VBOX_STRICT */
1798
1799
1800/**
1801 * Flushes the TLB using EPT.
1802 *
1803 * @returns VBox status code.
1804 * @param pVCpu The cross context virtual CPU structure of the calling
1805 * EMT. Can be NULL depending on @a enmTlbFlush.
1806 * @param enmTlbFlush Type of flush.
1807 *
1808 * @remarks Caller is responsible for making sure this function is called only
1809 * when NestedPaging is supported and providing @a enmTlbFlush that is
1810 * supported by the CPU.
1811 * @remarks Can be called with interrupts disabled.
1812 */
1813static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush)
1814{
1815 uint64_t au64Descriptor[2];
1816 if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
1817 au64Descriptor[0] = 0;
1818 else
1819 {
1820 Assert(pVCpu);
1821 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1822 }
1823 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1824
1825 int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
1826 AssertMsg(rc == VINF_SUCCESS,
1827 ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0, rc));
1828
1829 if ( RT_SUCCESS(rc)
1830 && pVCpu)
1831 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1832}
1833
1834
1835/**
1836 * Flushes the TLB using VPID.
1837 *
1838 * @returns VBox status code.
1839 * @param pVCpu The cross context virtual CPU structure of the calling
1840 * EMT. Can be NULL depending on @a enmTlbFlush.
1841 * @param enmTlbFlush Type of flush.
1842 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1843 * on @a enmTlbFlush).
1844 *
1845 * @remarks Can be called with interrupts disabled.
1846 */
1847static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
1848{
1849 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
1850
1851 uint64_t au64Descriptor[2];
1852 if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
1853 {
1854 au64Descriptor[0] = 0;
1855 au64Descriptor[1] = 0;
1856 }
1857 else
1858 {
1859 AssertPtr(pVCpu);
1860 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1861 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1862 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1863 au64Descriptor[1] = GCPtr;
1864 }
1865
1866 int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
1867 AssertMsg(rc == VINF_SUCCESS,
1868 ("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1869
1870 if ( RT_SUCCESS(rc)
1871 && pVCpu)
1872 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1873 NOREF(rc);
1874}
1875
1876
1877/**
1878 * Invalidates a guest page by guest virtual address. Only relevant for
1879 * EPT/VPID, otherwise there is nothing really to invalidate.
1880 *
1881 * @returns VBox status code.
1882 * @param pVCpu The cross context virtual CPU structure.
1883 * @param GCVirt Guest virtual address of the page to invalidate.
1884 */
1885VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
1886{
1887 AssertPtr(pVCpu);
1888 LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
1889
1890 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1891 if (!fFlushPending)
1892 {
1893 /*
1894 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1895 * the EPT case. See @bugref{6043} and @bugref{6177}.
1896 *
1897 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1898 * as this function maybe called in a loop with individual addresses.
1899 */
1900 PVM pVM = pVCpu->CTX_SUFF(pVM);
1901 if (pVM->hm.s.vmx.fVpid)
1902 {
1903 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1904
1905#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1906 /*
1907 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1908 * where executing INVVPID outside 64-bit mode does not flush translations of
1909 * 64-bit linear addresses, see @bugref{6208#c72}.
1910 */
1911 if (RT_HI_U32(GCVirt))
1912 fVpidFlush = false;
1913#endif
1914
1915 if (fVpidFlush)
1916 {
1917 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
1918 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1919 }
1920 else
1921 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1922 }
1923 else if (pVM->hm.s.fNestedPaging)
1924 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1925 }
1926
1927 return VINF_SUCCESS;
1928}
1929
1930
1931/**
1932 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1933 * case where neither EPT nor VPID is supported by the CPU.
1934 *
1935 * @param pVCpu The cross context virtual CPU structure.
1936 * @param pCpu Pointer to the global HM struct.
1937 *
1938 * @remarks Called with interrupts disabled.
1939 */
1940static void hmR0VmxFlushTaggedTlbNone(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1941{
1942 AssertPtr(pVCpu);
1943 AssertPtr(pCpu);
1944
1945 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1946
1947 Assert(pCpu->idCpu != NIL_RTCPUID);
1948 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1949 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1950 pVCpu->hm.s.fForceTLBFlush = false;
1951 return;
1952}
1953
1954
1955/**
1956 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1957 *
1958 * @param pVCpu The cross context virtual CPU structure.
1959 * @param pCpu Pointer to the global HM CPU struct.
1960 *
1961 * @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
1962 * nomenclature. The reason is, to avoid confusion in compare statements
1963 * since the host-CPU copies are named "ASID".
1964 *
1965 * @remarks Called with interrupts disabled.
1966 */
1967static void hmR0VmxFlushTaggedTlbBoth(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1968{
1969#ifdef VBOX_WITH_STATISTICS
1970 bool fTlbFlushed = false;
1971# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1972# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1973 if (!fTlbFlushed) \
1974 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1975 } while (0)
1976#else
1977# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1978# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1979#endif
1980
1981 AssertPtr(pCpu);
1982 AssertPtr(pVCpu);
1983 Assert(pCpu->idCpu != NIL_RTCPUID);
1984
1985 PVM pVM = pVCpu->CTX_SUFF(pVM);
1986 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1987 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1988 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1989
1990 /*
1991 * Force a TLB flush for the first world-switch if the current CPU differs from the one we
1992 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1993 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1994 * cannot reuse the current ASID anymore.
1995 */
1996 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1997 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1998 {
1999 ++pCpu->uCurrentAsid;
2000 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2001 {
2002 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2003 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2004 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2005 }
2006
2007 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2008 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2009 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2010
2011 /*
2012 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2013 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2014 */
2015 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
2016 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2017 HMVMX_SET_TAGGED_TLB_FLUSHED();
2018 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2019 }
2020 else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2021 {
2022 /*
2023 * Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2024 * creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2025 * tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2026 * VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2027 * mappings, see @bugref{6568}.
2028 *
2029 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2030 */
2031 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
2032 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2033 HMVMX_SET_TAGGED_TLB_FLUSHED();
2034 }
2035
2036 pVCpu->hm.s.fForceTLBFlush = false;
2037 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2038
2039 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2040 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2041 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2042 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2043 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2044 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2045 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2046 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2047 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2048
2049 /* Update VMCS with the VPID. */
2050 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2051 AssertRC(rc);
2052
2053#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2054}
2055
2056
2057/**
2058 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2059 *
2060 * @returns VBox status code.
2061 * @param pVCpu The cross context virtual CPU structure.
2062 * @param pCpu Pointer to the global HM CPU struct.
2063 *
2064 * @remarks Called with interrupts disabled.
2065 */
2066static void hmR0VmxFlushTaggedTlbEpt(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2067{
2068 AssertPtr(pVCpu);
2069 AssertPtr(pCpu);
2070 Assert(pCpu->idCpu != NIL_RTCPUID);
2071 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2072 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2073
2074 /*
2075 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2076 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2077 */
2078 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2079 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2080 {
2081 pVCpu->hm.s.fForceTLBFlush = true;
2082 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2083 }
2084
2085 /* Check for explicit TLB flushes. */
2086 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2087 {
2088 pVCpu->hm.s.fForceTLBFlush = true;
2089 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2090 }
2091
2092 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2093 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2094
2095 if (pVCpu->hm.s.fForceTLBFlush)
2096 {
2097 hmR0VmxFlushEpt(pVCpu, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2098 pVCpu->hm.s.fForceTLBFlush = false;
2099 }
2100}
2101
2102
2103/**
2104 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2105 *
2106 * @returns VBox status code.
2107 * @param pVCpu The cross context virtual CPU structure.
2108 * @param pCpu Pointer to the global HM CPU struct.
2109 *
2110 * @remarks Called with interrupts disabled.
2111 */
2112static void hmR0VmxFlushTaggedTlbVpid(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2113{
2114 AssertPtr(pVCpu);
2115 AssertPtr(pCpu);
2116 Assert(pCpu->idCpu != NIL_RTCPUID);
2117 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2118 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2119
2120 /*
2121 * Force a TLB flush for the first world switch if the current CPU differs from the one we
2122 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2123 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2124 * cannot reuse the current ASID anymore.
2125 */
2126 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2127 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2128 {
2129 pVCpu->hm.s.fForceTLBFlush = true;
2130 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2131 }
2132
2133 /* Check for explicit TLB flushes. */
2134 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2135 {
2136 /*
2137 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2138 * hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2139 * fExplicitFlush = true here and change the pCpu->fFlushAsidBeforeUse check below to
2140 * include fExplicitFlush's too) - an obscure corner case.
2141 */
2142 pVCpu->hm.s.fForceTLBFlush = true;
2143 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2144 }
2145
2146 PVM pVM = pVCpu->CTX_SUFF(pVM);
2147 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2148 if (pVCpu->hm.s.fForceTLBFlush)
2149 {
2150 ++pCpu->uCurrentAsid;
2151 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2152 {
2153 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2154 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2155 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2156 }
2157
2158 pVCpu->hm.s.fForceTLBFlush = false;
2159 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2160 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2161 if (pCpu->fFlushAsidBeforeUse)
2162 {
2163 if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2164 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2165 else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2166 {
2167 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2168 pCpu->fFlushAsidBeforeUse = false;
2169 }
2170 else
2171 {
2172 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2173 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2174 }
2175 }
2176 }
2177
2178 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2179 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2180 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2181 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2182 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2183 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2184 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2185
2186 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2187 AssertRC(rc);
2188}
2189
2190
2191/**
2192 * Flushes the guest TLB entry based on CPU capabilities.
2193 *
2194 * @param pVCpu The cross context virtual CPU structure.
2195 * @param pCpu Pointer to the global HM CPU struct.
2196 */
2197DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2198{
2199#ifdef HMVMX_ALWAYS_FLUSH_TLB
2200 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2201#endif
2202 PVM pVM = pVCpu->CTX_SUFF(pVM);
2203 switch (pVM->hm.s.vmx.enmTlbFlushType)
2204 {
2205 case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVCpu, pCpu); break;
2206 case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pVCpu, pCpu); break;
2207 case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pVCpu, pCpu); break;
2208 case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pVCpu, pCpu); break;
2209 default:
2210 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2211 break;
2212 }
2213 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2214}
2215
2216
2217/**
2218 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2219 * TLB entries from the host TLB before VM-entry.
2220 *
2221 * @returns VBox status code.
2222 * @param pVM The cross context VM structure.
2223 */
2224static int hmR0VmxSetupTaggedTlb(PVM pVM)
2225{
2226 /*
2227 * Determine optimal flush type for Nested Paging.
2228 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2229 * guest execution (see hmR3InitFinalizeR0()).
2230 */
2231 if (pVM->hm.s.fNestedPaging)
2232 {
2233 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2234 {
2235 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2236 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2237 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2238 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2239 else
2240 {
2241 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2242 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2243 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2244 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2245 }
2246
2247 /* Make sure the write-back cacheable memory type for EPT is supported. */
2248 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2249 {
2250 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2251 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2252 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2253 }
2254
2255 /* EPT requires a page-walk length of 4. */
2256 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2257 {
2258 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2259 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2260 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2261 }
2262 }
2263 else
2264 {
2265 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2266 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2267 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2268 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2269 }
2270 }
2271
2272 /*
2273 * Determine optimal flush type for VPID.
2274 */
2275 if (pVM->hm.s.vmx.fVpid)
2276 {
2277 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2278 {
2279 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2280 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2281 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2282 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2283 else
2284 {
2285 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2286 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2287 LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2288 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2289 LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2290 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2291 pVM->hm.s.vmx.fVpid = false;
2292 }
2293 }
2294 else
2295 {
2296 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2297 Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2298 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2299 pVM->hm.s.vmx.fVpid = false;
2300 }
2301 }
2302
2303 /*
2304 * Setup the handler for flushing tagged-TLBs.
2305 */
2306 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2307 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2308 else if (pVM->hm.s.fNestedPaging)
2309 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2310 else if (pVM->hm.s.vmx.fVpid)
2311 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2312 else
2313 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2314 return VINF_SUCCESS;
2315}
2316
2317
2318/**
2319 * Sets up pin-based VM-execution controls in the VMCS.
2320 *
2321 * @returns VBox status code.
2322 * @param pVCpu The cross context virtual CPU structure.
2323 *
2324 * @remarks We don't really care about optimizing vmwrites here as it's done only
2325 * once per VM and hence we don't care about VMCS-field cache comparisons.
2326 */
2327static int hmR0VmxSetupPinCtls(PVMCPU pVCpu)
2328{
2329 PVM pVM = pVCpu->CTX_SUFF(pVM);
2330 uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.disallowed0; /* Bits set here must always be set. */
2331 uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2332
2333 fVal |= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2334 | VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2335
2336 if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2337 fVal |= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2338
2339 /* Enable the VMX preemption timer. */
2340 if (pVM->hm.s.vmx.fUsePreemptTimer)
2341 {
2342 Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2343 fVal |= VMX_PIN_CTLS_PREEMPT_TIMER;
2344 }
2345
2346#if 0
2347 /* Enable posted-interrupt processing. */
2348 if (pVM->hm.s.fPostedIntrs)
2349 {
2350 Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2351 Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2352 fVal |= VMX_PIN_CTL_POSTED_INT;
2353 }
2354#endif
2355
2356 if ((fVal & fZap) != fVal)
2357 {
2358 LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2359 pVM->hm.s.vmx.Msrs.PinCtls.n.disallowed0, fVal, fZap));
2360 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2361 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2362 }
2363
2364 /* Commit it to the VMCS and update our cache. */
2365 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2366 AssertRCReturn(rc, rc);
2367 pVCpu->hm.s.vmx.u32PinCtls = fVal;
2368
2369 return VINF_SUCCESS;
2370}
2371
2372
2373/**
2374 * Sets up secondary processor-based VM-execution controls in the VMCS.
2375 *
2376 * @returns VBox status code.
2377 * @param pVCpu The cross context virtual CPU structure.
2378 *
2379 * @remarks We don't really care about optimizing vmwrites here as it's done only
2380 * once per VM and hence we don't care about VMCS-field cache comparisons.
2381 */
2382static int hmR0VmxSetupProcCtls2(PVMCPU pVCpu)
2383{
2384 PVM pVM = pVCpu->CTX_SUFF(pVM);
2385 uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2386 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2387
2388 /* WBINVD causes a VM-exit. */
2389 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2390 fVal |= VMX_PROC_CTLS2_WBINVD_EXIT;
2391
2392 /* Enable EPT (aka nested-paging). */
2393 if (pVM->hm.s.fNestedPaging)
2394 fVal |= VMX_PROC_CTLS2_EPT;
2395
2396 /*
2397 * Enable the INVPCID instruction if supported by the hardware and we expose
2398 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2399 */
2400 if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID)
2401 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2402 fVal |= VMX_PROC_CTLS2_INVPCID;
2403
2404 /* Enable VPID. */
2405 if (pVM->hm.s.vmx.fVpid)
2406 fVal |= VMX_PROC_CTLS2_VPID;
2407
2408 /* Enable Unrestricted guest execution. */
2409 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2410 fVal |= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
2411
2412#if 0
2413 if (pVM->hm.s.fVirtApicRegs)
2414 {
2415 /* Enable APIC-register virtualization. */
2416 Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
2417 fVal |= VMX_PROC_CTLS2_APIC_REG_VIRT;
2418
2419 /* Enable virtual-interrupt delivery. */
2420 Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
2421 fVal |= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
2422 }
2423#endif
2424
2425 /* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is where the TPR shadow resides. */
2426 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2427 * done dynamically. */
2428 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
2429 {
2430 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2431 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2432 fVal |= VMX_PROC_CTLS2_VIRT_APIC_ACCESS; /* Virtualize APIC accesses. */
2433 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2434 AssertRCReturn(rc, rc);
2435 }
2436
2437 /* Enable RDTSCP. */
2438 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP)
2439 fVal |= VMX_PROC_CTLS2_RDTSCP;
2440
2441 /* Enable Pause-Loop exiting. */
2442 if ( pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT
2443 && pVM->hm.s.vmx.cPleGapTicks
2444 && pVM->hm.s.vmx.cPleWindowTicks)
2445 {
2446 fVal |= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
2447
2448 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2449 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2450 AssertRCReturn(rc, rc);
2451 }
2452
2453 if ((fVal & fZap) != fVal)
2454 {
2455 LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2456 pVM->hm.s.vmx.Msrs.ProcCtls2.n.disallowed0, fVal, fZap));
2457 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2458 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2459 }
2460
2461 /* Commit it to the VMCS and update our cache. */
2462 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2463 AssertRCReturn(rc, rc);
2464 pVCpu->hm.s.vmx.u32ProcCtls2 = fVal;
2465
2466 return VINF_SUCCESS;
2467}
2468
2469
2470/**
2471 * Sets up processor-based VM-execution controls in the VMCS.
2472 *
2473 * @returns VBox status code.
2474 * @param pVCpu The cross context virtual CPU structure.
2475 *
2476 * @remarks We don't really care about optimizing vmwrites here as it's done only
2477 * once per VM and hence we don't care about VMCS-field cache comparisons.
2478 */
2479static int hmR0VmxSetupProcCtls(PVMCPU pVCpu)
2480{
2481 PVM pVM = pVCpu->CTX_SUFF(pVM);
2482 uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2483 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2484
2485 fVal |= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
2486 | VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2487 | VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2488 | VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2489 | VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2490 | VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2491 | VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2492
2493 /* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2494 if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
2495 || (pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
2496 {
2497 LogRelFunc(("Unsupported VMX_PROC_CTLS_MOV_DR_EXIT combo!"));
2498 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2499 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2500 }
2501
2502 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2503 if (!pVM->hm.s.fNestedPaging)
2504 {
2505 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2506 fVal |= VMX_PROC_CTLS_INVLPG_EXIT
2507 | VMX_PROC_CTLS_CR3_LOAD_EXIT
2508 | VMX_PROC_CTLS_CR3_STORE_EXIT;
2509 }
2510
2511 /* Use TPR shadowing if supported by the CPU. */
2512 if ( PDMHasApic(pVM)
2513 && pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
2514 {
2515 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2516 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2517 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2518 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2519 AssertRCReturn(rc, rc);
2520
2521 fVal |= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2522 /* CR8 writes cause a VM-exit based on TPR threshold. */
2523 Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
2524 Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
2525 }
2526 else
2527 {
2528 /*
2529 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2530 * Set this control only for 64-bit guests.
2531 */
2532 if (pVM->hm.s.fAllow64BitGuests)
2533 {
2534 fVal |= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2535 | VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2536 }
2537 }
2538
2539 /* Use MSR-bitmaps if supported by the CPU. */
2540 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2541 {
2542 fVal |= VMX_PROC_CTLS_USE_MSR_BITMAPS;
2543
2544 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2545 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2546 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2547 AssertRCReturn(rc, rc);
2548
2549 /*
2550 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2551 * automatically using dedicated fields in the VMCS.
2552 */
2553 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2554 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2555 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2556 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2557 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2558#if HC_ARCH_BITS == 64
2559 /*
2560 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2561 */
2562 if (pVM->hm.s.fAllow64BitGuests)
2563 {
2564 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2565 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2566 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2567 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2568 }
2569#endif
2570 /*
2571 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2572 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2573 */
2574 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2575 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2576
2577 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2578 }
2579
2580 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2581 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2582 fVal |= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
2583
2584 if ((fVal & fZap) != fVal)
2585 {
2586 LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2587 pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0, fVal, fZap));
2588 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2589 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2590 }
2591
2592 /* Commit it to the VMCS and update our cache. */
2593 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2594 AssertRCReturn(rc, rc);
2595 pVCpu->hm.s.vmx.u32ProcCtls = fVal;
2596
2597 /* Set up secondary processor-based VM-execution controls if the CPU supports it. */
2598 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2599 return hmR0VmxSetupProcCtls2(pVCpu);
2600
2601 /* Sanity check, should not really happen. */
2602 if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2603 {
2604 LogRelFunc(("Unrestricted Guest enabled when secondary processor-based VM-execution controls not available\n"));
2605 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2606 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2607 }
2608
2609 /* Old CPUs without secondary processor-based VM-execution controls would end up here. */
2610 return VINF_SUCCESS;
2611}
2612
2613
2614/**
2615 * Sets up miscellaneous (everything other than Pin & Processor-based
2616 * VM-execution) control fields in the VMCS.
2617 *
2618 * @returns VBox status code.
2619 * @param pVCpu The cross context virtual CPU structure.
2620 */
2621static int hmR0VmxSetupMiscCtls(PVMCPU pVCpu)
2622{
2623 AssertPtr(pVCpu);
2624
2625 int rc = VERR_GENERAL_FAILURE;
2626
2627 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2628#if 0
2629 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxExportGuestCR3AndCR4())*/
2630 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2631 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2632
2633 /*
2634 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2635 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2636 * We thus use the exception bitmap to control it rather than use both.
2637 */
2638 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2639 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2640
2641 /* All IO & IOIO instructions cause VM-exits. */
2642 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2643 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2644
2645 /* Initialize the MSR-bitmap area. */
2646 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2647 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2648 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2649 AssertRCReturn(rc, rc);
2650#endif
2651
2652 /* Setup MSR auto-load/store area. */
2653 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2654 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2655 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2656 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2657 AssertRCReturn(rc, rc);
2658
2659 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2660 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2661 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2662 AssertRCReturn(rc, rc);
2663
2664 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2665 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2666 AssertRCReturn(rc, rc);
2667
2668 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2669#if 0
2670 /* Setup debug controls */
2671 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2672 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2673 AssertRCReturn(rc, rc);
2674#endif
2675
2676 return rc;
2677}
2678
2679
2680/**
2681 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2682 *
2683 * We shall setup those exception intercepts that don't change during the
2684 * lifetime of the VM here. The rest are done dynamically while loading the
2685 * guest state.
2686 *
2687 * @returns VBox status code.
2688 * @param pVCpu The cross context virtual CPU structure.
2689 */
2690static int hmR0VmxInitXcptBitmap(PVMCPU pVCpu)
2691{
2692 AssertPtr(pVCpu);
2693
2694 uint32_t uXcptBitmap;
2695
2696 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2697 uXcptBitmap = RT_BIT_32(X86_XCPT_AC);
2698
2699 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2700 and writes, and because recursive #DBs can cause the CPU hang, we must always
2701 intercept #DB. */
2702 uXcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2703
2704 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2705 if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
2706 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
2707
2708 /* Commit it to the VMCS. */
2709 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
2710 AssertRCReturn(rc, rc);
2711
2712 /* Update our cache of the exception bitmap. */
2713 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
2714 return VINF_SUCCESS;
2715}
2716
2717
2718/**
2719 * Does per-VM VT-x initialization.
2720 *
2721 * @returns VBox status code.
2722 * @param pVM The cross context VM structure.
2723 */
2724VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2725{
2726 LogFlowFunc(("pVM=%p\n", pVM));
2727
2728 int rc = hmR0VmxStructsAlloc(pVM);
2729 if (RT_FAILURE(rc))
2730 {
2731 LogRelFunc(("hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2732 return rc;
2733 }
2734
2735 return VINF_SUCCESS;
2736}
2737
2738
2739/**
2740 * Does per-VM VT-x termination.
2741 *
2742 * @returns VBox status code.
2743 * @param pVM The cross context VM structure.
2744 */
2745VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2746{
2747 LogFlowFunc(("pVM=%p\n", pVM));
2748
2749#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2750 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2751 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2752#endif
2753 hmR0VmxStructsFree(pVM);
2754 return VINF_SUCCESS;
2755}
2756
2757
2758/**
2759 * Sets up the VM for execution under VT-x.
2760 * This function is only called once per-VM during initialization.
2761 *
2762 * @returns VBox status code.
2763 * @param pVM The cross context VM structure.
2764 */
2765VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2766{
2767 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2768 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2769
2770 LogFlowFunc(("pVM=%p\n", pVM));
2771
2772 /*
2773 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be
2774 * allocated. We no longer support the highly unlikely case of UnrestrictedGuest without
2775 * pRealModeTSS, see hmR3InitFinalizeR0Intel().
2776 */
2777 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2778 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2779 || !pVM->hm.s.vmx.pRealModeTSS))
2780 {
2781 LogRelFunc(("Invalid real-on-v86 state.\n"));
2782 return VERR_INTERNAL_ERROR;
2783 }
2784
2785 /* Initialize these always, see hmR3InitFinalizeR0().*/
2786 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
2787 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
2788
2789 /* Setup the tagged-TLB flush handlers. */
2790 int rc = hmR0VmxSetupTaggedTlb(pVM);
2791 if (RT_FAILURE(rc))
2792 {
2793 LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2794 return rc;
2795 }
2796
2797 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2798 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2799#if HC_ARCH_BITS == 64
2800 if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
2801 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2802 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
2803 {
2804 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2805 }
2806#endif
2807
2808 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2809 RTCCUINTREG const uHostCR4 = ASMGetCR4();
2810 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2811 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2812
2813 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2814 {
2815 PVMCPU pVCpu = &pVM->aCpus[i];
2816 AssertPtr(pVCpu);
2817 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2818
2819 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2820 Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2821
2822 /* Set revision dword at the beginning of the VMCS structure. */
2823 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
2824
2825 /* Set the VMCS launch state to "clear", see Intel spec. 31.6 "Preparation and launch a virtual machine". */
2826 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2827 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc\n", rc),
2828 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2829
2830 /* Load this VMCS as the current VMCS. */
2831 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2832 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc\n", rc),
2833 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2834
2835 rc = hmR0VmxSetupPinCtls(pVCpu);
2836 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc\n", rc),
2837 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2838
2839 rc = hmR0VmxSetupProcCtls(pVCpu);
2840 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc\n", rc),
2841 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2842
2843 rc = hmR0VmxSetupMiscCtls(pVCpu);
2844 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc\n", rc),
2845 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2846
2847 rc = hmR0VmxInitXcptBitmap(pVCpu);
2848 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc\n", rc),
2849 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2850
2851#if HC_ARCH_BITS == 32
2852 rc = hmR0VmxInitVmcsReadCache(pVCpu);
2853 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc\n", rc),
2854 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2855#endif
2856
2857 /* Sync any CPU internal VMCS data back into our VMCS in memory. */
2858 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2859 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc\n", rc),
2860 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2861
2862 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2863
2864 hmR0VmxUpdateErrorRecord(pVCpu, rc);
2865 }
2866
2867 return VINF_SUCCESS;
2868}
2869
2870
2871/**
2872 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2873 * the VMCS.
2874 *
2875 * @returns VBox status code.
2876 */
2877static int hmR0VmxExportHostControlRegs(void)
2878{
2879 RTCCUINTREG uReg = ASMGetCR0();
2880 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2881 AssertRCReturn(rc, rc);
2882
2883 uReg = ASMGetCR3();
2884 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2885 AssertRCReturn(rc, rc);
2886
2887 uReg = ASMGetCR4();
2888 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2889 AssertRCReturn(rc, rc);
2890 return rc;
2891}
2892
2893
2894/**
2895 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2896 * the host-state area in the VMCS.
2897 *
2898 * @returns VBox status code.
2899 * @param pVCpu The cross context virtual CPU structure.
2900 */
2901static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
2902{
2903#if HC_ARCH_BITS == 64
2904/**
2905 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2906 * requirements. See hmR0VmxExportHostSegmentRegs().
2907 */
2908# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2909 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2910 { \
2911 bool fValidSelector = true; \
2912 if ((selValue) & X86_SEL_LDT) \
2913 { \
2914 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2915 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2916 } \
2917 if (fValidSelector) \
2918 { \
2919 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2920 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2921 } \
2922 (selValue) = 0; \
2923 }
2924
2925 /*
2926 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2927 * should -not- save the messed up state without restoring the original host-state,
2928 * see @bugref{7240}.
2929 *
2930 * This apparently can happen (most likely the FPU changes), deal with it rather than
2931 * asserting. Was observed booting Solaris 10u10 32-bit guest.
2932 */
2933 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2934 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2935 {
2936 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2937 pVCpu->idCpu));
2938 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2939 }
2940 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2941#else
2942 RT_NOREF(pVCpu);
2943#endif
2944
2945 /*
2946 * Host DS, ES, FS and GS segment registers.
2947 */
2948#if HC_ARCH_BITS == 64
2949 RTSEL uSelDS = ASMGetDS();
2950 RTSEL uSelES = ASMGetES();
2951 RTSEL uSelFS = ASMGetFS();
2952 RTSEL uSelGS = ASMGetGS();
2953#else
2954 RTSEL uSelDS = 0;
2955 RTSEL uSelES = 0;
2956 RTSEL uSelFS = 0;
2957 RTSEL uSelGS = 0;
2958#endif
2959
2960 /*
2961 * Host CS and SS segment registers.
2962 */
2963 RTSEL uSelCS = ASMGetCS();
2964 RTSEL uSelSS = ASMGetSS();
2965
2966 /*
2967 * Host TR segment register.
2968 */
2969 RTSEL uSelTR = ASMGetTR();
2970
2971#if HC_ARCH_BITS == 64
2972 /*
2973 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
2974 * gain VM-entry and restore them before we get preempted.
2975 *
2976 * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2977 */
2978 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2979 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2980 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2981 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2982# undef VMXLOCAL_ADJUST_HOST_SEG
2983#endif
2984
2985 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2986 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2987 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2988 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2989 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2990 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2991 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2992 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2993 Assert(uSelCS);
2994 Assert(uSelTR);
2995
2996 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2997#if 0
2998 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE))
2999 Assert(uSelSS != 0);
3000#endif
3001
3002 /* Write these host selector fields into the host-state area in the VMCS. */
3003 int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3004 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3005#if HC_ARCH_BITS == 64
3006 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3007 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3008 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3009 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3010#else
3011 NOREF(uSelDS);
3012 NOREF(uSelES);
3013 NOREF(uSelFS);
3014 NOREF(uSelGS);
3015#endif
3016 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3017 AssertRCReturn(rc, rc);
3018
3019 /*
3020 * Host GDTR and IDTR.
3021 */
3022 RTGDTR Gdtr;
3023 RTIDTR Idtr;
3024 RT_ZERO(Gdtr);
3025 RT_ZERO(Idtr);
3026 ASMGetGDTR(&Gdtr);
3027 ASMGetIDTR(&Idtr);
3028 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3029 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3030 AssertRCReturn(rc, rc);
3031
3032#if HC_ARCH_BITS == 64
3033 /*
3034 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
3035 * them to the maximum limit (0xffff) on every VM-exit.
3036 */
3037 if (Gdtr.cbGdt != 0xffff)
3038 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3039
3040 /*
3041 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
3042 * Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
3043 * as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
3044 * However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
3045 * uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
3046 * alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
3047 * at 0xffff on hosts where we are sure it won't cause trouble.
3048 */
3049# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3050 if (Idtr.cbIdt < 0x0fff)
3051# else
3052 if (Idtr.cbIdt != 0xffff)
3053# endif
3054 {
3055 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3056 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3057 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3058 }
3059#endif
3060
3061 /*
3062 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3063 * and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3064 * RPL should be too in most cases.
3065 */
3066 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3067 ("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3068
3069 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3070#if HC_ARCH_BITS == 64
3071 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3072
3073 /*
3074 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3075 * all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3076 * restoration if the host has something else. Task switching is not supported in 64-bit
3077 * mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3078 * limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3079 *
3080 * [1] See Intel spec. 3.5 "System Descriptor Types".
3081 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3082 */
3083 PVM pVM = pVCpu->CTX_SUFF(pVM);
3084 Assert(pDesc->System.u4Type == 11);
3085 if ( pDesc->System.u16LimitLow != 0x67
3086 || pDesc->System.u4LimitHigh)
3087 {
3088 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3089 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3090 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3091 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3092 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3093 }
3094
3095 /*
3096 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3097 */
3098 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3099 {
3100 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3101 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3102 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3103 {
3104 /* The GDT is read-only but the writable GDT is available. */
3105 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3106 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3107 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3108 AssertRCReturn(rc, rc);
3109 }
3110 }
3111#else
3112 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3113#endif
3114 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3115 AssertRCReturn(rc, rc);
3116
3117 /*
3118 * Host FS base and GS base.
3119 */
3120#if HC_ARCH_BITS == 64
3121 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3122 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3123 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3124 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3125 AssertRCReturn(rc, rc);
3126
3127 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3128 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3129 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3130 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3131 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3132#endif
3133 return VINF_SUCCESS;
3134}
3135
3136
3137/**
3138 * Exports certain host MSRs in the VM-exit MSR-load area and some in the
3139 * host-state area of the VMCS.
3140 *
3141 * Theses MSRs will be automatically restored on the host after every successful
3142 * VM-exit.
3143 *
3144 * @returns VBox status code.
3145 * @param pVCpu The cross context virtual CPU structure.
3146 *
3147 * @remarks No-long-jump zone!!!
3148 */
3149static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
3150{
3151 AssertPtr(pVCpu);
3152 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3153
3154 /*
3155 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3156 * rather than swapping them on every VM-entry.
3157 */
3158 hmR0VmxLazySaveHostMsrs(pVCpu);
3159
3160 /*
3161 * Host Sysenter MSRs.
3162 */
3163 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3164#if HC_ARCH_BITS == 32
3165 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3166 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3167#else
3168 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3169 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3170#endif
3171 AssertRCReturn(rc, rc);
3172
3173 /*
3174 * Host EFER MSR.
3175 *
3176 * If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3177 * done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3178 */
3179 PVM pVM = pVCpu->CTX_SUFF(pVM);
3180 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3181 {
3182 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3183 AssertRCReturn(rc, rc);
3184 }
3185
3186 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see hmR0VmxExportGuestExitCtls(). */
3187
3188 return VINF_SUCCESS;
3189}
3190
3191
3192/**
3193 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3194 *
3195 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3196 * these two bits are handled by VM-entry, see hmR0VmxExportGuestExitCtls() and
3197 * hmR0VMxExportGuestEntryCtls().
3198 *
3199 * @returns true if we need to load guest EFER, false otherwise.
3200 * @param pVCpu The cross context virtual CPU structure.
3201 *
3202 * @remarks Requires EFER, CR4.
3203 * @remarks No-long-jump zone!!!
3204 */
3205static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu)
3206{
3207#ifdef HMVMX_ALWAYS_SWAP_EFER
3208 RT_NOREF(pVCpu);
3209 return true;
3210#else
3211
3212 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3213#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3214 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3215 if (CPUMIsGuestInLongModeEx(pCtx))
3216 return false;
3217#endif
3218
3219 PVM pVM = pVCpu->CTX_SUFF(pVM);
3220 uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3221 uint64_t const u64GuestEfer = pCtx->msrEFER;
3222
3223 /*
3224 * For 64-bit guests, if EFER.SCE bit differs, we need to swap EFER to ensure that the
3225 * guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3226 */
3227 if ( CPUMIsGuestInLongModeEx(pCtx)
3228 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3229 {
3230 return true;
3231 }
3232
3233 /*
3234 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3235 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3236 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3237 */
3238 if ( (pCtx->cr4 & X86_CR4_PAE)
3239 && (pCtx->cr0 & X86_CR0_PG)
3240 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3241 {
3242 /* Assert that host is NX capable. */
3243 Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
3244 return true;
3245 }
3246
3247 return false;
3248#endif
3249}
3250
3251
3252/**
3253 * Exports the guest state with appropriate VM-entry controls in the VMCS.
3254 *
3255 * These controls can affect things done on VM-exit; e.g. "load debug controls",
3256 * see Intel spec. 24.8.1 "VM-entry controls".
3257 *
3258 * @returns VBox status code.
3259 * @param pVCpu The cross context virtual CPU structure.
3260 *
3261 * @remarks Requires EFER.
3262 * @remarks No-long-jump zone!!!
3263 */
3264static int hmR0VmxExportGuestEntryCtls(PVMCPU pVCpu)
3265{
3266 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_CTLS)
3267 {
3268 PVM pVM = pVCpu->CTX_SUFF(pVM);
3269 uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
3270 uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3271
3272 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3273 fVal |= VMX_ENTRY_CTLS_LOAD_DEBUG;
3274
3275 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3276 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
3277 {
3278 fVal |= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
3279 Log4Func(("VMX_ENTRY_CTLS_IA32E_MODE_GUEST\n"));
3280 }
3281 else
3282 Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
3283
3284 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3285 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3286 && hmR0VmxShouldSwapEferMsr(pVCpu))
3287 {
3288 fVal |= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
3289 Log4Func(("VMX_ENTRY_CTLS_LOAD_EFER_MSR\n"));
3290 }
3291
3292 /*
3293 * The following should -not- be set (since we're not in SMM mode):
3294 * - VMX_ENTRY_CTLS_ENTRY_TO_SMM
3295 * - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
3296 */
3297
3298 /** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
3299 * VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
3300
3301 if ((fVal & fZap) != fVal)
3302 {
3303 Log4Func(("Invalid VM-entry controls combo! Cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3304 pVM->hm.s.vmx.Msrs.EntryCtls.n.disallowed0, fVal, fZap));
3305 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3306 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3307 }
3308
3309 /* Commit it to the VMCS and update our cache. */
3310 if (pVCpu->hm.s.vmx.u32EntryCtls != fVal)
3311 {
3312 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
3313 AssertRCReturn(rc, rc);
3314 pVCpu->hm.s.vmx.u32EntryCtls = fVal;
3315 }
3316
3317 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_CTLS);
3318 }
3319 return VINF_SUCCESS;
3320}
3321
3322
3323/**
3324 * Exports the guest state with appropriate VM-exit controls in the VMCS.
3325 *
3326 * @returns VBox status code.
3327 * @param pVCpu The cross context virtual CPU structure.
3328 *
3329 * @remarks Requires EFER.
3330 */
3331static int hmR0VmxExportGuestExitCtls(PVMCPU pVCpu)
3332{
3333 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_EXIT_CTLS)
3334 {
3335 PVM pVM = pVCpu->CTX_SUFF(pVM);
3336 uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
3337 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3338
3339 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3340 fVal |= VMX_EXIT_CTLS_SAVE_DEBUG;
3341
3342 /*
3343 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3344 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in
3345 * hmR0VmxExportHostMsrs().
3346 */
3347#if HC_ARCH_BITS == 64
3348 fVal |= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
3349 Log4Func(("VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE\n"));
3350#else
3351 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3352 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3353 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3354 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3355 {
3356 /* The switcher returns to long mode, EFER is managed by the switcher. */
3357 fVal |= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
3358 Log4Func(("VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE\n"));
3359 }
3360 else
3361 Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
3362#endif
3363
3364 /* If the newer VMCS fields for managing EFER exists, use it. */
3365 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3366 && hmR0VmxShouldSwapEferMsr(pVCpu))
3367 {
3368 fVal |= VMX_EXIT_CTLS_SAVE_EFER_MSR
3369 | VMX_EXIT_CTLS_LOAD_EFER_MSR;
3370 Log4Func(("VMX_EXIT_CTLS_SAVE_EFER_MSR and VMX_EXIT_CTLS_LOAD_EFER_MSR\n"));
3371 }
3372
3373 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3374 Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
3375
3376 /** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
3377 * VMX_EXIT_CTLS_SAVE_PAT_MSR,
3378 * VMX_EXIT_CTLS_LOAD_PAT_MSR. */
3379
3380 /* Enable saving of the VMX preemption timer value on VM-exit. */
3381 if ( pVM->hm.s.vmx.fUsePreemptTimer
3382 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
3383 fVal |= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
3384
3385 if ((fVal & fZap) != fVal)
3386 {
3387 LogRelFunc(("Invalid VM-exit controls combo! cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3388 pVM->hm.s.vmx.Msrs.ExitCtls.n.disallowed0, fVal, fZap));
3389 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3390 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3391 }
3392
3393 /* Commit it to the VMCS and update our cache. */
3394 if (pVCpu->hm.s.vmx.u32ExitCtls != fVal)
3395 {
3396 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
3397 AssertRCReturn(rc, rc);
3398 pVCpu->hm.s.vmx.u32ExitCtls = fVal;
3399 }
3400
3401 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_EXIT_CTLS);
3402 }
3403 return VINF_SUCCESS;
3404}
3405
3406
3407/**
3408 * Sets the TPR threshold in the VMCS.
3409 *
3410 * @returns VBox status code.
3411 * @param pVCpu The cross context virtual CPU structure.
3412 * @param u32TprThreshold The TPR threshold (task-priority class only).
3413 */
3414DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3415{
3416 Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
3417 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3418 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3419}
3420
3421
3422/**
3423 * Exports the guest APIC TPR state into the VMCS.
3424 *
3425 * @returns VBox status code.
3426 * @param pVCpu The cross context virtual CPU structure.
3427 *
3428 * @remarks No-long-jump zone!!!
3429 */
3430static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu)
3431{
3432 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
3433 {
3434 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
3435
3436 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3437 && APICIsEnabled(pVCpu))
3438 {
3439 /*
3440 * Setup TPR shadowing.
3441 */
3442 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
3443 {
3444 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3445
3446 bool fPendingIntr = false;
3447 uint8_t u8Tpr = 0;
3448 uint8_t u8PendingIntr = 0;
3449 int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3450 AssertRCReturn(rc, rc);
3451
3452 /*
3453 * If there are interrupts pending but masked by the TPR, instruct VT-x to
3454 * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
3455 * priority of the pending interrupt so we can deliver the interrupt. If there
3456 * are no interrupts pending, set threshold to 0 to not cause any
3457 * TPR-below-threshold VM-exits.
3458 */
3459 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3460 uint32_t u32TprThreshold = 0;
3461 if (fPendingIntr)
3462 {
3463 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3464 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3465 const uint8_t u8TprPriority = u8Tpr >> 4;
3466 if (u8PendingPriority <= u8TprPriority)
3467 u32TprThreshold = u8PendingPriority;
3468 }
3469
3470 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3471 AssertRCReturn(rc, rc);
3472 }
3473 }
3474 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
3475 }
3476 return VINF_SUCCESS;
3477}
3478
3479
3480/**
3481 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3482 *
3483 * @returns Guest's interruptibility-state.
3484 * @param pVCpu The cross context virtual CPU structure.
3485 *
3486 * @remarks No-long-jump zone!!!
3487 */
3488static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu)
3489{
3490 /*
3491 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3492 */
3493 uint32_t fIntrState = 0;
3494 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3495 {
3496 /* If inhibition is active, RIP & RFLAGS should've been accessed
3497 (i.e. read previously from the VMCS or from ring-3). */
3498 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3499#ifdef VBOX_STRICT
3500 uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
3501 AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
3502#endif
3503 if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3504 {
3505 if (pCtx->eflags.Bits.u1IF)
3506 fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
3507 else
3508 fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
3509 }
3510 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3511 {
3512 /*
3513 * We can clear the inhibit force flag as even if we go back to the recompiler
3514 * without executing guest code in VT-x, the flag's condition to be cleared is
3515 * met and thus the cleared state is correct.
3516 */
3517 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3518 }
3519 }
3520
3521 /*
3522 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3523 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3524 * setting this would block host-NMIs and IRET will not clear the blocking.
3525 *
3526 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3527 */
3528 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3529 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
3530 {
3531 fIntrState |= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
3532 }
3533
3534 return fIntrState;
3535}
3536
3537
3538/**
3539 * Exports the guest's interruptibility-state into the guest-state area in the
3540 * VMCS.
3541 *
3542 * @returns VBox status code.
3543 * @param pVCpu The cross context virtual CPU structure.
3544 * @param fIntrState The interruptibility-state to set.
3545 */
3546static int hmR0VmxExportGuestIntrState(PVMCPU pVCpu, uint32_t fIntrState)
3547{
3548 NOREF(pVCpu);
3549 AssertMsg(!(fIntrState & 0xfffffff0), ("%#x\n", fIntrState)); /* Bits 31:4 MBZ. */
3550 Assert((fIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3551 return VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
3552}
3553
3554
3555/**
3556 * Exports the exception intercepts required for guest execution in the VMCS.
3557 *
3558 * @returns VBox status code.
3559 * @param pVCpu The cross context virtual CPU structure.
3560 *
3561 * @remarks No-long-jump zone!!!
3562 */
3563static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu)
3564{
3565 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
3566 {
3567 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3568
3569 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxExportSharedCR0(). */
3570 if (pVCpu->hm.s.fGIMTrapXcptUD)
3571 uXcptBitmap |= RT_BIT(X86_XCPT_UD);
3572#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3573 else
3574 uXcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3575#endif
3576
3577 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_AC));
3578 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_DB));
3579
3580 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3581 {
3582 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3583 AssertRCReturn(rc, rc);
3584 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3585 }
3586
3587 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3588 Log4Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64\n", uXcptBitmap));
3589 }
3590 return VINF_SUCCESS;
3591}
3592
3593
3594/**
3595 * Exports the guest's RIP into the guest-state area in the VMCS.
3596 *
3597 * @returns VBox status code.
3598 * @param pVCpu The cross context virtual CPU structure.
3599 *
3600 * @remarks No-long-jump zone!!!
3601 */
3602static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
3603{
3604 int rc = VINF_SUCCESS;
3605 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
3606 {
3607 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
3608
3609 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
3610 AssertRCReturn(rc, rc);
3611
3612 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3613 Log4Func(("RIP=%#RX64\n", pVCpu->cpum.GstCtx.rip));
3614 }
3615 return rc;
3616}
3617
3618
3619/**
3620 * Exports the guest's RSP into the guest-state area in the VMCS.
3621 *
3622 * @returns VBox status code.
3623 * @param pVCpu The cross context virtual CPU structure.
3624 *
3625 * @remarks No-long-jump zone!!!
3626 */
3627static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
3628{
3629 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3630 {
3631 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
3632
3633 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
3634 AssertRCReturn(rc, rc);
3635
3636 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3637 }
3638 return VINF_SUCCESS;
3639}
3640
3641
3642/**
3643 * Exports the guest's RFLAGS into the guest-state area in the VMCS.
3644 *
3645 * @returns VBox status code.
3646 * @param pVCpu The cross context virtual CPU structure.
3647 *
3648 * @remarks No-long-jump zone!!!
3649 */
3650static int hmR0VmxExportGuestRflags(PVMCPU pVCpu)
3651{
3652 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3653 {
3654 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
3655
3656 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3657 Let us assert it as such and use 32-bit VMWRITE. */
3658 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
3659 X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
3660 Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3661 Assert(!(fEFlags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3662
3663 /*
3664 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3665 * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3666 * can run the real-mode guest code under Virtual 8086 mode.
3667 */
3668 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3669 {
3670 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3671 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3672 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3673 fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3674 fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3675 }
3676
3677 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3678 AssertRCReturn(rc, rc);
3679
3680 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3681 Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
3682 }
3683 return VINF_SUCCESS;
3684}
3685
3686
3687/**
3688 * Exports the guest CR0 control register into the guest-state area in the VMCS.
3689 *
3690 * The guest FPU state is always pre-loaded hence we don't need to bother about
3691 * sharing FPU related CR0 bits between the guest and host.
3692 *
3693 * @returns VBox status code.
3694 * @param pVCpu The cross context virtual CPU structure.
3695 *
3696 * @remarks No-long-jump zone!!!
3697 */
3698static int hmR0VmxExportGuestCR0(PVMCPU pVCpu)
3699{
3700 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
3701 {
3702 PVM pVM = pVCpu->CTX_SUFF(pVM);
3703 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
3704 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.cr0));
3705
3706 uint32_t const u32ShadowCr0 = pVCpu->cpum.GstCtx.cr0;
3707 uint32_t u32GuestCr0 = pVCpu->cpum.GstCtx.cr0;
3708
3709 /*
3710 * Setup VT-x's view of the guest CR0.
3711 * Minimize VM-exits due to CR3 changes when we have NestedPaging.
3712 */
3713 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
3714 if (pVM->hm.s.fNestedPaging)
3715 {
3716 if (CPUMIsGuestPagingEnabled(pVCpu))
3717 {
3718 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3719 uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
3720 | VMX_PROC_CTLS_CR3_STORE_EXIT);
3721 }
3722 else
3723 {
3724 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3725 uProcCtls |= VMX_PROC_CTLS_CR3_LOAD_EXIT
3726 | VMX_PROC_CTLS_CR3_STORE_EXIT;
3727 }
3728
3729 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3730 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3731 uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
3732 }
3733 else
3734 {
3735 /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3736 u32GuestCr0 |= X86_CR0_WP;
3737 }
3738
3739 /*
3740 * Guest FPU bits.
3741 *
3742 * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3743 * using CR0.TS.
3744 *
3745 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3746 * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3747 */
3748 u32GuestCr0 |= X86_CR0_NE;
3749
3750 /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3751 bool const fInterceptMF = !(u32ShadowCr0 & X86_CR0_NE);
3752
3753 /*
3754 * Update exception intercepts.
3755 */
3756 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3757 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3758 {
3759 Assert(PDMVmmDevHeapIsEnabled(pVM));
3760 Assert(pVM->hm.s.vmx.pRealModeTSS);
3761 uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3762 }
3763 else
3764 {
3765 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3766 uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3767 if (fInterceptMF)
3768 uXcptBitmap |= RT_BIT(X86_XCPT_MF);
3769 }
3770
3771 /* Additional intercepts for debugging, define these yourself explicitly. */
3772#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3773 uXcptBitmap |= 0
3774 | RT_BIT(X86_XCPT_BP)
3775 | RT_BIT(X86_XCPT_DE)
3776 | RT_BIT(X86_XCPT_NM)
3777 | RT_BIT(X86_XCPT_TS)
3778 | RT_BIT(X86_XCPT_UD)
3779 | RT_BIT(X86_XCPT_NP)
3780 | RT_BIT(X86_XCPT_SS)
3781 | RT_BIT(X86_XCPT_GP)
3782 | RT_BIT(X86_XCPT_PF)
3783 | RT_BIT(X86_XCPT_MF)
3784 ;
3785#elif defined(HMVMX_ALWAYS_TRAP_PF)
3786 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
3787#endif
3788 Assert(pVM->hm.s.fNestedPaging || (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
3789
3790 /*
3791 * Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW).
3792 */
3793 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3794 uint32_t fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3795 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3796 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
3797 else
3798 Assert((fSetCr0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3799
3800 u32GuestCr0 |= fSetCr0;
3801 u32GuestCr0 &= fZapCr0;
3802 u32GuestCr0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3803
3804 /*
3805 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3806 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3807 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3808 */
3809 uint32_t u32Cr0Mask = X86_CR0_PE
3810 | X86_CR0_NE
3811 | (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
3812 | X86_CR0_PG
3813 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3814 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3815 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3816
3817 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3818 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3819 * and @bugref{6944}. */
3820#if 0
3821 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3822 u32Cr0Mask &= ~X86_CR0_PE;
3823#endif
3824 /*
3825 * Finally, update VMCS fields with the CR0 values and the exception bitmap.
3826 */
3827 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCr0);
3828 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32ShadowCr0);
3829 if (u32Cr0Mask != pVCpu->hm.s.vmx.u32Cr0Mask)
3830 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32Cr0Mask);
3831 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
3832 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3833 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3834 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3835 AssertRCReturn(rc, rc);
3836
3837 /* Update our caches. */
3838 pVCpu->hm.s.vmx.u32Cr0Mask = u32Cr0Mask;
3839 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
3840 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3841
3842 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3843
3844 Log4Func(("u32Cr0Mask=%#RX32 u32ShadowCr0=%#RX32 u32GuestCr0=%#RX32 (fSetCr0=%#RX32 fZapCr0=%#RX32\n", u32Cr0Mask,
3845 u32ShadowCr0, u32GuestCr0, fSetCr0, fZapCr0));
3846 }
3847
3848 return VINF_SUCCESS;
3849}
3850
3851
3852/**
3853 * Exports the guest control registers (CR3, CR4) into the guest-state area
3854 * in the VMCS.
3855 *
3856 * @returns VBox strict status code.
3857 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3858 * without unrestricted guest access and the VMMDev is not presently
3859 * mapped (e.g. EFI32).
3860 *
3861 * @param pVCpu The cross context virtual CPU structure.
3862 *
3863 * @remarks No-long-jump zone!!!
3864 */
3865static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu)
3866{
3867 int rc = VINF_SUCCESS;
3868 PVM pVM = pVCpu->CTX_SUFF(pVM);
3869
3870 /*
3871 * Guest CR2.
3872 * It's always loaded in the assembler code. Nothing to do here.
3873 */
3874
3875 /*
3876 * Guest CR3.
3877 */
3878 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
3879 {
3880 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
3881
3882 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3883 if (pVM->hm.s.fNestedPaging)
3884 {
3885 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3886
3887 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3888 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3889 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3890 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3891
3892 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3893 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3894 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3895
3896 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3897 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3898 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3899 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3900 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3901 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3902 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3903
3904 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3905 AssertRCReturn(rc, rc);
3906
3907 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3908 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3909 || CPUMIsGuestPagingEnabledEx(pCtx))
3910 {
3911 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3912 if (CPUMIsGuestInPAEModeEx(pCtx))
3913 {
3914 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3915 AssertRCReturn(rc, rc);
3916 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3917 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3918 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3919 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3920 AssertRCReturn(rc, rc);
3921 }
3922
3923 /*
3924 * The guest's view of its CR3 is unblemished with Nested Paging when the
3925 * guest is using paging or we have unrestricted guest execution to handle
3926 * the guest when it's not using paging.
3927 */
3928 GCPhysGuestCR3 = pCtx->cr3;
3929 }
3930 else
3931 {
3932 /*
3933 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
3934 * thinks it accesses physical memory directly, we use our identity-mapped
3935 * page table to map guest-linear to guest-physical addresses. EPT takes care
3936 * of translating it to host-physical addresses.
3937 */
3938 RTGCPHYS GCPhys;
3939 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3940
3941 /* We obtain it here every time as the guest could have relocated this PCI region. */
3942 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3943 if (RT_SUCCESS(rc))
3944 { /* likely */ }
3945 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3946 {
3947 Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3948 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3949 }
3950 else
3951 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3952
3953 GCPhysGuestCR3 = GCPhys;
3954 }
3955
3956 Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
3957 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3958 AssertRCReturn(rc, rc);
3959 }
3960 else
3961 {
3962 /* Non-nested paging case, just use the hypervisor's CR3. */
3963 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3964
3965 Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
3966 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3967 AssertRCReturn(rc, rc);
3968 }
3969
3970 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3971 }
3972
3973 /*
3974 * Guest CR4.
3975 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3976 */
3977 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
3978 {
3979 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3980 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
3981 Assert(!RT_HI_U32(pCtx->cr4));
3982
3983 uint32_t u32GuestCr4 = pCtx->cr4;
3984 uint32_t const u32ShadowCr4 = pCtx->cr4;
3985
3986 /*
3987 * Setup VT-x's view of the guest CR4.
3988 *
3989 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3990 * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3991 * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3992 *
3993 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3994 */
3995 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3996 {
3997 Assert(pVM->hm.s.vmx.pRealModeTSS);
3998 Assert(PDMVmmDevHeapIsEnabled(pVM));
3999 u32GuestCr4 &= ~X86_CR4_VME;
4000 }
4001
4002 if (pVM->hm.s.fNestedPaging)
4003 {
4004 if ( !CPUMIsGuestPagingEnabledEx(pCtx)
4005 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4006 {
4007 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4008 u32GuestCr4 |= X86_CR4_PSE;
4009 /* Our identity mapping is a 32-bit page directory. */
4010 u32GuestCr4 &= ~X86_CR4_PAE;
4011 }
4012 /* else use guest CR4.*/
4013 }
4014 else
4015 {
4016 /*
4017 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4018 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4019 */
4020 switch (pVCpu->hm.s.enmShadowMode)
4021 {
4022 case PGMMODE_REAL: /* Real-mode. */
4023 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4024 case PGMMODE_32_BIT: /* 32-bit paging. */
4025 {
4026 u32GuestCr4 &= ~X86_CR4_PAE;
4027 break;
4028 }
4029
4030 case PGMMODE_PAE: /* PAE paging. */
4031 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4032 {
4033 u32GuestCr4 |= X86_CR4_PAE;
4034 break;
4035 }
4036
4037 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4038 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4039#ifdef VBOX_ENABLE_64_BITS_GUESTS
4040 break;
4041#endif
4042 default:
4043 AssertFailed();
4044 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4045 }
4046 }
4047
4048 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4049 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4050 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4051 u32GuestCr4 |= fSetCr4;
4052 u32GuestCr4 &= fZapCr4;
4053
4054 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them,
4055 that would cause a VM-exit. */
4056 uint32_t u32Cr4Mask = X86_CR4_VME
4057 | X86_CR4_PAE
4058 | X86_CR4_PGE
4059 | X86_CR4_PSE
4060 | X86_CR4_VMXE;
4061 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4062 u32Cr4Mask |= X86_CR4_OSXSAVE;
4063 if (pVM->cpum.ro.GuestFeatures.fPcid)
4064 u32Cr4Mask |= X86_CR4_PCIDE;
4065
4066 /* Write VT-x's view of the guest CR4, the CR4 modify mask and the read-only CR4 shadow
4067 into the VMCS and update our cache. */
4068 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCr4);
4069 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32ShadowCr4);
4070 if (pVCpu->hm.s.vmx.u32Cr4Mask != u32Cr4Mask)
4071 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32Cr4Mask);
4072 AssertRCReturn(rc, rc);
4073 pVCpu->hm.s.vmx.u32Cr4Mask = u32Cr4Mask;
4074
4075 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4076 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
4077
4078 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
4079
4080 Log4Func(("u32GuestCr4=%#RX32 u32ShadowCr4=%#RX32 (fSetCr4=%#RX32 fZapCr4=%#RX32)\n", u32GuestCr4, u32ShadowCr4, fSetCr4,
4081 fZapCr4));
4082 }
4083 return rc;
4084}
4085
4086
4087/**
4088 * Exports the guest debug registers into the guest-state area in the VMCS.
4089 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4090 *
4091 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4092 *
4093 * @returns VBox status code.
4094 * @param pVCpu The cross context virtual CPU structure.
4095 *
4096 * @remarks No-long-jump zone!!!
4097 */
4098static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu)
4099{
4100 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4101
4102#ifdef VBOX_STRICT
4103 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4104 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
4105 {
4106 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4107 Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0);
4108 Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
4109 }
4110#endif
4111
4112 bool fSteppingDB = false;
4113 bool fInterceptMovDRx = false;
4114 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
4115 if (pVCpu->hm.s.fSingleInstruction)
4116 {
4117 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4118 PVM pVM = pVCpu->CTX_SUFF(pVM);
4119 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
4120 {
4121 uProcCtls |= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
4122 Assert(fSteppingDB == false);
4123 }
4124 else
4125 {
4126 pVCpu->cpum.GstCtx.eflags.u32 |= X86_EFL_TF;
4127 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_RFLAGS;
4128 pVCpu->hm.s.fClearTrapFlag = true;
4129 fSteppingDB = true;
4130 }
4131 }
4132
4133 uint32_t u32GuestDr7;
4134 if ( fSteppingDB
4135 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4136 {
4137 /*
4138 * Use the combined guest and host DRx values found in the hypervisor register set
4139 * because the debugger has breakpoints active or someone is single stepping on the
4140 * host side without a monitor trap flag.
4141 *
4142 * Note! DBGF expects a clean DR6 state before executing guest code.
4143 */
4144#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4145 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4146 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4147 {
4148 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4149 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4150 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4151 }
4152 else
4153#endif
4154 if (!CPUMIsHyperDebugStateActive(pVCpu))
4155 {
4156 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4157 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4158 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4159 }
4160
4161 /* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
4162 u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
4163 pVCpu->hm.s.fUsingHyperDR7 = true;
4164 fInterceptMovDRx = true;
4165 }
4166 else
4167 {
4168 /*
4169 * If the guest has enabled debug registers, we need to load them prior to
4170 * executing guest code so they'll trigger at the right time.
4171 */
4172 if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD))
4173 {
4174#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4175 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4176 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4177 {
4178 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4179 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4180 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4181 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4182 }
4183 else
4184#endif
4185 if (!CPUMIsGuestDebugStateActive(pVCpu))
4186 {
4187 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4188 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4189 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4190 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4191 }
4192 Assert(!fInterceptMovDRx);
4193 }
4194 /*
4195 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4196 * must intercept #DB in order to maintain a correct DR6 guest value, and
4197 * because we need to intercept it to prevent nested #DBs from hanging the
4198 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4199 */
4200#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4201 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4202 && !CPUMIsGuestDebugStateActive(pVCpu))
4203#else
4204 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4205#endif
4206 {
4207 fInterceptMovDRx = true;
4208 }
4209
4210 /* Update DR7 with the actual guest value. */
4211 u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
4212 pVCpu->hm.s.fUsingHyperDR7 = false;
4213 }
4214
4215 if (fInterceptMovDRx)
4216 uProcCtls |= VMX_PROC_CTLS_MOV_DR_EXIT;
4217 else
4218 uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
4219
4220 /*
4221 * Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
4222 * monitor-trap flag and update our cache.
4223 */
4224 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
4225 {
4226 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4227 AssertRCReturn(rc2, rc2);
4228 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
4229 }
4230
4231 /*
4232 * Update guest DR7.
4233 */
4234 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
4235 AssertRCReturn(rc, rc);
4236
4237 return VINF_SUCCESS;
4238}
4239
4240
4241#ifdef VBOX_STRICT
4242/**
4243 * Strict function to validate segment registers.
4244 *
4245 * @param pVCpu The cross context virtual CPU structure.
4246 *
4247 * @remarks Will import guest CR0 on strict builds during validation of
4248 * segments.
4249 */
4250static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu)
4251{
4252 /*
4253 * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
4254 *
4255 * The reason we check for attribute value 0 in this function and not just the unusable bit is
4256 * because hmR0VmxExportGuestSegmentReg() only updates the VMCS' copy of the value with the unusable bit
4257 * and doesn't change the guest-context value.
4258 */
4259 PVM pVM = pVCpu->CTX_SUFF(pVM);
4260 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4261 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
4262 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4263 && ( !CPUMIsGuestInRealModeEx(pCtx)
4264 && !CPUMIsGuestInV86ModeEx(pCtx)))
4265 {
4266 /* Protected mode checks */
4267 /* CS */
4268 Assert(pCtx->cs.Attr.n.u1Present);
4269 Assert(!(pCtx->cs.Attr.u & 0xf00));
4270 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4271 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4272 || !(pCtx->cs.Attr.n.u1Granularity));
4273 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4274 || (pCtx->cs.Attr.n.u1Granularity));
4275 /* CS cannot be loaded with NULL in protected mode. */
4276 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4277 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4278 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4279 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4280 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4281 else
4282 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4283 /* SS */
4284 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4285 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4286 if ( !(pCtx->cr0 & X86_CR0_PE)
4287 || pCtx->cs.Attr.n.u4Type == 3)
4288 {
4289 Assert(!pCtx->ss.Attr.n.u2Dpl);
4290 }
4291 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4292 {
4293 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4294 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4295 Assert(pCtx->ss.Attr.n.u1Present);
4296 Assert(!(pCtx->ss.Attr.u & 0xf00));
4297 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4298 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4299 || !(pCtx->ss.Attr.n.u1Granularity));
4300 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4301 || (pCtx->ss.Attr.n.u1Granularity));
4302 }
4303 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmentReg(). */
4304 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4305 {
4306 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4307 Assert(pCtx->ds.Attr.n.u1Present);
4308 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4309 Assert(!(pCtx->ds.Attr.u & 0xf00));
4310 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4311 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4312 || !(pCtx->ds.Attr.n.u1Granularity));
4313 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4314 || (pCtx->ds.Attr.n.u1Granularity));
4315 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4316 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4317 }
4318 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4319 {
4320 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4321 Assert(pCtx->es.Attr.n.u1Present);
4322 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4323 Assert(!(pCtx->es.Attr.u & 0xf00));
4324 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4325 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4326 || !(pCtx->es.Attr.n.u1Granularity));
4327 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4328 || (pCtx->es.Attr.n.u1Granularity));
4329 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4330 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4331 }
4332 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4333 {
4334 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4335 Assert(pCtx->fs.Attr.n.u1Present);
4336 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4337 Assert(!(pCtx->fs.Attr.u & 0xf00));
4338 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4339 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4340 || !(pCtx->fs.Attr.n.u1Granularity));
4341 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4342 || (pCtx->fs.Attr.n.u1Granularity));
4343 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4344 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4345 }
4346 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4347 {
4348 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4349 Assert(pCtx->gs.Attr.n.u1Present);
4350 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4351 Assert(!(pCtx->gs.Attr.u & 0xf00));
4352 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4353 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4354 || !(pCtx->gs.Attr.n.u1Granularity));
4355 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4356 || (pCtx->gs.Attr.n.u1Granularity));
4357 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4358 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4359 }
4360 /* 64-bit capable CPUs. */
4361# if HC_ARCH_BITS == 64
4362 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4363 Assert(!pCtx->ss.Attr.u || !RT_HI_U32(pCtx->ss.u64Base));
4364 Assert(!pCtx->ds.Attr.u || !RT_HI_U32(pCtx->ds.u64Base));
4365 Assert(!pCtx->es.Attr.u || !RT_HI_U32(pCtx->es.u64Base));
4366# endif
4367 }
4368 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4369 || ( CPUMIsGuestInRealModeEx(pCtx)
4370 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4371 {
4372 /* Real and v86 mode checks. */
4373 /* hmR0VmxExportGuestSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4374 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4375 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4376 {
4377 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4378 }
4379 else
4380 {
4381 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4382 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4383 }
4384
4385 /* CS */
4386 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4387 Assert(pCtx->cs.u32Limit == 0xffff);
4388 Assert(u32CSAttr == 0xf3);
4389 /* SS */
4390 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4391 Assert(pCtx->ss.u32Limit == 0xffff);
4392 Assert(u32SSAttr == 0xf3);
4393 /* DS */
4394 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4395 Assert(pCtx->ds.u32Limit == 0xffff);
4396 Assert(u32DSAttr == 0xf3);
4397 /* ES */
4398 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4399 Assert(pCtx->es.u32Limit == 0xffff);
4400 Assert(u32ESAttr == 0xf3);
4401 /* FS */
4402 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4403 Assert(pCtx->fs.u32Limit == 0xffff);
4404 Assert(u32FSAttr == 0xf3);
4405 /* GS */
4406 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4407 Assert(pCtx->gs.u32Limit == 0xffff);
4408 Assert(u32GSAttr == 0xf3);
4409 /* 64-bit capable CPUs. */
4410# if HC_ARCH_BITS == 64
4411 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4412 Assert(!u32SSAttr || !RT_HI_U32(pCtx->ss.u64Base));
4413 Assert(!u32DSAttr || !RT_HI_U32(pCtx->ds.u64Base));
4414 Assert(!u32ESAttr || !RT_HI_U32(pCtx->es.u64Base));
4415# endif
4416 }
4417}
4418#endif /* VBOX_STRICT */
4419
4420
4421/**
4422 * Exports a guest segment register into the guest-state area in the VMCS.
4423 *
4424 * @returns VBox status code.
4425 * @param pVCpu The cross context virtual CPU structure.
4426 * @param idxSel Index of the selector in the VMCS.
4427 * @param idxLimit Index of the segment limit in the VMCS.
4428 * @param idxBase Index of the segment base in the VMCS.
4429 * @param idxAccess Index of the access rights of the segment in the VMCS.
4430 * @param pSelReg Pointer to the segment selector.
4431 *
4432 * @remarks No-long-jump zone!!!
4433 */
4434static int hmR0VmxExportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
4435 PCCPUMSELREG pSelReg)
4436{
4437 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4438 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4439 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4440 AssertRCReturn(rc, rc);
4441
4442 uint32_t u32Access = pSelReg->Attr.u;
4443 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4444 {
4445 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4446 u32Access = 0xf3;
4447 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4448 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4449 }
4450 else
4451 {
4452 /*
4453 * The way to differentiate between whether this is really a null selector or was just
4454 * a selector loaded with 0 in real-mode is using the segment attributes. A selector
4455 * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4456 * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4457 * NULL selectors loaded in protected-mode have their attribute as 0.
4458 */
4459 if (!u32Access)
4460 u32Access = X86DESCATTR_UNUSABLE;
4461 }
4462
4463 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4464 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4465 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4466
4467 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4468 AssertRCReturn(rc, rc);
4469 return rc;
4470}
4471
4472
4473/**
4474 * Exports the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4475 * into the guest-state area in the VMCS.
4476 *
4477 * @returns VBox status code.
4478 * @param pVCpu The cross context virtual CPU structure.
4479 *
4480 * @remarks Will import guest CR0 on strict builds during validation of
4481 * segments.
4482 * @remarks No-long-jump zone!!!
4483 */
4484static int hmR0VmxExportGuestSegmentRegs(PVMCPU pVCpu)
4485{
4486 int rc = VERR_INTERNAL_ERROR_5;
4487 PVM pVM = pVCpu->CTX_SUFF(pVM);
4488 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4489
4490 /*
4491 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4492 */
4493 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4494 {
4495#ifdef VBOX_WITH_REM
4496 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4497 {
4498 Assert(pVM->hm.s.vmx.pRealModeTSS);
4499 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4500 if ( pVCpu->hm.s.vmx.fWasInRealMode
4501 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4502 {
4503 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4504 in real-mode (e.g. OpenBSD 4.0) */
4505 REMFlushTBs(pVM);
4506 Log4Func(("Switch to protected mode detected!\n"));
4507 pVCpu->hm.s.vmx.fWasInRealMode = false;
4508 }
4509 }
4510#endif
4511 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
4512 {
4513 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
4514 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4515 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pCtx->cs.Attr.u;
4516 rc = HMVMX_EXPORT_SREG(CS, &pCtx->cs);
4517 AssertRCReturn(rc, rc);
4518 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
4519 }
4520
4521 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
4522 {
4523 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
4524 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4525 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pCtx->ss.Attr.u;
4526 rc = HMVMX_EXPORT_SREG(SS, &pCtx->ss);
4527 AssertRCReturn(rc, rc);
4528 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
4529 }
4530
4531 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
4532 {
4533 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
4534 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4535 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pCtx->ds.Attr.u;
4536 rc = HMVMX_EXPORT_SREG(DS, &pCtx->ds);
4537 AssertRCReturn(rc, rc);
4538 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
4539 }
4540
4541 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
4542 {
4543 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
4544 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4545 pVCpu->hm.s.vmx.RealMode.AttrES.u = pCtx->es.Attr.u;
4546 rc = HMVMX_EXPORT_SREG(ES, &pCtx->es);
4547 AssertRCReturn(rc, rc);
4548 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
4549 }
4550
4551 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
4552 {
4553 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
4554 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4555 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pCtx->fs.Attr.u;
4556 rc = HMVMX_EXPORT_SREG(FS, &pCtx->fs);
4557 AssertRCReturn(rc, rc);
4558 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
4559 }
4560
4561 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
4562 {
4563 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
4564 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4565 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pCtx->gs.Attr.u;
4566 rc = HMVMX_EXPORT_SREG(GS, &pCtx->gs);
4567 AssertRCReturn(rc, rc);
4568 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
4569 }
4570
4571#ifdef VBOX_STRICT
4572 hmR0VmxValidateSegmentRegs(pVCpu);
4573#endif
4574
4575 Log4Func(("CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pCtx->cs.Sel, pCtx->cs.u64Base,
4576 pCtx->cs.u32Limit, pCtx->cs.Attr.u));
4577 }
4578
4579 /*
4580 * Guest TR.
4581 */
4582 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
4583 {
4584 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
4585
4586 /*
4587 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
4588 * achieved using the interrupt redirection bitmap (all bits cleared to let the guest
4589 * handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4590 */
4591 uint16_t u16Sel = 0;
4592 uint32_t u32Limit = 0;
4593 uint64_t u64Base = 0;
4594 uint32_t u32AccessRights = 0;
4595
4596 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4597 {
4598 u16Sel = pCtx->tr.Sel;
4599 u32Limit = pCtx->tr.u32Limit;
4600 u64Base = pCtx->tr.u64Base;
4601 u32AccessRights = pCtx->tr.Attr.u;
4602 }
4603 else
4604 {
4605 Assert(pVM->hm.s.vmx.pRealModeTSS);
4606 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
4607
4608 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4609 RTGCPHYS GCPhys;
4610 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4611 AssertRCReturn(rc, rc);
4612
4613 X86DESCATTR DescAttr;
4614 DescAttr.u = 0;
4615 DescAttr.n.u1Present = 1;
4616 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4617
4618 u16Sel = 0;
4619 u32Limit = HM_VTX_TSS_SIZE;
4620 u64Base = GCPhys; /* in real-mode phys = virt. */
4621 u32AccessRights = DescAttr.u;
4622 }
4623
4624 /* Validate. */
4625 Assert(!(u16Sel & RT_BIT(2)));
4626 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4627 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4628 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4629 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4630 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4631 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4632 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4633 Assert( (u32Limit & 0xfff) == 0xfff
4634 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4635 Assert( !(pCtx->tr.u32Limit & 0xfff00000)
4636 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4637
4638 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4639 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4640 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4641 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4642 AssertRCReturn(rc, rc);
4643
4644 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
4645 Log4Func(("TR base=%#RX64\n", pCtx->tr.u64Base));
4646 }
4647
4648 /*
4649 * Guest GDTR.
4650 */
4651 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4652 {
4653 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
4654
4655 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
4656 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
4657 AssertRCReturn(rc, rc);
4658
4659 /* Validate. */
4660 Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4661
4662 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4663 Log4Func(("GDTR base=%#RX64\n", pCtx->gdtr.pGdt));
4664 }
4665
4666 /*
4667 * Guest LDTR.
4668 */
4669 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4670 {
4671 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
4672
4673 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4674 uint32_t u32Access = 0;
4675 if (!pCtx->ldtr.Attr.u)
4676 u32Access = X86DESCATTR_UNUSABLE;
4677 else
4678 u32Access = pCtx->ldtr.Attr.u;
4679
4680 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
4681 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
4682 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4683 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
4684 AssertRCReturn(rc, rc);
4685
4686 /* Validate. */
4687 if (!(u32Access & X86DESCATTR_UNUSABLE))
4688 {
4689 Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4690 Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4691 Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4692 Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4693 Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4694 Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4695 Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
4696 || !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4697 Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
4698 || pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4699 }
4700
4701 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4702 Log4Func(("LDTR base=%#RX64\n", pCtx->ldtr.u64Base));
4703 }
4704
4705 /*
4706 * Guest IDTR.
4707 */
4708 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4709 {
4710 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
4711
4712 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
4713 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
4714 AssertRCReturn(rc, rc);
4715
4716 /* Validate. */
4717 Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4718
4719 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4720 Log4Func(("IDTR base=%#RX64\n", pCtx->idtr.pIdt));
4721 }
4722
4723 return VINF_SUCCESS;
4724}
4725
4726
4727/**
4728 * Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4729 * areas.
4730 *
4731 * These MSRs will automatically be loaded to the host CPU on every successful
4732 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4733 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4734 * -not- updated here for performance reasons. See hmR0VmxExportHostMsrs().
4735 *
4736 * Also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4737 *
4738 * @returns VBox status code.
4739 * @param pVCpu The cross context virtual CPU structure.
4740 *
4741 * @remarks No-long-jump zone!!!
4742 */
4743static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu)
4744{
4745 AssertPtr(pVCpu);
4746 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4747
4748 /*
4749 * MSRs that we use the auto-load/store MSR area in the VMCS.
4750 * For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
4751 */
4752 PVM pVM = pVCpu->CTX_SUFF(pVM);
4753 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4754 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4755 {
4756 if (pVM->hm.s.fAllow64BitGuests)
4757 {
4758#if HC_ARCH_BITS == 32
4759 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS | CPUMCTX_EXTRN_KERNEL_GS_BASE);
4760
4761 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pCtx->msrLSTAR, false, NULL);
4762 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pCtx->msrSTAR, false, NULL);
4763 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pCtx->msrSFMASK, false, NULL);
4764 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, false, NULL);
4765 AssertRCReturn(rc, rc);
4766# ifdef LOG_ENABLED
4767 PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4768 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4769 Log4Func(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4770# endif
4771#endif
4772 }
4773 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4774 }
4775
4776 /*
4777 * Guest Sysenter MSRs.
4778 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4779 * VM-exits on WRMSRs for these MSRs.
4780 */
4781 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4782 {
4783 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
4784
4785 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4786 {
4787 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
4788 AssertRCReturn(rc, rc);
4789 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4790 }
4791
4792 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4793 {
4794 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
4795 AssertRCReturn(rc, rc);
4796 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4797 }
4798
4799 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4800 {
4801 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
4802 AssertRCReturn(rc, rc);
4803 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4804 }
4805 }
4806
4807 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4808 {
4809 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
4810
4811 if (hmR0VmxShouldSwapEferMsr(pVCpu))
4812 {
4813 /*
4814 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4815 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4816 */
4817 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4818 {
4819 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
4820 AssertRCReturn(rc,rc);
4821 Log4Func(("EFER=%#RX64\n", pCtx->msrEFER));
4822 }
4823 else
4824 {
4825 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pCtx->msrEFER, false /* fUpdateHostMsr */,
4826 NULL /* pfAddedAndUpdated */);
4827 AssertRCReturn(rc, rc);
4828
4829 /* We need to intercept reads too, see @bugref{7386#c16}. */
4830 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
4831 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4832 Log4Func(("MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pCtx->msrEFER,
4833 pVCpu->hm.s.vmx.cMsrs));
4834 }
4835 }
4836 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4837 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4838 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4839 }
4840
4841 return VINF_SUCCESS;
4842}
4843
4844
4845#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4846/**
4847 * Check if guest state allows safe use of 32-bit switcher again.
4848 *
4849 * Segment bases and protected mode structures must be 32-bit addressable
4850 * because the 32-bit switcher will ignore high dword when writing these VMCS
4851 * fields. See @bugref{8432} for details.
4852 *
4853 * @returns true if safe, false if must continue to use the 64-bit switcher.
4854 * @param pCtx Pointer to the guest-CPU context.
4855 *
4856 * @remarks No-long-jump zone!!!
4857 */
4858static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
4859{
4860 if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4861 if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4862 if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4863 if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4864 if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4865 if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4866 if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4867 if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4868 if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4869 if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4870
4871 /* All good, bases are 32-bit. */
4872 return true;
4873}
4874#endif
4875
4876
4877/**
4878 * Selects up the appropriate function to run guest code.
4879 *
4880 * @returns VBox status code.
4881 * @param pVCpu The cross context virtual CPU structure.
4882 *
4883 * @remarks No-long-jump zone!!!
4884 */
4885static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu)
4886{
4887 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4888 if (CPUMIsGuestInLongModeEx(pCtx))
4889 {
4890#ifndef VBOX_ENABLE_64_BITS_GUESTS
4891 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4892#endif
4893 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4894#if HC_ARCH_BITS == 32
4895 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4896 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4897 {
4898#ifdef VBOX_STRICT
4899 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4900 {
4901 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4902 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4903 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4904 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4905 | HM_CHANGED_VMX_ENTRY_CTLS
4906 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4907 }
4908#endif
4909 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4910
4911 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4912 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4913 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4914 Log4Func(("Selected 64-bit switcher\n"));
4915 }
4916#else
4917 /* 64-bit host. */
4918 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4919#endif
4920 }
4921 else
4922 {
4923 /* Guest is not in long mode, use the 32-bit handler. */
4924#if HC_ARCH_BITS == 32
4925 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4926 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4927 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4928 {
4929# ifdef VBOX_STRICT
4930 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4931 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4932 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4933 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4934 | HM_CHANGED_VMX_ENTRY_CTLS
4935 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4936# endif
4937 }
4938# ifdef VBOX_ENABLE_64_BITS_GUESTS
4939 /*
4940 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
4941 * design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
4942 * switcher flag because now we know the guest is in a sane state where it's safe
4943 * to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4944 * the much faster 32-bit switcher again.
4945 */
4946 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4947 {
4948 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4949 Log4Func(("Selected 32-bit switcher\n"));
4950 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4951 }
4952 else
4953 {
4954 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4955 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4956 || hmR0VmxIs32BitSwitcherSafe(pCtx))
4957 {
4958 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4959 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4960 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
4961 | HM_CHANGED_VMX_ENTRY_CTLS
4962 | HM_CHANGED_VMX_EXIT_CTLS
4963 | HM_CHANGED_HOST_CONTEXT);
4964 Log4Func(("Selected 32-bit switcher (safe)\n"));
4965 }
4966 }
4967# else
4968 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4969# endif
4970#else
4971 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4972#endif
4973 }
4974 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4975 return VINF_SUCCESS;
4976}
4977
4978
4979/**
4980 * Wrapper for running the guest code in VT-x.
4981 *
4982 * @returns VBox status code, no informational status codes.
4983 * @param pVCpu The cross context virtual CPU structure.
4984 *
4985 * @remarks No-long-jump zone!!!
4986 */
4987DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu)
4988{
4989 /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4990 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4991 pCtx->fExtrn |= HMVMX_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
4992
4993 /*
4994 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4995 * floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
4996 * callee-saved and thus the need for this XMM wrapper.
4997 *
4998 * See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
4999 */
5000 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
5001 /** @todo Add stats for resume vs launch. */
5002 PVM pVM = pVCpu->CTX_SUFF(pVM);
5003#ifdef VBOX_WITH_KERNEL_USING_XMM
5004 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
5005#else
5006 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
5007#endif
5008 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
5009 return rc;
5010}
5011
5012
5013/**
5014 * Reports world-switch error and dumps some useful debug info.
5015 *
5016 * @param pVCpu The cross context virtual CPU structure.
5017 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5018 * @param pVmxTransient Pointer to the VMX transient structure (only
5019 * exitReason updated).
5020 */
5021static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
5022{
5023 Assert(pVCpu);
5024 Assert(pVmxTransient);
5025 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
5026
5027 Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
5028 switch (rcVMRun)
5029 {
5030 case VERR_VMX_INVALID_VMXON_PTR:
5031 AssertFailed();
5032 break;
5033 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5034 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5035 {
5036 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5037 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5038 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
5039 AssertRC(rc);
5040
5041 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5042 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5043 Cannot do it here as we may have been long preempted. */
5044
5045#ifdef VBOX_STRICT
5046 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5047 pVmxTransient->uExitReason));
5048 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
5049 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5050 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5051 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5052 else
5053 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5054 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5055 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5056
5057 /* VMX control bits. */
5058 uint32_t u32Val;
5059 uint64_t u64Val;
5060 RTHCUINTREG uHCReg;
5061 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5062 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5063 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5064 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5065 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
5066 {
5067 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5068 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5069 }
5070 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5071 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5072 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5073 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5074 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5075 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5076 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5077 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5078 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5079 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5080 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5081 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5082 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5083 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5084 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5085 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5086 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5087 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5088 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5089 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5090 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5091 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5092 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5093 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5094 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5095 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5096 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5097 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5098 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5099 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5100 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5101 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5102 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5103 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5104 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5105 {
5106 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5107 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5108 }
5109
5110 /* Guest bits. */
5111 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5112 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
5113 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5114 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
5115 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5116 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
5117 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
5118 {
5119 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5120 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5121 }
5122
5123 /* Host bits. */
5124 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5125 Log4(("Host CR0 %#RHr\n", uHCReg));
5126 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5127 Log4(("Host CR3 %#RHr\n", uHCReg));
5128 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5129 Log4(("Host CR4 %#RHr\n", uHCReg));
5130
5131 RTGDTR HostGdtr;
5132 PCX86DESCHC pDesc;
5133 ASMGetGDTR(&HostGdtr);
5134 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5135 Log4(("Host CS %#08x\n", u32Val));
5136 if (u32Val < HostGdtr.cbGdt)
5137 {
5138 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5139 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5140 }
5141
5142 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5143 Log4(("Host DS %#08x\n", u32Val));
5144 if (u32Val < HostGdtr.cbGdt)
5145 {
5146 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5147 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5148 }
5149
5150 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5151 Log4(("Host ES %#08x\n", u32Val));
5152 if (u32Val < HostGdtr.cbGdt)
5153 {
5154 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5155 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5156 }
5157
5158 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5159 Log4(("Host FS %#08x\n", u32Val));
5160 if (u32Val < HostGdtr.cbGdt)
5161 {
5162 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5163 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5164 }
5165
5166 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5167 Log4(("Host GS %#08x\n", u32Val));
5168 if (u32Val < HostGdtr.cbGdt)
5169 {
5170 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5171 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5172 }
5173
5174 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5175 Log4(("Host SS %#08x\n", u32Val));
5176 if (u32Val < HostGdtr.cbGdt)
5177 {
5178 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5179 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5180 }
5181
5182 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5183 Log4(("Host TR %#08x\n", u32Val));
5184 if (u32Val < HostGdtr.cbGdt)
5185 {
5186 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5187 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5188 }
5189
5190 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5191 Log4(("Host TR Base %#RHv\n", uHCReg));
5192 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5193 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5194 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5195 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5196 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5197 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5198 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5199 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5200 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5201 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5202 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5203 Log4(("Host RSP %#RHv\n", uHCReg));
5204 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5205 Log4(("Host RIP %#RHv\n", uHCReg));
5206# if HC_ARCH_BITS == 64
5207 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5208 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5209 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5210 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5211 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5212 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5213# endif
5214#endif /* VBOX_STRICT */
5215 break;
5216 }
5217
5218 default:
5219 /* Impossible */
5220 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5221 break;
5222 }
5223}
5224
5225
5226#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5227#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5228# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5229#endif
5230#ifdef VBOX_STRICT
5231static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5232{
5233 switch (idxField)
5234 {
5235 case VMX_VMCS_GUEST_RIP:
5236 case VMX_VMCS_GUEST_RSP:
5237 case VMX_VMCS_GUEST_SYSENTER_EIP:
5238 case VMX_VMCS_GUEST_SYSENTER_ESP:
5239 case VMX_VMCS_GUEST_GDTR_BASE:
5240 case VMX_VMCS_GUEST_IDTR_BASE:
5241 case VMX_VMCS_GUEST_CS_BASE:
5242 case VMX_VMCS_GUEST_DS_BASE:
5243 case VMX_VMCS_GUEST_ES_BASE:
5244 case VMX_VMCS_GUEST_FS_BASE:
5245 case VMX_VMCS_GUEST_GS_BASE:
5246 case VMX_VMCS_GUEST_SS_BASE:
5247 case VMX_VMCS_GUEST_LDTR_BASE:
5248 case VMX_VMCS_GUEST_TR_BASE:
5249 case VMX_VMCS_GUEST_CR3:
5250 return true;
5251 }
5252 return false;
5253}
5254
5255static bool hmR0VmxIsValidReadField(uint32_t idxField)
5256{
5257 switch (idxField)
5258 {
5259 /* Read-only fields. */
5260 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5261 return true;
5262 }
5263 /* Remaining readable fields should also be writable. */
5264 return hmR0VmxIsValidWriteField(idxField);
5265}
5266#endif /* VBOX_STRICT */
5267
5268
5269/**
5270 * Executes the specified handler in 64-bit mode.
5271 *
5272 * @returns VBox status code (no informational status codes).
5273 * @param pVCpu The cross context virtual CPU structure.
5274 * @param enmOp The operation to perform.
5275 * @param cParams Number of parameters.
5276 * @param paParam Array of 32-bit parameters.
5277 */
5278VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
5279{
5280 PVM pVM = pVCpu->CTX_SUFF(pVM);
5281 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5282 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5283 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5284 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5285
5286#ifdef VBOX_STRICT
5287 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5288 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5289
5290 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5291 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5292#endif
5293
5294 /* Disable interrupts. */
5295 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5296
5297#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5298 RTCPUID idHostCpu = RTMpCpuId();
5299 CPUMR0SetLApic(pVCpu, idHostCpu);
5300#endif
5301
5302 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5303 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5304
5305 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5306 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5307 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5308
5309 /* Leave VMX Root Mode. */
5310 VMXDisable();
5311
5312 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5313
5314 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5315 CPUMSetHyperEIP(pVCpu, enmOp);
5316 for (int i = (int)cParams - 1; i >= 0; i--)
5317 CPUMPushHyper(pVCpu, paParam[i]);
5318
5319 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5320
5321 /* Call the switcher. */
5322 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
5323 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5324
5325 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5326 /* Make sure the VMX instructions don't cause #UD faults. */
5327 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5328
5329 /* Re-enter VMX Root Mode */
5330 int rc2 = VMXEnable(HCPhysCpuPage);
5331 if (RT_FAILURE(rc2))
5332 {
5333 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5334 ASMSetFlags(fOldEFlags);
5335 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5336 return rc2;
5337 }
5338
5339 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5340 AssertRC(rc2);
5341 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5342 Assert(!(ASMGetFlags() & X86_EFL_IF));
5343 ASMSetFlags(fOldEFlags);
5344 return rc;
5345}
5346
5347
5348/**
5349 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5350 * supporting 64-bit guests.
5351 *
5352 * @returns VBox status code.
5353 * @param fResume Whether to VMLAUNCH or VMRESUME.
5354 * @param pCtx Pointer to the guest-CPU context.
5355 * @param pCache Pointer to the VMCS cache.
5356 * @param pVM The cross context VM structure.
5357 * @param pVCpu The cross context virtual CPU structure.
5358 */
5359DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5360{
5361 NOREF(fResume);
5362
5363 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5364 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5365
5366#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5367 pCache->uPos = 1;
5368 pCache->interPD = PGMGetInterPaeCR3(pVM);
5369 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5370#endif
5371
5372#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5373 pCache->TestIn.HCPhysCpuPage = 0;
5374 pCache->TestIn.HCPhysVmcs = 0;
5375 pCache->TestIn.pCache = 0;
5376 pCache->TestOut.HCPhysVmcs = 0;
5377 pCache->TestOut.pCache = 0;
5378 pCache->TestOut.pCtx = 0;
5379 pCache->TestOut.eflags = 0;
5380#else
5381 NOREF(pCache);
5382#endif
5383
5384 uint32_t aParam[10];
5385 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5386 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5387 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5388 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5389 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5390 aParam[5] = 0;
5391 aParam[6] = VM_RC_ADDR(pVM, pVM);
5392 aParam[7] = 0;
5393 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5394 aParam[9] = 0;
5395
5396#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5397 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5398 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5399#endif
5400 int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5401
5402#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5403 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5404 Assert(pCtx->dr[4] == 10);
5405 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5406#endif
5407
5408#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5409 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5410 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5411 pVCpu->hm.s.vmx.HCPhysVmcs));
5412 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5413 pCache->TestOut.HCPhysVmcs));
5414 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5415 pCache->TestOut.pCache));
5416 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5417 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5418 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5419 pCache->TestOut.pCtx));
5420 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5421#endif
5422 NOREF(pCtx);
5423 return rc;
5424}
5425
5426
5427/**
5428 * Initialize the VMCS-Read cache.
5429 *
5430 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5431 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5432 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5433 * (those that have a 32-bit FULL & HIGH part).
5434 *
5435 * @returns VBox status code.
5436 * @param pVCpu The cross context virtual CPU structure.
5437 */
5438static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
5439{
5440#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5441 do { \
5442 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5443 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5444 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5445 ++cReadFields; \
5446 } while (0)
5447
5448 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5449 uint32_t cReadFields = 0;
5450
5451 /*
5452 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5453 * and serve to indicate exceptions to the rules.
5454 */
5455
5456 /* Guest-natural selector base fields. */
5457#if 0
5458 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5459 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5460 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5461#endif
5462 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5463 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5464 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5465 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5466 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5467 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5468 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5469 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5470 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5471 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5472 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5473 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5474#if 0
5475 /* Unused natural width guest-state fields. */
5476 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5477 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5478#endif
5479 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5480 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5481
5482 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
5483 these 64-bit fields (using "FULL" and "HIGH" fields). */
5484#if 0
5485 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5486 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5487 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5488 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5489 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5490 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5491 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5492 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5493 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5494#endif
5495
5496 /* Natural width guest-state fields. */
5497 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5498#if 0
5499 /* Currently unused field. */
5500 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5501#endif
5502
5503 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5504 {
5505 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5506 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5507 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5508 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5509 }
5510 else
5511 {
5512 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5513 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5514 }
5515
5516#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5517 return VINF_SUCCESS;
5518}
5519
5520
5521/**
5522 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5523 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5524 * darwin, running 64-bit guests).
5525 *
5526 * @returns VBox status code.
5527 * @param pVCpu The cross context virtual CPU structure.
5528 * @param idxField The VMCS field encoding.
5529 * @param u64Val 16, 32 or 64-bit value.
5530 */
5531VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5532{
5533 int rc;
5534 switch (idxField)
5535 {
5536 /*
5537 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5538 */
5539 /* 64-bit Control fields. */
5540 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5541 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5542 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5543 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5544 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5545 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5546 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5547 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5548 case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
5549 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5550 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5551 case VMX_VMCS64_CTRL_EPTP_FULL:
5552 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5553 /* 64-bit Guest-state fields. */
5554 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5555 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5556 case VMX_VMCS64_GUEST_PAT_FULL:
5557 case VMX_VMCS64_GUEST_EFER_FULL:
5558 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5559 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5560 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5561 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5562 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5563 /* 64-bit Host-state fields. */
5564 case VMX_VMCS64_HOST_PAT_FULL:
5565 case VMX_VMCS64_HOST_EFER_FULL:
5566 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5567 {
5568 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5569 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5570 break;
5571 }
5572
5573 /*
5574 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5575 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5576 */
5577 /* Natural-width Guest-state fields. */
5578 case VMX_VMCS_GUEST_CR3:
5579 case VMX_VMCS_GUEST_ES_BASE:
5580 case VMX_VMCS_GUEST_CS_BASE:
5581 case VMX_VMCS_GUEST_SS_BASE:
5582 case VMX_VMCS_GUEST_DS_BASE:
5583 case VMX_VMCS_GUEST_FS_BASE:
5584 case VMX_VMCS_GUEST_GS_BASE:
5585 case VMX_VMCS_GUEST_LDTR_BASE:
5586 case VMX_VMCS_GUEST_TR_BASE:
5587 case VMX_VMCS_GUEST_GDTR_BASE:
5588 case VMX_VMCS_GUEST_IDTR_BASE:
5589 case VMX_VMCS_GUEST_RSP:
5590 case VMX_VMCS_GUEST_RIP:
5591 case VMX_VMCS_GUEST_SYSENTER_ESP:
5592 case VMX_VMCS_GUEST_SYSENTER_EIP:
5593 {
5594 if (!(RT_HI_U32(u64Val)))
5595 {
5596 /* If this field is 64-bit, VT-x will zero out the top bits. */
5597 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5598 }
5599 else
5600 {
5601 /* Assert that only the 32->64 switcher case should ever come here. */
5602 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5603 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5604 }
5605 break;
5606 }
5607
5608 default:
5609 {
5610 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5611 rc = VERR_INVALID_PARAMETER;
5612 break;
5613 }
5614 }
5615 AssertRCReturn(rc, rc);
5616 return rc;
5617}
5618
5619
5620/**
5621 * Queue up a VMWRITE by using the VMCS write cache.
5622 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5623 *
5624 * @param pVCpu The cross context virtual CPU structure.
5625 * @param idxField The VMCS field encoding.
5626 * @param u64Val 16, 32 or 64-bit value.
5627 */
5628VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5629{
5630 AssertPtr(pVCpu);
5631 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5632
5633 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5634 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5635
5636 /* Make sure there are no duplicates. */
5637 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5638 {
5639 if (pCache->Write.aField[i] == idxField)
5640 {
5641 pCache->Write.aFieldVal[i] = u64Val;
5642 return VINF_SUCCESS;
5643 }
5644 }
5645
5646 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5647 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5648 pCache->Write.cValidEntries++;
5649 return VINF_SUCCESS;
5650}
5651#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5652
5653
5654/**
5655 * Sets up the usage of TSC-offsetting and updates the VMCS.
5656 *
5657 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5658 * VMX preemption timer.
5659 *
5660 * @returns VBox status code.
5661 * @param pVCpu The cross context virtual CPU structure.
5662 *
5663 * @remarks No-long-jump zone!!!
5664 */
5665static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5666{
5667 bool fOffsettedTsc;
5668 bool fParavirtTsc;
5669 PVM pVM = pVCpu->CTX_SUFF(pVM);
5670 uint64_t uTscOffset;
5671 if (pVM->hm.s.vmx.fUsePreemptTimer)
5672 {
5673 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
5674
5675 /* Make sure the returned values have sane upper and lower boundaries. */
5676 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5677 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5678 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5679 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5680
5681 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5682 int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
5683 AssertRC(rc);
5684 }
5685 else
5686 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
5687
5688 if (fParavirtTsc)
5689 {
5690 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5691 information before every VM-entry, hence disable it for performance sake. */
5692#if 0
5693 int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5694 AssertRC(rc);
5695#endif
5696 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5697 }
5698
5699 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
5700 if ( fOffsettedTsc
5701 && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5702 {
5703 if (pVCpu->hm.s.vmx.u64TscOffset != uTscOffset)
5704 {
5705 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
5706 AssertRC(rc);
5707 pVCpu->hm.s.vmx.u64TscOffset = uTscOffset;
5708 }
5709
5710 if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
5711 {
5712 uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
5713 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5714 AssertRC(rc);
5715 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5716 }
5717 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5718 }
5719 else
5720 {
5721 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5722 if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
5723 {
5724 uProcCtls |= VMX_PROC_CTLS_RDTSC_EXIT;
5725 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5726 AssertRC(rc);
5727 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5728 }
5729 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5730 }
5731}
5732
5733
5734/**
5735 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5736 * VM-exit interruption info type.
5737 *
5738 * @returns The IEM exception flags.
5739 * @param uVector The event vector.
5740 * @param uVmxVectorType The VMX event type.
5741 *
5742 * @remarks This function currently only constructs flags required for
5743 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5744 * and CR2 aspects of an exception are not included).
5745 */
5746static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5747{
5748 uint32_t fIemXcptFlags;
5749 switch (uVmxVectorType)
5750 {
5751 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5752 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5753 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5754 break;
5755
5756 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5757 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5758 break;
5759
5760 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5761 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5762 break;
5763
5764 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5765 {
5766 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5767 if (uVector == X86_XCPT_BP)
5768 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5769 else if (uVector == X86_XCPT_OF)
5770 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5771 else
5772 {
5773 fIemXcptFlags = 0;
5774 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5775 }
5776 break;
5777 }
5778
5779 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5780 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5781 break;
5782
5783 default:
5784 fIemXcptFlags = 0;
5785 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5786 break;
5787 }
5788 return fIemXcptFlags;
5789}
5790
5791
5792/**
5793 * Sets an event as a pending event to be injected into the guest.
5794 *
5795 * @param pVCpu The cross context virtual CPU structure.
5796 * @param u32IntInfo The VM-entry interruption-information field.
5797 * @param cbInstr The VM-entry instruction length in bytes (for software
5798 * interrupts, exceptions and privileged software
5799 * exceptions).
5800 * @param u32ErrCode The VM-entry exception error code.
5801 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5802 * page-fault.
5803 *
5804 * @remarks Statistics counter assumes this is a guest event being injected or
5805 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5806 * always incremented.
5807 */
5808DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5809 RTGCUINTPTR GCPtrFaultAddress)
5810{
5811 Assert(!pVCpu->hm.s.Event.fPending);
5812 pVCpu->hm.s.Event.fPending = true;
5813 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5814 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5815 pVCpu->hm.s.Event.cbInstr = cbInstr;
5816 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5817}
5818
5819
5820/**
5821 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5822 *
5823 * @param pVCpu The cross context virtual CPU structure.
5824 */
5825DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
5826{
5827 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
5828 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5829 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
5830 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5831 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5832}
5833
5834
5835/**
5836 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
5837 *
5838 * @param pVCpu The cross context virtual CPU structure.
5839 */
5840DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
5841{
5842 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
5843 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5844 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
5845 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5846 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5847}
5848
5849
5850/**
5851 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
5852 *
5853 * @param pVCpu The cross context virtual CPU structure.
5854 */
5855DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
5856{
5857 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
5858 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5859 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
5860 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5861 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5862}
5863
5864
5865#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5866/**
5867 * Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
5868 *
5869 * @param pVCpu The cross context virtual CPU structure.
5870 * @param u32ErrCode The error code for the general-protection exception.
5871 */
5872DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
5873{
5874 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
5875 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5876 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
5877 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5878 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrCode, 0 /* GCPtrFaultAddress */);
5879}
5880
5881
5882/**
5883 * Sets a stack (\#SS) exception as pending-for-injection into the VM.
5884 *
5885 * @param pVCpu The cross context virtual CPU structure.
5886 * @param u32ErrCode The error code for the stack exception.
5887 */
5888DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
5889{
5890 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
5891 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5892 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
5893 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5894 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrCode, 0 /* GCPtrFaultAddress */);
5895}
5896
5897
5898/**
5899 * Decodes the memory operand of an instruction that caused a VM-exit.
5900 *
5901 * The VM-exit qualification field provides the displacement field for memory
5902 * operand instructions, if any.
5903 *
5904 * @returns Strict VBox status code (i.e. informational status codes too).
5905 * @retval VINF_SUCCESS if the operand was successfully decoded.
5906 * @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
5907 * operand.
5908 * @param pVCpu The cross context virtual CPU structure.
5909 * @param uExitInstrInfo The VM-exit instruction information field.
5910 * @param enmMemAccess The memory operand's access type (read or write).
5911 * @param GCPtrDisp The instruction displacement field, if any. For
5912 * RIP-relative addressing pass RIP + displacement here.
5913 * @param pGCPtrMem Where to store the effective destination memory address.
5914 */
5915static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
5916 PRTGCPTR pGCPtrMem)
5917{
5918 Assert(pGCPtrMem);
5919 Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
5920 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_EFER | CPUMCTX_EXTRN_CR0);
5921
5922 static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
5923 static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
5924 AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
5925
5926 VMXEXITINSTRINFO ExitInstrInfo;
5927 ExitInstrInfo.u = uExitInstrInfo;
5928 uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
5929 uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
5930 bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
5931 uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
5932 uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
5933 bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
5934 uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
5935 bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
5936 bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
5937
5938 /*
5939 * Validate instruction information.
5940 * This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
5941 */
5942 AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
5943 ("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
5944 AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
5945 ("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
5946 AssertLogRelMsgReturn(fIsMemOperand,
5947 ("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
5948
5949 /*
5950 * Compute the complete effective address.
5951 *
5952 * See AMD instruction spec. 1.4.2 "SIB Byte Format"
5953 * See AMD spec. 4.5.2 "Segment Registers".
5954 */
5955 RTGCPTR GCPtrMem = GCPtrDisp;
5956 if (fBaseRegValid)
5957 GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
5958 if (fIdxRegValid)
5959 GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
5960
5961 RTGCPTR const GCPtrOff = GCPtrMem;
5962 if ( !fIsLongMode
5963 || iSegReg >= X86_SREG_FS)
5964 GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
5965 GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
5966
5967 /*
5968 * Validate effective address.
5969 * See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
5970 */
5971 uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
5972 Assert(cbAccess > 0);
5973 if (fIsLongMode)
5974 {
5975 if (X86_IS_CANONICAL(GCPtrMem))
5976 {
5977 *pGCPtrMem = GCPtrMem;
5978 return VINF_SUCCESS;
5979 }
5980
5981 /** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
5982 * "Data Limit Checks in 64-bit Mode". */
5983 Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
5984 hmR0VmxSetPendingXcptGP(pVCpu, 0);
5985 return VINF_HM_PENDING_XCPT;
5986 }
5987
5988 /*
5989 * This is a watered down version of iemMemApplySegment().
5990 * Parts that are not applicable for VMX instructions like real-or-v8086 mode
5991 * and segment CPL/DPL checks are skipped.
5992 */
5993 RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
5994 RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
5995 PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
5996
5997 /* Check if the segment is present and usable. */
5998 if ( pSel->Attr.n.u1Present
5999 && !pSel->Attr.n.u1Unusable)
6000 {
6001 Assert(pSel->Attr.n.u1DescType);
6002 if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
6003 {
6004 /* Check permissions for the data segment. */
6005 if ( enmMemAccess == VMXMEMACCESS_WRITE
6006 && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
6007 {
6008 Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6009 hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
6010 return VINF_HM_PENDING_XCPT;
6011 }
6012
6013 /* Check limits if it's a normal data segment. */
6014 if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
6015 {
6016 if ( GCPtrFirst32 > pSel->u32Limit
6017 || GCPtrLast32 > pSel->u32Limit)
6018 {
6019 Log4Func(("Data segment limit exceeded."
6020 "iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6021 GCPtrLast32, pSel->u32Limit));
6022 if (iSegReg == X86_SREG_SS)
6023 hmR0VmxSetPendingXcptSS(pVCpu, 0);
6024 else
6025 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6026 return VINF_HM_PENDING_XCPT;
6027 }
6028 }
6029 else
6030 {
6031 /* Check limits if it's an expand-down data segment.
6032 Note! The upper boundary is defined by the B bit, not the G bit! */
6033 if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
6034 || GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
6035 {
6036 Log4Func(("Expand-down data segment limit exceeded."
6037 "iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6038 GCPtrLast32, pSel->u32Limit));
6039 if (iSegReg == X86_SREG_SS)
6040 hmR0VmxSetPendingXcptSS(pVCpu, 0);
6041 else
6042 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6043 return VINF_HM_PENDING_XCPT;
6044 }
6045 }
6046 }
6047 else
6048 {
6049 /* Check permissions for the code segment. */
6050 if ( enmMemAccess == VMXMEMACCESS_WRITE
6051 || ( enmMemAccess == VMXMEMACCESS_READ
6052 && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
6053 {
6054 Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
6055 Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
6056 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6057 return VINF_HM_PENDING_XCPT;
6058 }
6059
6060 /* Check limits for the code segment (normal/expand-down not applicable for code segments). */
6061 if ( GCPtrFirst32 > pSel->u32Limit
6062 || GCPtrLast32 > pSel->u32Limit)
6063 {
6064 Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
6065 GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
6066 if (iSegReg == X86_SREG_SS)
6067 hmR0VmxSetPendingXcptSS(pVCpu, 0);
6068 else
6069 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6070 return VINF_HM_PENDING_XCPT;
6071 }
6072 }
6073 }
6074 else
6075 {
6076 Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6077 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6078 return VINF_HM_PENDING_XCPT;
6079 }
6080
6081 *pGCPtrMem = GCPtrMem;
6082 return VINF_SUCCESS;
6083}
6084
6085
6086/**
6087 * Perform the relevant VMX instruction checks for VM-exits that occurred due to the
6088 * guest attempting to execute a VMX instruction.
6089 *
6090 * @returns Strict VBox status code (i.e. informational status codes too).
6091 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
6092 * @retval VINF_HM_PENDING_XCPT if an exception was raised.
6093 *
6094 * @param pVCpu The cross context virtual CPU structure.
6095 * @param uExitReason The VM-exit reason.
6096 *
6097 * @todo NstVmx: Document other error codes when VM-exit is implemented.
6098 * @remarks No-long-jump zone!!!
6099 */
6100static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
6101{
6102 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS
6103 | CPUMCTX_EXTRN_HWVIRT);
6104
6105 if ( CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx)
6106 || ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6107 && !CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx)))
6108 {
6109 Log4Func(("In real/v86-mode or long-mode outside 64-bit code segment -> #UD\n"));
6110 hmR0VmxSetPendingXcptUD(pVCpu);
6111 return VINF_HM_PENDING_XCPT;
6112 }
6113
6114 if (uExitReason == VMX_EXIT_VMXON)
6115 {
6116 /*
6117 * We check CR4.VMXE because it is required to be always set while in VMX operation
6118 * by physical CPUs and our CR4 read shadow is only consulted when executing specific
6119 * instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
6120 * otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
6121 */
6122 if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
6123 {
6124 Log4Func(("CR4.VMXE is not set -> #UD\n"));
6125 hmR0VmxSetPendingXcptUD(pVCpu);
6126 return VINF_HM_PENDING_XCPT;
6127 }
6128 }
6129 else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
6130 {
6131 /*
6132 * The guest has not entered VMX operation but attempted to execute a VMX instruction
6133 * (other than VMXON), we need to raise a #UD.
6134 */
6135 Log4Func(("Not in VMX root mode -> #UD\n"));
6136 hmR0VmxSetPendingXcptUD(pVCpu);
6137 return VINF_HM_PENDING_XCPT;
6138 }
6139
6140 if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
6141 {
6142 /*
6143 * The nested-guest attempted to execute a VMX instruction, cause a VM-exit and let
6144 * the guest hypervisor deal with it.
6145 */
6146 /** @todo NSTVMX: Trigger a VM-exit */
6147 }
6148
6149 /*
6150 * VMX instructions require CPL 0 except in VMX non-root mode where the VM-exit intercept
6151 * (above) takes preceedence over the CPL check.
6152 */
6153 if (CPUMGetGuestCPL(pVCpu) > 0)
6154 {
6155 Log4Func(("CPL > 0 -> #GP(0)\n"));
6156 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6157 return VINF_HM_PENDING_XCPT;
6158 }
6159
6160 return VINF_SUCCESS;
6161}
6162
6163#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
6164
6165
6166/**
6167 * Handle a condition that occurred while delivering an event through the guest
6168 * IDT.
6169 *
6170 * @returns Strict VBox status code (i.e. informational status codes too).
6171 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
6172 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
6173 * to continue execution of the guest which will delivery the \#DF.
6174 * @retval VINF_EM_RESET if we detected a triple-fault condition.
6175 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
6176 *
6177 * @param pVCpu The cross context virtual CPU structure.
6178 * @param pVmxTransient Pointer to the VMX transient structure.
6179 *
6180 * @remarks No-long-jump zone!!!
6181 */
6182static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6183{
6184 uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
6185
6186 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
6187 rc2 |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
6188 AssertRCReturn(rc2, rc2);
6189
6190 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
6191 if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6192 {
6193 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
6194 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
6195
6196 /*
6197 * If the event was a software interrupt (generated with INT n) or a software exception
6198 * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
6199 * can handle the VM-exit and continue guest execution which will re-execute the
6200 * instruction rather than re-injecting the exception, as that can cause premature
6201 * trips to ring-3 before injection and involve TRPM which currently has no way of
6202 * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
6203 * the problem).
6204 */
6205 IEMXCPTRAISE enmRaise;
6206 IEMXCPTRAISEINFO fRaiseInfo;
6207 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6208 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6209 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6210 {
6211 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
6212 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6213 }
6214 else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
6215 {
6216 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
6217 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
6218 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
6219 /** @todo Make AssertMsgReturn as just AssertMsg later. */
6220 AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
6221 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
6222 uExitVectorType), VERR_VMX_IPE_5);
6223
6224 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
6225
6226 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
6227 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
6228 {
6229 pVmxTransient->fVectoringPF = true;
6230 enmRaise = IEMXCPTRAISE_PREV_EVENT;
6231 }
6232 }
6233 else
6234 {
6235 /*
6236 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
6237 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
6238 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
6239 */
6240 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6241 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6242 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
6243 enmRaise = IEMXCPTRAISE_PREV_EVENT;
6244 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6245 }
6246
6247 /*
6248 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
6249 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
6250 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
6251 * subsequent VM-entry would fail.
6252 *
6253 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
6254 */
6255 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
6256 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6257 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
6258 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
6259 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
6260 {
6261 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6262 }
6263
6264 switch (enmRaise)
6265 {
6266 case IEMXCPTRAISE_CURRENT_XCPT:
6267 {
6268 Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
6269 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
6270 Assert(rcStrict == VINF_SUCCESS);
6271 break;
6272 }
6273
6274 case IEMXCPTRAISE_PREV_EVENT:
6275 {
6276 uint32_t u32ErrCode;
6277 if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
6278 {
6279 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6280 AssertRCReturn(rc2, rc2);
6281 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6282 }
6283 else
6284 u32ErrCode = 0;
6285
6286 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
6287 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6288 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6289 0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
6290
6291 Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
6292 pVCpu->hm.s.Event.u32ErrCode));
6293 Assert(rcStrict == VINF_SUCCESS);
6294 break;
6295 }
6296
6297 case IEMXCPTRAISE_REEXEC_INSTR:
6298 Assert(rcStrict == VINF_SUCCESS);
6299 break;
6300
6301 case IEMXCPTRAISE_DOUBLE_FAULT:
6302 {
6303 /*
6304 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
6305 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
6306 */
6307 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6308 {
6309 pVmxTransient->fVectoringDoublePF = true;
6310 Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
6311 pVCpu->cpum.GstCtx.cr2));
6312 rcStrict = VINF_SUCCESS;
6313 }
6314 else
6315 {
6316 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6317 hmR0VmxSetPendingXcptDF(pVCpu);
6318 Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
6319 uIdtVector, uExitVector));
6320 rcStrict = VINF_HM_DOUBLE_FAULT;
6321 }
6322 break;
6323 }
6324
6325 case IEMXCPTRAISE_TRIPLE_FAULT:
6326 {
6327 Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
6328 rcStrict = VINF_EM_RESET;
6329 break;
6330 }
6331
6332 case IEMXCPTRAISE_CPU_HANG:
6333 {
6334 Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
6335 rcStrict = VERR_EM_GUEST_CPU_HANG;
6336 break;
6337 }
6338
6339 default:
6340 {
6341 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6342 rcStrict = VERR_VMX_IPE_2;
6343 break;
6344 }
6345 }
6346 }
6347 else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6348 && VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6349 && uExitVector != X86_XCPT_DF
6350 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
6351 {
6352 /*
6353 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6354 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6355 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6356 */
6357 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6358 {
6359 Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
6360 VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6361 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6362 }
6363 }
6364
6365 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
6366 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
6367 return rcStrict;
6368}
6369
6370
6371/**
6372 * Imports a guest segment register from the current VMCS into
6373 * the guest-CPU context.
6374 *
6375 * @returns VBox status code.
6376 * @param pVCpu The cross context virtual CPU structure.
6377 * @param idxSel Index of the selector in the VMCS.
6378 * @param idxLimit Index of the segment limit in the VMCS.
6379 * @param idxBase Index of the segment base in the VMCS.
6380 * @param idxAccess Index of the access rights of the segment in the VMCS.
6381 * @param pSelReg Pointer to the segment selector.
6382 *
6383 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6384 * do not log!
6385 *
6386 * @remarks Never call this function directly!!! Use the
6387 * HMVMX_IMPORT_SREG() macro as that takes care
6388 * of whether to read from the VMCS cache or not.
6389 */
6390static int hmR0VmxImportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6391 PCPUMSELREG pSelReg)
6392{
6393 NOREF(pVCpu);
6394
6395 uint32_t u32Sel;
6396 uint32_t u32Limit;
6397 uint32_t u32Attr;
6398 uint64_t u64Base;
6399 int rc = VMXReadVmcs32(idxSel, &u32Sel);
6400 rc |= VMXReadVmcs32(idxLimit, &u32Limit);
6401 rc |= VMXReadVmcs32(idxAccess, &u32Attr);
6402 rc |= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
6403 AssertRCReturn(rc, rc);
6404
6405 pSelReg->Sel = (uint16_t)u32Sel;
6406 pSelReg->ValidSel = (uint16_t)u32Sel;
6407 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6408 pSelReg->u32Limit = u32Limit;
6409 pSelReg->u64Base = u64Base;
6410 pSelReg->Attr.u = u32Attr;
6411
6412 /*
6413 * If VT-x marks the segment as unusable, most other bits remain undefined:
6414 * - For CS the L, D and G bits have meaning.
6415 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6416 * - For the remaining data segments no bits are defined.
6417 *
6418 * The present bit and the unusable bit has been observed to be set at the
6419 * same time (the selector was supposed to be invalid as we started executing
6420 * a V8086 interrupt in ring-0).
6421 *
6422 * What should be important for the rest of the VBox code, is that the P bit is
6423 * cleared. Some of the other VBox code recognizes the unusable bit, but
6424 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6425 * safe side here, we'll strip off P and other bits we don't care about. If
6426 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6427 *
6428 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6429 */
6430 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6431 {
6432 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6433
6434 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6435 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6436 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6437#ifdef VBOX_STRICT
6438 VMMRZCallRing3Disable(pVCpu);
6439 Log4Func(("Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Sel, pSelReg->Attr.u));
6440# ifdef DEBUG_bird
6441 AssertMsg((u32Attr & ~X86DESCATTR_P) == pSelReg->Attr.u,
6442 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6443 idxSel, u32Sel, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6444# endif
6445 VMMRZCallRing3Enable(pVCpu);
6446#endif
6447 }
6448 return VINF_SUCCESS;
6449}
6450
6451
6452/**
6453 * Imports the guest RIP from the VMCS back into the guest-CPU context.
6454 *
6455 * @returns VBox status code.
6456 * @param pVCpu The cross context virtual CPU structure.
6457 *
6458 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6459 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6460 * instead!!!
6461 */
6462DECLINLINE(int) hmR0VmxImportGuestRip(PVMCPU pVCpu)
6463{
6464 uint64_t u64Val;
6465 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6466 if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
6467 {
6468 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6469 if (RT_SUCCESS(rc))
6470 {
6471 pCtx->rip = u64Val;
6472 EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
6473 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
6474 }
6475 return rc;
6476 }
6477 return VINF_SUCCESS;
6478}
6479
6480
6481/**
6482 * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
6483 *
6484 * @returns VBox status code.
6485 * @param pVCpu The cross context virtual CPU structure.
6486 *
6487 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6488 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6489 * instead!!!
6490 */
6491DECLINLINE(int) hmR0VmxImportGuestRFlags(PVMCPU pVCpu)
6492{
6493 uint32_t u32Val;
6494 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6495 if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
6496 {
6497 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
6498 if (RT_SUCCESS(rc))
6499 {
6500 pCtx->eflags.u32 = u32Val;
6501
6502 /* Restore eflags for real-on-v86-mode hack. */
6503 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6504 {
6505 pCtx->eflags.Bits.u1VM = 0;
6506 pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6507 }
6508 }
6509 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
6510 return rc;
6511 }
6512 return VINF_SUCCESS;
6513}
6514
6515
6516/**
6517 * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
6518 * context.
6519 *
6520 * @returns VBox status code.
6521 * @param pVCpu The cross context virtual CPU structure.
6522 *
6523 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6524 * do not log!
6525 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6526 * instead!!!
6527 */
6528DECLINLINE(int) hmR0VmxImportGuestIntrState(PVMCPU pVCpu)
6529{
6530 uint32_t u32Val;
6531 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6532 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
6533 if (RT_SUCCESS(rc))
6534 {
6535 /*
6536 * We additionally have a requirement to import RIP, RFLAGS depending on whether we
6537 * might need them in hmR0VmxEvaluatePendingEvent().
6538 */
6539 if (!u32Val)
6540 {
6541 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6542 {
6543 rc = hmR0VmxImportGuestRip(pVCpu);
6544 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6545 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6546 }
6547
6548 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6549 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6550 }
6551 else
6552 {
6553 rc = hmR0VmxImportGuestRip(pVCpu);
6554 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6555
6556 if (u32Val & ( VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
6557 | VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
6558 {
6559 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
6560 }
6561 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6562 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6563
6564 if (u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
6565 {
6566 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6567 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6568 }
6569 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6570 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6571 }
6572 }
6573 return rc;
6574}
6575
6576
6577/**
6578 * Worker for VMXR0ImportStateOnDemand.
6579 *
6580 * @returns VBox status code.
6581 * @param pVCpu The cross context virtual CPU structure.
6582 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6583 */
6584static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
6585{
6586#define VMXLOCAL_BREAK_RC(a_rc) \
6587 if (RT_FAILURE(a_rc)) \
6588 break
6589
6590 int rc = VINF_SUCCESS;
6591 PVM pVM = pVCpu->CTX_SUFF(pVM);
6592 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6593 uint64_t u64Val;
6594 uint32_t u32Val;
6595
6596 Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
6597 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
6598
6599 /*
6600 * We disable interrupts to make the updating of the state and in particular
6601 * the fExtrn modification atomic wrt to preemption hooks.
6602 */
6603 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
6604
6605 fWhat &= pCtx->fExtrn;
6606 if (fWhat)
6607 {
6608 do
6609 {
6610 if (fWhat & CPUMCTX_EXTRN_RIP)
6611 {
6612 rc = hmR0VmxImportGuestRip(pVCpu);
6613 VMXLOCAL_BREAK_RC(rc);
6614 }
6615
6616 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
6617 {
6618 rc = hmR0VmxImportGuestRFlags(pVCpu);
6619 VMXLOCAL_BREAK_RC(rc);
6620 }
6621
6622 if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
6623 {
6624 rc = hmR0VmxImportGuestIntrState(pVCpu);
6625 VMXLOCAL_BREAK_RC(rc);
6626 }
6627
6628 if (fWhat & CPUMCTX_EXTRN_RSP)
6629 {
6630 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6631 VMXLOCAL_BREAK_RC(rc);
6632 pCtx->rsp = u64Val;
6633 }
6634
6635 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
6636 {
6637 if (fWhat & CPUMCTX_EXTRN_CS)
6638 {
6639 rc = HMVMX_IMPORT_SREG(CS, &pCtx->cs);
6640 rc |= hmR0VmxImportGuestRip(pVCpu);
6641 VMXLOCAL_BREAK_RC(rc);
6642 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6643 pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6644 EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true);
6645 }
6646 if (fWhat & CPUMCTX_EXTRN_SS)
6647 {
6648 rc = HMVMX_IMPORT_SREG(SS, &pCtx->ss);
6649 VMXLOCAL_BREAK_RC(rc);
6650 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6651 pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6652 }
6653 if (fWhat & CPUMCTX_EXTRN_DS)
6654 {
6655 rc = HMVMX_IMPORT_SREG(DS, &pCtx->ds);
6656 VMXLOCAL_BREAK_RC(rc);
6657 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6658 pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6659 }
6660 if (fWhat & CPUMCTX_EXTRN_ES)
6661 {
6662 rc = HMVMX_IMPORT_SREG(ES, &pCtx->es);
6663 VMXLOCAL_BREAK_RC(rc);
6664 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6665 pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6666 }
6667 if (fWhat & CPUMCTX_EXTRN_FS)
6668 {
6669 rc = HMVMX_IMPORT_SREG(FS, &pCtx->fs);
6670 VMXLOCAL_BREAK_RC(rc);
6671 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6672 pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6673 }
6674 if (fWhat & CPUMCTX_EXTRN_GS)
6675 {
6676 rc = HMVMX_IMPORT_SREG(GS, &pCtx->gs);
6677 VMXLOCAL_BREAK_RC(rc);
6678 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6679 pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6680 }
6681 }
6682
6683 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
6684 {
6685 if (fWhat & CPUMCTX_EXTRN_LDTR)
6686 {
6687 rc = HMVMX_IMPORT_SREG(LDTR, &pCtx->ldtr);
6688 VMXLOCAL_BREAK_RC(rc);
6689 }
6690
6691 if (fWhat & CPUMCTX_EXTRN_GDTR)
6692 {
6693 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6694 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
6695 VMXLOCAL_BREAK_RC(rc);
6696 pCtx->gdtr.pGdt = u64Val;
6697 pCtx->gdtr.cbGdt = u32Val;
6698 }
6699
6700 /* Guest IDTR. */
6701 if (fWhat & CPUMCTX_EXTRN_IDTR)
6702 {
6703 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6704 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
6705 VMXLOCAL_BREAK_RC(rc);
6706 pCtx->idtr.pIdt = u64Val;
6707 pCtx->idtr.cbIdt = u32Val;
6708 }
6709
6710 /* Guest TR. */
6711 if (fWhat & CPUMCTX_EXTRN_TR)
6712 {
6713 /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
6714 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6715 {
6716 rc = HMVMX_IMPORT_SREG(TR, &pCtx->tr);
6717 VMXLOCAL_BREAK_RC(rc);
6718 }
6719 }
6720 }
6721
6722 if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
6723 {
6724 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
6725 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
6726 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
6727 pCtx->SysEnter.cs = u32Val;
6728 VMXLOCAL_BREAK_RC(rc);
6729 }
6730
6731#if HC_ARCH_BITS == 64
6732 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
6733 {
6734 if ( pVM->hm.s.fAllow64BitGuests
6735 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6736 pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
6737 }
6738
6739 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
6740 {
6741 if ( pVM->hm.s.fAllow64BitGuests
6742 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6743 {
6744 pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
6745 pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
6746 pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
6747 }
6748 }
6749#endif
6750
6751 if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
6752#if HC_ARCH_BITS == 32
6753 || (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS))
6754#endif
6755 )
6756 {
6757 PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6758 uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
6759 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6760 {
6761 switch (pMsr->u32Msr)
6762 {
6763#if HC_ARCH_BITS == 32
6764 case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
6765 case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
6766 case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
6767 case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6768#endif
6769 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6770 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6771 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit */ break;
6772 default:
6773 {
6774 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6775 ASMSetFlags(fEFlags);
6776 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
6777 cMsrs));
6778 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6779 }
6780 }
6781 }
6782 }
6783
6784 if (fWhat & CPUMCTX_EXTRN_DR7)
6785 {
6786 if (!pVCpu->hm.s.fUsingHyperDR7)
6787 {
6788 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6789 rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
6790 VMXLOCAL_BREAK_RC(rc);
6791 pCtx->dr[7] = u32Val;
6792 }
6793 }
6794
6795 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
6796 {
6797 uint32_t u32Shadow;
6798 if (fWhat & CPUMCTX_EXTRN_CR0)
6799 {
6800 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
6801 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
6802 VMXLOCAL_BREAK_RC(rc);
6803 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr0Mask)
6804 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr0Mask);
6805 VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
6806 CPUMSetGuestCR0(pVCpu, u32Val);
6807 VMMRZCallRing3Enable(pVCpu);
6808 }
6809
6810 if (fWhat & CPUMCTX_EXTRN_CR4)
6811 {
6812 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
6813 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
6814 VMXLOCAL_BREAK_RC(rc);
6815 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr4Mask)
6816 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr4Mask);
6817 CPUMSetGuestCR4(pVCpu, u32Val);
6818 }
6819
6820 if (fWhat & CPUMCTX_EXTRN_CR3)
6821 {
6822 /* CR0.PG bit changes are always intercepted, so it's up to date. */
6823 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6824 || ( pVM->hm.s.fNestedPaging
6825 && CPUMIsGuestPagingEnabledEx(pCtx)))
6826 {
6827 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6828 if (pCtx->cr3 != u64Val)
6829 {
6830 CPUMSetGuestCR3(pVCpu, u64Val);
6831 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6832 }
6833
6834 /* If the guest is in PAE mode, sync back the PDPE's into the guest state.
6835 Note: CR4.PAE, CR0.PG, EFER bit changes are always intercepted, so they're up to date. */
6836 if (CPUMIsGuestInPAEModeEx(pCtx))
6837 {
6838 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6839 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6840 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6841 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6842 VMXLOCAL_BREAK_RC(rc);
6843 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6844 }
6845 }
6846 }
6847 }
6848 } while (0);
6849
6850 if (RT_SUCCESS(rc))
6851 {
6852 /* Update fExtrn. */
6853 pCtx->fExtrn &= ~fWhat;
6854
6855 /* If everything has been imported, clear the HM keeper bit. */
6856 if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
6857 {
6858 pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
6859 Assert(!pCtx->fExtrn);
6860 }
6861 }
6862 }
6863 else
6864 AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
6865
6866 ASMSetFlags(fEFlags);
6867
6868 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
6869
6870 /*
6871 * Honor any pending CR3 updates.
6872 *
6873 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6874 * -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6875 * -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
6876 *
6877 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6878 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6879 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6880 * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
6881 *
6882 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6883 */
6884 if (VMMRZCallRing3IsEnabled(pVCpu))
6885 {
6886 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6887 {
6888 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
6889 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6890 }
6891
6892 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6893 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6894
6895 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6896 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6897 }
6898
6899 return VINF_SUCCESS;
6900#undef VMXLOCAL_BREAK_RC
6901}
6902
6903
6904/**
6905 * Saves the guest state from the VMCS into the guest-CPU context.
6906 *
6907 * @returns VBox status code.
6908 * @param pVCpu The cross context virtual CPU structure.
6909 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6910 */
6911VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
6912{
6913 return hmR0VmxImportGuestState(pVCpu, fWhat);
6914}
6915
6916
6917/**
6918 * Check per-VM and per-VCPU force flag actions that require us to go back to
6919 * ring-3 for one reason or another.
6920 *
6921 * @returns Strict VBox status code (i.e. informational status codes too)
6922 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6923 * ring-3.
6924 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6925 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6926 * interrupts)
6927 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6928 * all EMTs to be in ring-3.
6929 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6930 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6931 * to the EM loop.
6932 *
6933 * @param pVCpu The cross context virtual CPU structure.
6934 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6935 */
6936static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
6937{
6938 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6939
6940 /*
6941 * Anything pending? Should be more likely than not if we're doing a good job.
6942 */
6943 PVM pVM = pVCpu->CTX_SUFF(pVM);
6944 if ( !fStepping
6945 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6946 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6947 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6948 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6949 return VINF_SUCCESS;
6950
6951 /* Pending PGM C3 sync. */
6952 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6953 {
6954 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6955 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
6956 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
6957 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6958 if (rcStrict2 != VINF_SUCCESS)
6959 {
6960 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6961 Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6962 return rcStrict2;
6963 }
6964 }
6965
6966 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6967 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6968 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6969 {
6970 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6971 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6972 Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6973 return rc2;
6974 }
6975
6976 /* Pending VM request packets, such as hardware interrupts. */
6977 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6978 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6979 {
6980 Log4Func(("Pending VM request forcing us back to ring-3\n"));
6981 return VINF_EM_PENDING_REQUEST;
6982 }
6983
6984 /* Pending PGM pool flushes. */
6985 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6986 {
6987 Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
6988 return VINF_PGM_POOL_FLUSH_PENDING;
6989 }
6990
6991 /* Pending DMA requests. */
6992 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6993 {
6994 Log4Func(("Pending DMA request forcing us back to ring-3\n"));
6995 return VINF_EM_RAW_TO_R3;
6996 }
6997
6998 return VINF_SUCCESS;
6999}
7000
7001
7002/**
7003 * Converts any TRPM trap into a pending HM event. This is typically used when
7004 * entering from ring-3 (not longjmp returns).
7005 *
7006 * @param pVCpu The cross context virtual CPU structure.
7007 */
7008static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7009{
7010 Assert(TRPMHasTrap(pVCpu));
7011 Assert(!pVCpu->hm.s.Event.fPending);
7012
7013 uint8_t uVector;
7014 TRPMEVENT enmTrpmEvent;
7015 RTGCUINT uErrCode;
7016 RTGCUINTPTR GCPtrFaultAddress;
7017 uint8_t cbInstr;
7018
7019 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7020 AssertRC(rc);
7021
7022 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7023 uint32_t u32IntInfo = uVector | VMX_EXIT_INT_INFO_VALID;
7024 if (enmTrpmEvent == TRPM_TRAP)
7025 {
7026 switch (uVector)
7027 {
7028 case X86_XCPT_NMI:
7029 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7030 break;
7031
7032 case X86_XCPT_BP:
7033 case X86_XCPT_OF:
7034 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7035 break;
7036
7037 case X86_XCPT_PF:
7038 case X86_XCPT_DF:
7039 case X86_XCPT_TS:
7040 case X86_XCPT_NP:
7041 case X86_XCPT_SS:
7042 case X86_XCPT_GP:
7043 case X86_XCPT_AC:
7044 u32IntInfo |= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7045 RT_FALL_THRU();
7046 default:
7047 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7048 break;
7049 }
7050 }
7051 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7052 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7053 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7054 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7055 else
7056 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7057
7058 rc = TRPMResetTrap(pVCpu);
7059 AssertRC(rc);
7060 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7061 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7062
7063 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7064}
7065
7066
7067/**
7068 * Converts the pending HM event into a TRPM trap.
7069 *
7070 * @param pVCpu The cross context virtual CPU structure.
7071 */
7072static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7073{
7074 Assert(pVCpu->hm.s.Event.fPending);
7075
7076 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7077 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7078 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
7079 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7080
7081 /* If a trap was already pending, we did something wrong! */
7082 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7083
7084 TRPMEVENT enmTrapType;
7085 switch (uVectorType)
7086 {
7087 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7088 enmTrapType = TRPM_HARDWARE_INT;
7089 break;
7090
7091 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7092 enmTrapType = TRPM_SOFTWARE_INT;
7093 break;
7094
7095 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7096 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7097 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7098 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7099 enmTrapType = TRPM_TRAP;
7100 break;
7101
7102 default:
7103 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7104 enmTrapType = TRPM_32BIT_HACK;
7105 break;
7106 }
7107
7108 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7109
7110 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7111 AssertRC(rc);
7112
7113 if (fErrorCodeValid)
7114 TRPMSetErrorCode(pVCpu, uErrorCode);
7115
7116 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7117 && uVector == X86_XCPT_PF)
7118 {
7119 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7120 }
7121 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7122 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7123 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7124 {
7125 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7126 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
7127 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7128 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7129 }
7130
7131 /* Clear any pending events from the VMCS. */
7132 VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7133 VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, 0);
7134
7135 /* We're now done converting the pending event. */
7136 pVCpu->hm.s.Event.fPending = false;
7137}
7138
7139
7140/**
7141 * Does the necessary state syncing before returning to ring-3 for any reason
7142 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7143 *
7144 * @returns VBox status code.
7145 * @param pVCpu The cross context virtual CPU structure.
7146 * @param fImportState Whether to import the guest state from the VMCS back
7147 * to the guest-CPU context.
7148 *
7149 * @remarks No-long-jmp zone!!!
7150 */
7151static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
7152{
7153 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7154 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7155
7156 RTCPUID idCpu = RTMpCpuId();
7157 Log4Func(("HostCpuId=%u\n", idCpu));
7158
7159 /*
7160 * !!! IMPORTANT !!!
7161 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7162 */
7163
7164 /* Save the guest state if necessary. */
7165 if (fImportState)
7166 {
7167 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
7168 AssertRCReturn(rc, rc);
7169 }
7170
7171 /* Restore host FPU state if necessary. We will resync on next R0 reentry. */
7172 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7173 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
7174
7175 /* Restore host debug registers if necessary. We will resync on next R0 reentry. */
7176#ifdef VBOX_STRICT
7177 if (CPUMIsHyperDebugStateActive(pVCpu))
7178 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
7179#endif
7180 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7181 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7182 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7183
7184#if HC_ARCH_BITS == 64
7185 /* Restore host-state bits that VT-x only restores partially. */
7186 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7187 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7188 {
7189 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7190 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7191 }
7192 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7193#endif
7194
7195 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7196 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7197 {
7198 /* We shouldn't restore the host MSRs without saving the guest MSRs first. */
7199 if (!fImportState)
7200 {
7201 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS);
7202 AssertRCReturn(rc, rc);
7203 }
7204 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7205 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7206 }
7207 else
7208 pVCpu->hm.s.vmx.fLazyMsrs = 0;
7209
7210 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7211 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7212
7213 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7214 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
7215 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
7216 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
7217 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
7218 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7219 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7220 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7221 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7222
7223 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7224
7225 /** @todo This partially defeats the purpose of having preemption hooks.
7226 * The problem is, deregistering the hooks should be moved to a place that
7227 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7228 * context.
7229 */
7230 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7231 {
7232 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7233 AssertRCReturn(rc, rc);
7234
7235 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7236 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7237 }
7238 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7239 NOREF(idCpu);
7240
7241 return VINF_SUCCESS;
7242}
7243
7244
7245/**
7246 * Leaves the VT-x session.
7247 *
7248 * @returns VBox status code.
7249 * @param pVCpu The cross context virtual CPU structure.
7250 *
7251 * @remarks No-long-jmp zone!!!
7252 */
7253static int hmR0VmxLeaveSession(PVMCPU pVCpu)
7254{
7255 HM_DISABLE_PREEMPT(pVCpu);
7256 HMVMX_ASSERT_CPU_SAFE(pVCpu);
7257 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7258 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7259
7260 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7261 and done this from the VMXR0ThreadCtxCallback(). */
7262 if (!pVCpu->hm.s.fLeaveDone)
7263 {
7264 int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
7265 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7266 pVCpu->hm.s.fLeaveDone = true;
7267 }
7268 Assert(!pVCpu->cpum.GstCtx.fExtrn);
7269
7270 /*
7271 * !!! IMPORTANT !!!
7272 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7273 */
7274
7275 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7276 /** @todo Deregistering here means we need to VMCLEAR always
7277 * (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
7278 * for calling VMMR0ThreadCtxHookDisable here! */
7279 VMMR0ThreadCtxHookDisable(pVCpu);
7280
7281 /* Leave HM context. This takes care of local init (term). */
7282 int rc = HMR0LeaveCpu(pVCpu);
7283
7284 HM_RESTORE_PREEMPT();
7285 return rc;
7286}
7287
7288
7289/**
7290 * Does the necessary state syncing before doing a longjmp to ring-3.
7291 *
7292 * @returns VBox status code.
7293 * @param pVCpu The cross context virtual CPU structure.
7294 *
7295 * @remarks No-long-jmp zone!!!
7296 */
7297DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
7298{
7299 return hmR0VmxLeaveSession(pVCpu);
7300}
7301
7302
7303/**
7304 * Take necessary actions before going back to ring-3.
7305 *
7306 * An action requires us to go back to ring-3. This function does the necessary
7307 * steps before we can safely return to ring-3. This is not the same as longjmps
7308 * to ring-3, this is voluntary and prepares the guest so it may continue
7309 * executing outside HM (recompiler/IEM).
7310 *
7311 * @returns VBox status code.
7312 * @param pVCpu The cross context virtual CPU structure.
7313 * @param rcExit The reason for exiting to ring-3. Can be
7314 * VINF_VMM_UNKNOWN_RING3_CALL.
7315 */
7316static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
7317{
7318 Assert(pVCpu);
7319 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7320
7321 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7322 {
7323 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7324 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7325 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7326 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7327 }
7328
7329 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7330 VMMRZCallRing3Disable(pVCpu);
7331 Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
7332
7333 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7334 if (pVCpu->hm.s.Event.fPending)
7335 {
7336 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7337 Assert(!pVCpu->hm.s.Event.fPending);
7338 }
7339
7340 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7341 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7342
7343 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7344 and if we're injecting an event we should have a TRPM trap pending. */
7345 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7346#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
7347 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7348#endif
7349
7350 /* Save guest state and restore host state bits. */
7351 int rc = hmR0VmxLeaveSession(pVCpu);
7352 AssertRCReturn(rc, rc);
7353 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7354 /* Thread-context hooks are unregistered at this point!!! */
7355
7356 /* Sync recompiler state. */
7357 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7358 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7359 | CPUM_CHANGED_LDTR
7360 | CPUM_CHANGED_GDTR
7361 | CPUM_CHANGED_IDTR
7362 | CPUM_CHANGED_TR
7363 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7364 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
7365 && CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
7366 {
7367 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7368 }
7369
7370 Assert(!pVCpu->hm.s.fClearTrapFlag);
7371
7372 /* Update the exit-to-ring 3 reason. */
7373 pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
7374
7375 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7376 if (rcExit != VINF_EM_RAW_INTERRUPT)
7377 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7378
7379 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7380
7381 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7382 VMMRZCallRing3RemoveNotification(pVCpu);
7383 VMMRZCallRing3Enable(pVCpu);
7384
7385 return rc;
7386}
7387
7388
7389/**
7390 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7391 * longjump to ring-3 and possibly get preempted.
7392 *
7393 * @returns VBox status code.
7394 * @param pVCpu The cross context virtual CPU structure.
7395 * @param enmOperation The operation causing the ring-3 longjump.
7396 * @param pvUser User argument, currently unused, NULL.
7397 */
7398static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7399{
7400 RT_NOREF(pvUser);
7401 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7402 {
7403 /*
7404 * !!! IMPORTANT !!!
7405 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7406 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7407 */
7408 VMMRZCallRing3RemoveNotification(pVCpu);
7409 VMMRZCallRing3Disable(pVCpu);
7410 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7411 RTThreadPreemptDisable(&PreemptState);
7412
7413 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
7414 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7415 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7416
7417#if HC_ARCH_BITS == 64
7418 /* Restore host-state bits that VT-x only restores partially. */
7419 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7420 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7421 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7422 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7423#endif
7424
7425 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7426 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7427 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7428
7429 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7430 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7431 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7432 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7433 {
7434 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7435 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7436 }
7437
7438 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7439 VMMR0ThreadCtxHookDisable(pVCpu);
7440 HMR0LeaveCpu(pVCpu);
7441 RTThreadPreemptRestore(&PreemptState);
7442 return VINF_SUCCESS;
7443 }
7444
7445 Assert(pVCpu);
7446 Assert(pvUser);
7447 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7448 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7449
7450 VMMRZCallRing3Disable(pVCpu);
7451 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7452
7453 Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
7454
7455 int rc = hmR0VmxLongJmpToRing3(pVCpu);
7456 AssertRCReturn(rc, rc);
7457
7458 VMMRZCallRing3Enable(pVCpu);
7459 return VINF_SUCCESS;
7460}
7461
7462
7463/**
7464 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7465 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7466 *
7467 * @param pVCpu The cross context virtual CPU structure.
7468 */
7469DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7470{
7471 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7472 {
7473 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7474 {
7475 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_PROC_CTLS_INT_WINDOW_EXIT;
7476 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7477 AssertRC(rc);
7478 Log4Func(("Setup interrupt-window exiting\n"));
7479 }
7480 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7481}
7482
7483
7484/**
7485 * Clears the interrupt-window exiting control in the VMCS.
7486 *
7487 * @param pVCpu The cross context virtual CPU structure.
7488 */
7489DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7490{
7491 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT);
7492 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
7493 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7494 AssertRC(rc);
7495 Log4Func(("Cleared interrupt-window exiting\n"));
7496}
7497
7498
7499/**
7500 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7501 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7502 *
7503 * @param pVCpu The cross context virtual CPU structure.
7504 */
7505DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7506{
7507 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7508 {
7509 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7510 {
7511 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7512 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7513 AssertRC(rc);
7514 Log4Func(("Setup NMI-window exiting\n"));
7515 }
7516 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7517}
7518
7519
7520/**
7521 * Clears the NMI-window exiting control in the VMCS.
7522 *
7523 * @param pVCpu The cross context virtual CPU structure.
7524 */
7525DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7526{
7527 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT);
7528 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7529 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7530 AssertRC(rc);
7531 Log4Func(("Cleared NMI-window exiting\n"));
7532}
7533
7534
7535/**
7536 * Evaluates the event to be delivered to the guest and sets it as the pending
7537 * event.
7538 *
7539 * @returns The VT-x guest-interruptibility state.
7540 * @param pVCpu The cross context virtual CPU structure.
7541 */
7542static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu)
7543{
7544 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7545 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7546 uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu);
7547 bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7548 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7549 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7550
7551 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7552 Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7553 Assert(!fBlockSti || pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7554 Assert(!TRPMHasTrap(pVCpu));
7555
7556 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7557 APICUpdatePendingInterrupts(pVCpu);
7558
7559 /*
7560 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7561 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7562 */
7563 /** @todo SMI. SMIs take priority over NMIs. */
7564 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7565 {
7566 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7567 if ( !pVCpu->hm.s.Event.fPending
7568 && !fBlockNmi
7569 && !fBlockSti
7570 && !fBlockMovSS)
7571 {
7572 Log4Func(("Pending NMI\n"));
7573 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INT_INFO_VALID;
7574 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7575
7576 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7577 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7578 }
7579 else
7580 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7581 }
7582 /*
7583 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7584 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7585 */
7586 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7587 && !pVCpu->hm.s.fSingleInstruction)
7588 {
7589 Assert(!DBGFIsStepping(pVCpu));
7590 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7591 AssertRCReturn(rc, 0);
7592 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7593 if ( !pVCpu->hm.s.Event.fPending
7594 && !fBlockInt
7595 && !fBlockSti
7596 && !fBlockMovSS)
7597 {
7598 uint8_t u8Interrupt;
7599 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7600 if (RT_SUCCESS(rc))
7601 {
7602 Log4Func(("Pending external interrupt u8Interrupt=%#x\n", u8Interrupt));
7603 uint32_t u32IntInfo = u8Interrupt
7604 | VMX_EXIT_INT_INFO_VALID
7605 | (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7606
7607 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7608 }
7609 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7610 {
7611 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
7612 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7613 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7614
7615 /*
7616 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7617 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7618 * need to re-set this force-flag here.
7619 */
7620 }
7621 else
7622 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7623 }
7624 else
7625 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7626 }
7627
7628 return fIntrState;
7629}
7630
7631
7632/**
7633 * Sets a pending-debug exception to be delivered to the guest if the guest is
7634 * single-stepping in the VMCS.
7635 *
7636 * @param pVCpu The cross context virtual CPU structure.
7637 */
7638DECLINLINE(int) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7639{
7640 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7641 RT_NOREF(pVCpu);
7642 return VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
7643}
7644
7645
7646/**
7647 * Injects any pending events into the guest if the guest is in a state to
7648 * receive them.
7649 *
7650 * @returns Strict VBox status code (i.e. informational status codes too).
7651 * @param pVCpu The cross context virtual CPU structure.
7652 * @param fIntrState The VT-x guest-interruptibility state.
7653 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7654 * return VINF_EM_DBG_STEPPED if the event was
7655 * dispatched directly.
7656 */
7657static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, uint32_t fIntrState, bool fStepping)
7658{
7659 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7660 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7661
7662 bool fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7663 bool fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7664
7665 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7666 Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7667 Assert(!fBlockSti || pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7668 Assert(!TRPMHasTrap(pVCpu));
7669
7670 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7671 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7672 if (pVCpu->hm.s.Event.fPending)
7673 {
7674 /*
7675 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7676 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7677 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7678 *
7679 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7680 */
7681 uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7682#ifdef VBOX_STRICT
7683 if (uIntType == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
7684 {
7685 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7686 Assert(!fBlockInt);
7687 Assert(!fBlockSti);
7688 Assert(!fBlockMovSS);
7689 }
7690 else if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
7691 {
7692 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7693 Assert(!fBlockSti);
7694 Assert(!fBlockMovSS);
7695 Assert(!fBlockNmi);
7696 }
7697#endif
7698 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7699 uIntType));
7700 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7701 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping,
7702 &fIntrState);
7703 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7704
7705 /* Update the interruptibility-state as it could have been changed by
7706 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7707 fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7708 fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7709
7710 if (uIntType == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
7711 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7712 else
7713 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7714 }
7715
7716 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7717 if ( fBlockSti
7718 || fBlockMovSS)
7719 {
7720 if (!pVCpu->hm.s.fSingleInstruction)
7721 {
7722 /*
7723 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7724 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7725 * See Intel spec. 27.3.4 "Saving Non-Register State".
7726 */
7727 Assert(!DBGFIsStepping(pVCpu));
7728 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7729 AssertRCReturn(rc, rc);
7730 if (pCtx->eflags.Bits.u1TF)
7731 {
7732 int rc2 = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7733 AssertRCReturn(rc2, rc2);
7734 }
7735 }
7736 else if (pCtx->eflags.Bits.u1TF)
7737 {
7738 /*
7739 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7740 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7741 */
7742 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
7743 fIntrState = 0;
7744 }
7745 }
7746
7747 /*
7748 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7749 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7750 */
7751 int rc3 = hmR0VmxExportGuestIntrState(pVCpu, fIntrState);
7752 AssertRCReturn(rc3, rc3);
7753
7754 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7755 NOREF(fBlockMovSS); NOREF(fBlockSti);
7756 return rcStrict;
7757}
7758
7759
7760/**
7761 * Injects a double-fault (\#DF) exception into the VM.
7762 *
7763 * @returns Strict VBox status code (i.e. informational status codes too).
7764 * @param pVCpu The cross context virtual CPU structure.
7765 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7766 * and should return VINF_EM_DBG_STEPPED if the event
7767 * is injected directly (register modified by us, not
7768 * by hardware on VM-entry).
7769 * @param pfIntrState Pointer to the current guest interruptibility-state.
7770 * This interruptibility-state will be updated if
7771 * necessary. This cannot not be NULL.
7772 */
7773DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, bool fStepping, uint32_t *pfIntrState)
7774{
7775 uint32_t const u32IntInfo = X86_XCPT_DF | VMX_EXIT_INT_INFO_VALID
7776 | (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT)
7777 | VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7778 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */, fStepping,
7779 pfIntrState);
7780}
7781
7782
7783/**
7784 * Injects a general-protection (\#GP) fault into the VM.
7785 *
7786 * @returns Strict VBox status code (i.e. informational status codes too).
7787 * @param pVCpu The cross context virtual CPU structure.
7788 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7789 * mode, i.e. in real-mode it's not valid).
7790 * @param u32ErrorCode The error code associated with the \#GP.
7791 * @param fStepping Whether we're running in
7792 * hmR0VmxRunGuestCodeStep() and should return
7793 * VINF_EM_DBG_STEPPED if the event is injected
7794 * directly (register modified by us, not by
7795 * hardware on VM-entry).
7796 * @param pfIntrState Pointer to the current guest interruptibility-state.
7797 * This interruptibility-state will be updated if
7798 * necessary. This cannot not be NULL.
7799 */
7800DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, bool fErrorCodeValid, uint32_t u32ErrorCode, bool fStepping,
7801 uint32_t *pfIntrState)
7802{
7803 uint32_t const u32IntInfo = X86_XCPT_GP | VMX_EXIT_INT_INFO_VALID
7804 | (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT)
7805 | (fErrorCodeValid ? VMX_EXIT_INT_INFO_ERROR_CODE_VALID : 0);
7806 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */, fStepping,
7807 pfIntrState);
7808}
7809
7810
7811/**
7812 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7813 * stack.
7814 *
7815 * @returns Strict VBox status code (i.e. informational status codes too).
7816 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7817 * @param pVCpu The cross context virtual CPU structure.
7818 * @param uValue The value to push to the guest stack.
7819 */
7820static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
7821{
7822 /*
7823 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7824 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7825 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7826 */
7827 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7828 if (pCtx->sp == 1)
7829 return VINF_EM_RESET;
7830 pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7831 int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
7832 AssertRC(rc);
7833 return rc;
7834}
7835
7836
7837/**
7838 * Injects an event into the guest upon VM-entry by updating the relevant fields
7839 * in the VM-entry area in the VMCS.
7840 *
7841 * @returns Strict VBox status code (i.e. informational status codes too).
7842 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7843 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7844 *
7845 * @param pVCpu The cross context virtual CPU structure.
7846 * @param u64IntInfo The VM-entry interruption-information field.
7847 * @param cbInstr The VM-entry instruction length in bytes (for
7848 * software interrupts, exceptions and privileged
7849 * software exceptions).
7850 * @param u32ErrCode The VM-entry exception error code.
7851 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7852 * @param pfIntrState Pointer to the current guest interruptibility-state.
7853 * This interruptibility-state will be updated if
7854 * necessary. This cannot not be NULL.
7855 * @param fStepping Whether we're running in
7856 * hmR0VmxRunGuestCodeStep() and should return
7857 * VINF_EM_DBG_STEPPED if the event is injected
7858 * directly (register modified by us, not by
7859 * hardware on VM-entry).
7860 */
7861static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7862 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState)
7863{
7864 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7865 AssertMsg(!RT_HI_U32(u64IntInfo), ("%#RX64\n", u64IntInfo));
7866 Assert(pfIntrState);
7867
7868 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7869 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7870 uint32_t const uVector = VMX_EXIT_INT_INFO_VECTOR(u32IntInfo);
7871 uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(u32IntInfo);
7872
7873#ifdef VBOX_STRICT
7874 /*
7875 * Validate the error-code-valid bit for hardware exceptions.
7876 * No error codes for exceptions in real-mode.
7877 *
7878 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7879 */
7880 if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
7881 && !CPUMIsGuestInRealModeEx(pCtx))
7882 {
7883 switch (uVector)
7884 {
7885 case X86_XCPT_PF:
7886 case X86_XCPT_DF:
7887 case X86_XCPT_TS:
7888 case X86_XCPT_NP:
7889 case X86_XCPT_SS:
7890 case X86_XCPT_GP:
7891 case X86_XCPT_AC:
7892 AssertMsg(VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
7893 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7894 RT_FALL_THRU();
7895 default:
7896 break;
7897 }
7898 }
7899#endif
7900
7901 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7902 Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
7903 || !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
7904
7905 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7906
7907 /*
7908 * Hardware interrupts & exceptions cannot be delivered through the software interrupt
7909 * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
7910 * interrupt handler in the (real-mode) guest.
7911 *
7912 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
7913 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7914 */
7915 if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
7916 {
7917 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
7918 {
7919 /*
7920 * For unrestricted execution enabled CPUs running real-mode guests, we must not
7921 * set the deliver-error-code bit.
7922 *
7923 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7924 */
7925 u32IntInfo &= ~VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7926 }
7927 else
7928 {
7929 PVM pVM = pVCpu->CTX_SUFF(pVM);
7930 Assert(PDMVmmDevHeapIsEnabled(pVM));
7931 Assert(pVM->hm.s.vmx.pRealModeTSS);
7932
7933 /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
7934 int rc2 = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_TABLE_MASK | CPUMCTX_EXTRN_RIP
7935 | CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_RFLAGS);
7936 AssertRCReturn(rc2, rc2);
7937
7938 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7939 size_t const cbIdtEntry = sizeof(X86IDTR16);
7940 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
7941 {
7942 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7943 if (uVector == X86_XCPT_DF)
7944 return VINF_EM_RESET;
7945
7946 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7947 if (uVector == X86_XCPT_GP)
7948 return hmR0VmxInjectXcptDF(pVCpu, fStepping, pfIntrState);
7949
7950 /*
7951 * If we're injecting an event with no valid IDT entry, inject a #GP.
7952 * No error codes for exceptions in real-mode.
7953 *
7954 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7955 */
7956 return hmR0VmxInjectXcptGP(pVCpu, false /* fErrCodeValid */, 0 /* u32ErrCode */, fStepping, pfIntrState);
7957 }
7958
7959 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7960 uint16_t uGuestIp = pCtx->ip;
7961 if (uIntType == VMX_EXIT_INT_INFO_TYPE_SW_XCPT)
7962 {
7963 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7964 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7965 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7966 }
7967 else if (uIntType == VMX_EXIT_INT_INFO_TYPE_SW_INT)
7968 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7969
7970 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7971 X86IDTR16 IdtEntry;
7972 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
7973 rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7974 AssertRCReturn(rc2, rc2);
7975
7976 /* Construct the stack frame for the interrupt/exception handler. */
7977 VBOXSTRICTRC rcStrict;
7978 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
7979 if (rcStrict == VINF_SUCCESS)
7980 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
7981 if (rcStrict == VINF_SUCCESS)
7982 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
7983
7984 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7985 if (rcStrict == VINF_SUCCESS)
7986 {
7987 pCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7988 pCtx->rip = IdtEntry.offSel;
7989 pCtx->cs.Sel = IdtEntry.uSel;
7990 pCtx->cs.ValidSel = IdtEntry.uSel;
7991 pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7992 if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
7993 && uVector == X86_XCPT_PF)
7994 pCtx->cr2 = GCPtrFaultAddress;
7995
7996 /* If any other guest-state bits are changed here, make sure to update
7997 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7998 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS | HM_CHANGED_GUEST_CR2
7999 | HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
8000 | HM_CHANGED_GUEST_RSP);
8001
8002 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8003 if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
8004 {
8005 Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
8006 && uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
8007 Log4Func(("Clearing inhibition due to STI\n"));
8008 *pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
8009 }
8010 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8011 u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
8012
8013 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8014 it, if we are returning to ring-3 before executing guest code. */
8015 pVCpu->hm.s.Event.fPending = false;
8016
8017 /* Make hmR0VmxPreRunGuest() return if we're stepping since we've changed cs:rip. */
8018 if (fStepping)
8019 rcStrict = VINF_EM_DBG_STEPPED;
8020 }
8021 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8022 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8023 return rcStrict;
8024 }
8025 }
8026
8027 /* Validate. */
8028 Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8029 Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
8030
8031 /* Inject. */
8032 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8033 if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8034 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8035 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8036 AssertRCReturn(rc, rc);
8037
8038 /* Update CR2. */
8039 if ( VMX_EXIT_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8040 && uVector == X86_XCPT_PF)
8041 pCtx->cr2 = GCPtrFaultAddress;
8042
8043 Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8044
8045 return VINF_SUCCESS;
8046}
8047
8048
8049/**
8050 * Clears the interrupt-window exiting control in the VMCS and if necessary
8051 * clears the current event in the VMCS as well.
8052 *
8053 * @returns VBox status code.
8054 * @param pVCpu The cross context virtual CPU structure.
8055 *
8056 * @remarks Use this function only to clear events that have not yet been
8057 * delivered to the guest but are injected in the VMCS!
8058 * @remarks No-long-jump zone!!!
8059 */
8060static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8061{
8062 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8063 {
8064 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8065 Log4Func(("Cleared interrupt widow\n"));
8066 }
8067
8068 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8069 {
8070 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8071 Log4Func(("Cleared interrupt widow\n"));
8072 }
8073}
8074
8075
8076/**
8077 * Enters the VT-x session.
8078 *
8079 * @returns VBox status code.
8080 * @param pVCpu The cross context virtual CPU structure.
8081 * @param pHostCpu Pointer to the global CPU info struct.
8082 */
8083VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu, PHMGLOBALCPUINFO pHostCpu)
8084{
8085 AssertPtr(pVCpu);
8086 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8087 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8088 RT_NOREF(pHostCpu);
8089
8090 LogFlowFunc(("pVCpu=%p\n", pVCpu));
8091 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8092 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8093
8094#ifdef VBOX_STRICT
8095 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8096 RTCCUINTREG uHostCR4 = ASMGetCR4();
8097 if (!(uHostCR4 & X86_CR4_VMXE))
8098 {
8099 LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
8100 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8101 }
8102#endif
8103
8104 /*
8105 * Load the VCPU's VMCS as the current (and active) one.
8106 */
8107 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8108 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8109 if (RT_FAILURE(rc))
8110 return rc;
8111
8112 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8113 pVCpu->hm.s.fLeaveDone = false;
8114 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8115
8116 return VINF_SUCCESS;
8117}
8118
8119
8120/**
8121 * The thread-context callback (only on platforms which support it).
8122 *
8123 * @param enmEvent The thread-context event.
8124 * @param pVCpu The cross context virtual CPU structure.
8125 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8126 * @thread EMT(pVCpu)
8127 */
8128VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8129{
8130 NOREF(fGlobalInit);
8131
8132 switch (enmEvent)
8133 {
8134 case RTTHREADCTXEVENT_OUT:
8135 {
8136 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8137 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8138 VMCPU_ASSERT_EMT(pVCpu);
8139
8140 /* No longjmps (logger flushes, locks) in this fragile context. */
8141 VMMRZCallRing3Disable(pVCpu);
8142 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8143
8144 /*
8145 * Restore host-state (FPU, debug etc.)
8146 */
8147 if (!pVCpu->hm.s.fLeaveDone)
8148 {
8149 /*
8150 * Do -not- import the guest-state here as we might already be in the middle of importing
8151 * it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
8152 */
8153 hmR0VmxLeave(pVCpu, false /* fImportState */);
8154 pVCpu->hm.s.fLeaveDone = true;
8155 }
8156
8157 /* Leave HM context, takes care of local init (term). */
8158 int rc = HMR0LeaveCpu(pVCpu);
8159 AssertRC(rc); NOREF(rc);
8160
8161 /* Restore longjmp state. */
8162 VMMRZCallRing3Enable(pVCpu);
8163 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8164 break;
8165 }
8166
8167 case RTTHREADCTXEVENT_IN:
8168 {
8169 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8170 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8171 VMCPU_ASSERT_EMT(pVCpu);
8172
8173 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8174 VMMRZCallRing3Disable(pVCpu);
8175 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8176
8177 /* Initialize the bare minimum state required for HM. This takes care of
8178 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8179 int rc = hmR0EnterCpu(pVCpu);
8180 AssertRC(rc);
8181 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8182 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8183
8184 /* Load the active VMCS as the current one. */
8185 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8186 {
8187 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8188 AssertRC(rc); NOREF(rc);
8189 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8190 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8191 }
8192 pVCpu->hm.s.fLeaveDone = false;
8193
8194 /* Restore longjmp state. */
8195 VMMRZCallRing3Enable(pVCpu);
8196 break;
8197 }
8198
8199 default:
8200 break;
8201 }
8202}
8203
8204
8205/**
8206 * Exports the host state into the VMCS host-state area.
8207 * Sets up the VM-exit MSR-load area.
8208 *
8209 * The CPU state will be loaded from these fields on every successful VM-exit.
8210 *
8211 * @returns VBox status code.
8212 * @param pVCpu The cross context virtual CPU structure.
8213 *
8214 * @remarks No-long-jump zone!!!
8215 */
8216static int hmR0VmxExportHostState(PVMCPU pVCpu)
8217{
8218 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8219
8220 int rc = VINF_SUCCESS;
8221 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8222 {
8223 rc = hmR0VmxExportHostControlRegs();
8224 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8225
8226 rc = hmR0VmxExportHostSegmentRegs(pVCpu);
8227 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8228
8229 rc = hmR0VmxExportHostMsrs(pVCpu);
8230 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8231
8232 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
8233 }
8234 return rc;
8235}
8236
8237
8238/**
8239 * Saves the host state in the VMCS host-state.
8240 *
8241 * @returns VBox status code.
8242 * @param pVCpu The cross context virtual CPU structure.
8243 *
8244 * @remarks No-long-jump zone!!!
8245 */
8246VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
8247{
8248 AssertPtr(pVCpu);
8249 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8250
8251 /*
8252 * Export the host state here while entering HM context.
8253 * When thread-context hooks are used, we might get preempted and have to re-save the host
8254 * state but most of the time we won't be, so do it here before we disable interrupts.
8255 */
8256 return hmR0VmxExportHostState(pVCpu);
8257}
8258
8259
8260/**
8261 * Exports the guest state into the VMCS guest-state area.
8262 *
8263 * The will typically be done before VM-entry when the guest-CPU state and the
8264 * VMCS state may potentially be out of sync.
8265 *
8266 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8267 * VM-entry controls.
8268 * Sets up the appropriate VMX non-root function to execute guest code based on
8269 * the guest CPU mode.
8270 *
8271 * @returns VBox strict status code.
8272 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8273 * without unrestricted guest access and the VMMDev is not presently
8274 * mapped (e.g. EFI32).
8275 *
8276 * @param pVCpu The cross context virtual CPU structure.
8277 *
8278 * @remarks No-long-jump zone!!!
8279 */
8280static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu)
8281{
8282 AssertPtr(pVCpu);
8283 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8284
8285 LogFlowFunc(("pVCpu=%p\n", pVCpu));
8286
8287 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
8288
8289 /* Determine real-on-v86 mode. */
8290 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8291 if ( !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
8292 && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
8293 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8294
8295 /*
8296 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8297 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8298 */
8299 int rc = hmR0VmxSelectVMRunHandler(pVCpu);
8300 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8301
8302 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8303 rc = hmR0VmxExportGuestEntryCtls(pVCpu);
8304 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8305
8306 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8307 rc = hmR0VmxExportGuestExitCtls(pVCpu);
8308 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8309
8310 rc = hmR0VmxExportGuestCR0(pVCpu);
8311 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8312
8313 VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu);
8314 if (rcStrict == VINF_SUCCESS)
8315 { /* likely */ }
8316 else
8317 {
8318 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8319 return rcStrict;
8320 }
8321
8322 rc = hmR0VmxExportGuestSegmentRegs(pVCpu);
8323 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8324
8325 /* This needs to be done after hmR0VmxExportGuestEntryCtls() and hmR0VmxExportGuestExitCtls() as it
8326 may alter controls if we determine we don't have to swap EFER after all. */
8327 rc = hmR0VmxExportGuestMsrs(pVCpu);
8328 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8329
8330 rc = hmR0VmxExportGuestApicTpr(pVCpu);
8331 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8332
8333 rc = hmR0VmxExportGuestXcptIntercepts(pVCpu);
8334 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8335
8336 /* Exporting RFLAGS here is fine, even though RFLAGS.TF might depend on guest debug state which is
8337 not exported here. It is re-evaluated and updated if necessary in hmR0VmxExportSharedState(). */
8338 rc = hmR0VmxExportGuestRip(pVCpu);
8339 rc |= hmR0VmxExportGuestRsp(pVCpu);
8340 rc |= hmR0VmxExportGuestRflags(pVCpu);
8341 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8342
8343 /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
8344 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
8345 | HM_CHANGED_GUEST_CR2
8346 | (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
8347 | HM_CHANGED_GUEST_X87
8348 | HM_CHANGED_GUEST_SSE_AVX
8349 | HM_CHANGED_GUEST_OTHER_XSAVE
8350 | HM_CHANGED_GUEST_XCRx
8351 | HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
8352 | HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
8353 | HM_CHANGED_GUEST_TSC_AUX
8354 | HM_CHANGED_GUEST_OTHER_MSRS
8355 | HM_CHANGED_GUEST_HWVIRT
8356 | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
8357
8358 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
8359 return rc;
8360}
8361
8362
8363/**
8364 * Exports the state shared between the host and guest into the VMCS.
8365 *
8366 * @param pVCpu The cross context virtual CPU structure.
8367 *
8368 * @remarks No-long-jump zone!!!
8369 */
8370static void hmR0VmxExportSharedState(PVMCPU pVCpu)
8371{
8372 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8373 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8374
8375 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
8376 {
8377 int rc = hmR0VmxExportSharedDebugState(pVCpu);
8378 AssertRC(rc);
8379 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
8380
8381 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8382 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
8383 {
8384 rc = hmR0VmxExportGuestRflags(pVCpu);
8385 AssertRC(rc);
8386 }
8387 }
8388
8389 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
8390 {
8391 hmR0VmxLazyLoadGuestMsrs(pVCpu);
8392 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
8393 }
8394
8395 AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
8396 ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8397}
8398
8399
8400/**
8401 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8402 *
8403 * @returns Strict VBox status code (i.e. informational status codes too).
8404 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8405 * without unrestricted guest access and the VMMDev is not presently
8406 * mapped (e.g. EFI32).
8407 *
8408 * @param pVCpu The cross context virtual CPU structure.
8409 *
8410 * @remarks No-long-jump zone!!!
8411 */
8412static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu)
8413{
8414 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8415 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8416 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8417
8418#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8419 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8420#endif
8421
8422 /*
8423 * For many exits it's only RIP that changes and hence try to export it first
8424 * without going through a lot of change flag checks.
8425 */
8426 VBOXSTRICTRC rcStrict;
8427 uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8428 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8429 if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
8430 {
8431 rcStrict = hmR0VmxExportGuestRip(pVCpu);
8432 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8433 { /* likely */}
8434 else
8435 AssertMsgFailedReturn(("hmR0VmxExportGuestRip failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8436 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
8437 }
8438 else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8439 {
8440 rcStrict = hmR0VmxExportGuestState(pVCpu);
8441 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8442 { /* likely */}
8443 else
8444 {
8445 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("hmR0VmxExportGuestState failed! rc=%Rrc\n",
8446 VBOXSTRICTRC_VAL(rcStrict)));
8447 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8448 return rcStrict;
8449 }
8450 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
8451 }
8452 else
8453 rcStrict = VINF_SUCCESS;
8454
8455#ifdef VBOX_STRICT
8456 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8457 fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8458 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8459 AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
8460 ("fCtxChanged=%#RX64\n", fCtxChanged));
8461#endif
8462 return rcStrict;
8463}
8464
8465
8466/**
8467 * Does the preparations before executing guest code in VT-x.
8468 *
8469 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8470 * recompiler/IEM. We must be cautious what we do here regarding committing
8471 * guest-state information into the VMCS assuming we assuredly execute the
8472 * guest in VT-x mode.
8473 *
8474 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8475 * the common-state (TRPM/forceflags), we must undo those changes so that the
8476 * recompiler/IEM can (and should) use them when it resumes guest execution.
8477 * Otherwise such operations must be done when we can no longer exit to ring-3.
8478 *
8479 * @returns Strict VBox status code (i.e. informational status codes too).
8480 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8481 * have been disabled.
8482 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8483 * double-fault into the guest.
8484 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8485 * dispatched directly.
8486 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8487 *
8488 * @param pVCpu The cross context virtual CPU structure.
8489 * @param pVmxTransient Pointer to the VMX transient structure.
8490 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8491 * us ignore some of the reasons for returning to
8492 * ring-3, and return VINF_EM_DBG_STEPPED if event
8493 * dispatching took place.
8494 */
8495static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
8496{
8497 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8498
8499#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_ONLY_IN_IEM
8500 Log2(("hmR0SvmPreRunGuest: Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
8501 return VINF_EM_RESCHEDULE_REM;
8502#endif
8503
8504#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8505 PGMRZDynMapFlushAutoSet(pVCpu);
8506#endif
8507
8508 /* Check force flag actions that might require us to go back to ring-3. */
8509 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
8510 if (rcStrict == VINF_SUCCESS)
8511 { /* FFs doesn't get set all the time. */ }
8512 else
8513 return rcStrict;
8514
8515 /*
8516 * Setup the virtualized-APIC accesses.
8517 *
8518 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8519 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8520 *
8521 * This is the reason we do it here and not in hmR0VmxExportGuestState().
8522 */
8523 PVM pVM = pVCpu->CTX_SUFF(pVM);
8524 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8525 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
8526 && PDMHasApic(pVM))
8527 {
8528 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8529 Assert(u64MsrApicBase);
8530 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8531
8532 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8533
8534 /* Unalias any existing mapping. */
8535 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8536 AssertRCReturn(rc, rc);
8537
8538 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8539 Log4Func(("Mapped HC APIC-access page at %#RGp\n", GCPhysApicBase));
8540 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8541 AssertRCReturn(rc, rc);
8542
8543 /* Update the per-VCPU cache of the APIC base MSR. */
8544 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8545 }
8546
8547 if (TRPMHasTrap(pVCpu))
8548 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8549 uint32_t fIntrState = hmR0VmxEvaluatePendingEvent(pVCpu);
8550
8551 /*
8552 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
8553 * needs to be done with longjmps or interrupts + preemption enabled. Event injection might
8554 * also result in triple-faulting the VM.
8555 */
8556 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, fIntrState, fStepping);
8557 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8558 { /* likely */ }
8559 else
8560 {
8561 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8562 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8563 return rcStrict;
8564 }
8565
8566 /*
8567 * A longjump might result in importing CR3 even for VM-exits that don't necessarily
8568 * import CR3 themselves. We will need to update them here, as even as late as the above
8569 * hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
8570 * the below force flags to be set.
8571 */
8572 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8573 {
8574 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
8575 int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8576 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
8577 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
8578 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8579 }
8580 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8581 {
8582 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8583 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8584 }
8585
8586 /*
8587 * No longjmps to ring-3 from this point on!!!
8588 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8589 * This also disables flushing of the R0-logger instance (if any).
8590 */
8591 VMMRZCallRing3Disable(pVCpu);
8592
8593 /*
8594 * Export the guest state bits.
8595 *
8596 * We cannot perform longjmps while loading the guest state because we do not preserve the
8597 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8598 * CPU migration.
8599 *
8600 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8601 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8602 * Hence, loading of the guest state needs to be done -after- injection of events.
8603 */
8604 rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu);
8605 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8606 { /* likely */ }
8607 else
8608 {
8609 VMMRZCallRing3Enable(pVCpu);
8610 return rcStrict;
8611 }
8612
8613 /*
8614 * We disable interrupts so that we don't miss any interrupts that would flag preemption
8615 * (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
8616 * preemption disabled for a while. Since this is purly to aid the
8617 * RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
8618 * disable interrupt on NT.
8619 *
8620 * We need to check for force-flags that could've possible been altered since we last
8621 * checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
8622 * see @bugref{6398}).
8623 *
8624 * We also check a couple of other force-flags as a last opportunity to get the EMT back
8625 * to ring-3 before executing guest code.
8626 */
8627 pVmxTransient->fEFlags = ASMIntDisableFlags();
8628
8629 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8630 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8631 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8632 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8633 {
8634 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8635 {
8636 pVCpu->hm.s.Event.fPending = false;
8637
8638 /*
8639 * We've injected any pending events. This is really the point of no return (to ring-3).
8640 *
8641 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
8642 * returns from this function, so don't enable them here.
8643 */
8644 return VINF_SUCCESS;
8645 }
8646
8647 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
8648 rcStrict = VINF_EM_RAW_INTERRUPT;
8649 }
8650 else
8651 {
8652 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8653 rcStrict = VINF_EM_RAW_TO_R3;
8654 }
8655
8656 ASMSetFlags(pVmxTransient->fEFlags);
8657 VMMRZCallRing3Enable(pVCpu);
8658
8659 return rcStrict;
8660}
8661
8662
8663/**
8664 * Prepares to run guest code in VT-x and we've committed to doing so. This
8665 * means there is no backing out to ring-3 or anywhere else at this
8666 * point.
8667 *
8668 * @param pVCpu The cross context virtual CPU structure.
8669 * @param pVmxTransient Pointer to the VMX transient structure.
8670 *
8671 * @remarks Called with preemption disabled.
8672 * @remarks No-long-jump zone!!!
8673 */
8674static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
8675{
8676 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8677 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8678 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8679
8680 /*
8681 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8682 */
8683 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8684 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8685
8686 PVM pVM = pVCpu->CTX_SUFF(pVM);
8687 if (!CPUMIsGuestFPUStateActive(pVCpu))
8688 {
8689 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8690 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8691 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT;
8692 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8693 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
8694 }
8695
8696 /*
8697 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8698 */
8699 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8700 && pVCpu->hm.s.vmx.cMsrs > 0)
8701 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8702
8703 /*
8704 * Re-save the host state bits as we may've been preempted (only happens when
8705 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8706 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8707 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8708 * See @bugref{8432}.
8709 */
8710 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8711 {
8712 int rc = hmR0VmxExportHostState(pVCpu);
8713 AssertRC(rc);
8714 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
8715 }
8716 Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
8717
8718 /*
8719 * Export the state shared between host and guest (FPU, debug, lazy MSRs).
8720 */
8721 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
8722 hmR0VmxExportSharedState(pVCpu);
8723 AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8724
8725 /* Store status of the shared guest-host state at the time of VM-entry. */
8726#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8727 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
8728 {
8729 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8730 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8731 }
8732 else
8733#endif
8734 {
8735 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8736 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8737 }
8738
8739 /*
8740 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8741 */
8742 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
8743 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8744
8745 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8746 RTCPUID idCurrentCpu = pCpu->idCpu;
8747 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8748 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8749 {
8750 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8751 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8752 }
8753
8754 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8755 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8756 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8757 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8758
8759 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8760
8761 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8762 to start executing. */
8763
8764 /*
8765 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8766 */
8767 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
8768 {
8769 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
8770 {
8771 bool fMsrUpdated;
8772 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
8773 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8774 &fMsrUpdated);
8775 AssertRC(rc2);
8776 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8777 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8778 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8779 }
8780 else
8781 {
8782 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8783 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8784 }
8785 }
8786
8787 if (pVM->cpum.ro.GuestFeatures.fIbrs)
8788 {
8789 bool fMsrUpdated;
8790 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_OTHER_MSRS);
8791 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
8792 &fMsrUpdated);
8793 AssertRC(rc2);
8794 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8795 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8796 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8797 }
8798
8799#ifdef VBOX_STRICT
8800 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8801 hmR0VmxCheckHostEferMsr(pVCpu);
8802 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8803#endif
8804#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8805 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
8806 {
8807 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
8808 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8809 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8810 }
8811#endif
8812}
8813
8814
8815/**
8816 * Performs some essential restoration of state after running guest code in
8817 * VT-x.
8818 *
8819 * @param pVCpu The cross context virtual CPU structure.
8820 * @param pVmxTransient Pointer to the VMX transient structure.
8821 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8822 *
8823 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8824 *
8825 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8826 * unconditionally when it is safe to do so.
8827 */
8828static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8829{
8830 uint64_t const uHostTsc = ASMReadTSC();
8831 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8832
8833 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8834 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8835 pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8836 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8837 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8838 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8839
8840 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
8841 TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TscOffset);
8842
8843 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
8844 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8845 Assert(!ASMIntAreEnabled());
8846 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8847
8848#if HC_ARCH_BITS == 64
8849 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8850#endif
8851#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8852 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8853 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8854 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8855#else
8856 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8857#endif
8858#ifdef VBOX_STRICT
8859 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8860#endif
8861 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8862
8863 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8864 uint32_t uExitReason;
8865 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8866 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8867 AssertRC(rc);
8868 pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
8869 pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
8870
8871 if (rcVMRun == VINF_SUCCESS)
8872 {
8873 /*
8874 * Update the VM-exit history array here even if the VM-entry failed due to:
8875 * - Invalid guest state.
8876 * - MSR loading.
8877 * - Machine-check event.
8878 *
8879 * In any of the above cases we will still have a "valid" VM-exit reason
8880 * despite @a fVMEntryFailed being false.
8881 *
8882 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8883 *
8884 * Note! We don't have CS or RIP at this point. Will probably address that later
8885 * by amending the history entry added here.
8886 */
8887 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
8888 UINT64_MAX, uHostTsc);
8889
8890 if (!pVmxTransient->fVMEntryFailed)
8891 {
8892 VMMRZCallRing3Enable(pVCpu);
8893
8894 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8895 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8896
8897#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8898 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
8899 AssertRC(rc);
8900#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8901 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_RFLAGS);
8902 AssertRC(rc);
8903#else
8904 /*
8905 * Import the guest-interruptibility state always as we need it while evaluating
8906 * injecting events on re-entry.
8907 *
8908 * We don't import CR0 (when Unrestricted guest execution is unavailable) despite
8909 * checking for real-mode while exporting the state because all bits that cause
8910 * mode changes wrt CR0 are intercepted.
8911 */
8912 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
8913 AssertRC(rc);
8914#endif
8915
8916 /*
8917 * Sync the TPR shadow with our APIC state.
8918 */
8919 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
8920 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8921 {
8922 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8923 AssertRC(rc);
8924 ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8925 }
8926
8927 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8928 return;
8929 }
8930 }
8931 else
8932 Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
8933
8934 VMMRZCallRing3Enable(pVCpu);
8935}
8936
8937
8938/**
8939 * Runs the guest code using VT-x the normal way.
8940 *
8941 * @returns VBox status code.
8942 * @param pVCpu The cross context virtual CPU structure.
8943 *
8944 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8945 */
8946static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu)
8947{
8948 VMXTRANSIENT VmxTransient;
8949 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8950 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8951 uint32_t cLoops = 0;
8952
8953 for (;; cLoops++)
8954 {
8955 Assert(!HMR0SuspendPending());
8956 HMVMX_ASSERT_CPU_SAFE(pVCpu);
8957
8958 /* Preparatory work for running guest code, this may force us to return
8959 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8960 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8961 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
8962 if (rcStrict != VINF_SUCCESS)
8963 break;
8964
8965 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
8966 int rcRun = hmR0VmxRunGuest(pVCpu);
8967
8968 /* Restore any residual host-state and save any bits shared between host
8969 and guest into the guest-CPU state. Re-enables interrupts! */
8970 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
8971
8972 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8973 if (RT_SUCCESS(rcRun))
8974 { /* very likely */ }
8975 else
8976 {
8977 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
8978 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
8979 return rcRun;
8980 }
8981
8982 /* Profile the VM-exit. */
8983 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8984 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8985 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8986 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
8987 HMVMX_START_EXIT_DISPATCH_PROF();
8988
8989 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
8990
8991 /* Handle the VM-exit. */
8992#ifdef HMVMX_USE_FUNCTION_TABLE
8993 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
8994#else
8995 rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient, VmxTransient.uExitReason);
8996#endif
8997 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
8998 if (rcStrict == VINF_SUCCESS)
8999 {
9000 if (cLoops <= pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
9001 continue; /* likely */
9002 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9003 rcStrict = VINF_EM_RAW_INTERRUPT;
9004 }
9005 break;
9006 }
9007
9008 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9009 return rcStrict;
9010}
9011
9012
9013
9014/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
9015 * probes.
9016 *
9017 * The following few functions and associated structure contains the bloat
9018 * necessary for providing detailed debug events and dtrace probes as well as
9019 * reliable host side single stepping. This works on the principle of
9020 * "subclassing" the normal execution loop and workers. We replace the loop
9021 * method completely and override selected helpers to add necessary adjustments
9022 * to their core operation.
9023 *
9024 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
9025 * any performance for debug and analysis features.
9026 *
9027 * @{
9028 */
9029
9030/**
9031 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
9032 * the debug run loop.
9033 */
9034typedef struct VMXRUNDBGSTATE
9035{
9036 /** The RIP we started executing at. This is for detecting that we stepped. */
9037 uint64_t uRipStart;
9038 /** The CS we started executing with. */
9039 uint16_t uCsStart;
9040
9041 /** Whether we've actually modified the 1st execution control field. */
9042 bool fModifiedProcCtls : 1;
9043 /** Whether we've actually modified the 2nd execution control field. */
9044 bool fModifiedProcCtls2 : 1;
9045 /** Whether we've actually modified the exception bitmap. */
9046 bool fModifiedXcptBitmap : 1;
9047
9048 /** We desire the modified the CR0 mask to be cleared. */
9049 bool fClearCr0Mask : 1;
9050 /** We desire the modified the CR4 mask to be cleared. */
9051 bool fClearCr4Mask : 1;
9052 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9053 uint32_t fCpe1Extra;
9054 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9055 uint32_t fCpe1Unwanted;
9056 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9057 uint32_t fCpe2Extra;
9058 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9059 uint32_t bmXcptExtra;
9060 /** The sequence number of the Dtrace provider settings the state was
9061 * configured against. */
9062 uint32_t uDtraceSettingsSeqNo;
9063 /** VM-exits to check (one bit per VM-exit). */
9064 uint32_t bmExitsToCheck[3];
9065
9066 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9067 uint32_t fProcCtlsInitial;
9068 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9069 uint32_t fProcCtls2Initial;
9070 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9071 uint32_t bmXcptInitial;
9072} VMXRUNDBGSTATE;
9073AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9074typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9075
9076
9077/**
9078 * Initializes the VMXRUNDBGSTATE structure.
9079 *
9080 * @param pVCpu The cross context virtual CPU structure of the
9081 * calling EMT.
9082 * @param pDbgState The structure to initialize.
9083 */
9084static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9085{
9086 pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
9087 pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
9088
9089 pDbgState->fModifiedProcCtls = false;
9090 pDbgState->fModifiedProcCtls2 = false;
9091 pDbgState->fModifiedXcptBitmap = false;
9092 pDbgState->fClearCr0Mask = false;
9093 pDbgState->fClearCr4Mask = false;
9094 pDbgState->fCpe1Extra = 0;
9095 pDbgState->fCpe1Unwanted = 0;
9096 pDbgState->fCpe2Extra = 0;
9097 pDbgState->bmXcptExtra = 0;
9098 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9099 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9100 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9101}
9102
9103
9104/**
9105 * Updates the VMSC fields with changes requested by @a pDbgState.
9106 *
9107 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9108 * immediately before executing guest code, i.e. when interrupts are disabled.
9109 * We don't check status codes here as we cannot easily assert or return in the
9110 * latter case.
9111 *
9112 * @param pVCpu The cross context virtual CPU structure.
9113 * @param pDbgState The debug state.
9114 */
9115static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9116{
9117 /*
9118 * Ensure desired flags in VMCS control fields are set.
9119 * (Ignoring write failure here, as we're committed and it's just debug extras.)
9120 *
9121 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9122 * there should be no stale data in pCtx at this point.
9123 */
9124 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9125 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9126 {
9127 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
9128 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9129 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9130 Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9131 pDbgState->fModifiedProcCtls = true;
9132 }
9133
9134 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9135 {
9136 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
9137 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9138 Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9139 pDbgState->fModifiedProcCtls2 = true;
9140 }
9141
9142 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9143 {
9144 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
9145 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9146 Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9147 pDbgState->fModifiedXcptBitmap = true;
9148 }
9149
9150 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32Cr0Mask != 0)
9151 {
9152 pVCpu->hm.s.vmx.u32Cr0Mask = 0;
9153 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9154 Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9155 }
9156
9157 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32Cr4Mask != 0)
9158 {
9159 pVCpu->hm.s.vmx.u32Cr4Mask = 0;
9160 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9161 Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9162 }
9163}
9164
9165
9166/**
9167 * Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
9168 * re-entry next time around.
9169 *
9170 * @returns Strict VBox status code (i.e. informational status codes too).
9171 * @param pVCpu The cross context virtual CPU structure.
9172 * @param pDbgState The debug state.
9173 * @param rcStrict The return code from executing the guest using single
9174 * stepping.
9175 */
9176static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9177{
9178 /*
9179 * Restore VM-exit control settings as we may not reenter this function the
9180 * next time around.
9181 */
9182 /* We reload the initial value, trigger what we can of recalculations the
9183 next time around. From the looks of things, that's all that's required atm. */
9184 if (pDbgState->fModifiedProcCtls)
9185 {
9186 if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9187 pDbgState->fProcCtlsInitial |= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9188 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9189 AssertRCReturn(rc2, rc2);
9190 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9191 }
9192
9193 /* We're currently the only ones messing with this one, so just restore the
9194 cached value and reload the field. */
9195 if ( pDbgState->fModifiedProcCtls2
9196 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9197 {
9198 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9199 AssertRCReturn(rc2, rc2);
9200 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9201 }
9202
9203 /* If we've modified the exception bitmap, we restore it and trigger
9204 reloading and partial recalculation the next time around. */
9205 if (pDbgState->fModifiedXcptBitmap)
9206 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9207
9208 return rcStrict;
9209}
9210
9211
9212/**
9213 * Configures VM-exit controls for current DBGF and DTrace settings.
9214 *
9215 * This updates @a pDbgState and the VMCS execution control fields to reflect
9216 * the necessary VM-exits demanded by DBGF and DTrace.
9217 *
9218 * @param pVCpu The cross context virtual CPU structure.
9219 * @param pDbgState The debug state.
9220 * @param pVmxTransient Pointer to the VMX transient structure. May update
9221 * fUpdateTscOffsettingAndPreemptTimer.
9222 */
9223static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9224{
9225 /*
9226 * Take down the dtrace serial number so we can spot changes.
9227 */
9228 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9229 ASMCompilerBarrier();
9230
9231 /*
9232 * We'll rebuild most of the middle block of data members (holding the
9233 * current settings) as we go along here, so start by clearing it all.
9234 */
9235 pDbgState->bmXcptExtra = 0;
9236 pDbgState->fCpe1Extra = 0;
9237 pDbgState->fCpe1Unwanted = 0;
9238 pDbgState->fCpe2Extra = 0;
9239 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9240 pDbgState->bmExitsToCheck[i] = 0;
9241
9242 /*
9243 * Software interrupts (INT XXh) - no idea how to trigger these...
9244 */
9245 PVM pVM = pVCpu->CTX_SUFF(pVM);
9246 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9247 || VBOXVMM_INT_SOFTWARE_ENABLED())
9248 {
9249 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9250 }
9251
9252 /*
9253 * INT3 breakpoints - triggered by #BP exceptions.
9254 */
9255 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9256 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9257
9258 /*
9259 * Exception bitmap and XCPT events+probes.
9260 */
9261 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9262 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9263 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9264
9265 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9266 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9267 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9268 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9269 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9270 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9271 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9272 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9273 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9274 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9275 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9276 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9277 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9278 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9279 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9280 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9281 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9282 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9283
9284 if (pDbgState->bmXcptExtra)
9285 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9286
9287 /*
9288 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9289 *
9290 * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
9291 * So, when adding/changing/removing please don't forget to update it.
9292 *
9293 * Some of the macros are picking up local variables to save horizontal space,
9294 * (being able to see it in a table is the lesser evil here).
9295 */
9296#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9297 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9298 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9299#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9300 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9301 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9302 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9303 } else do { } while (0)
9304#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9305 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9306 { \
9307 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9308 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9309 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9310 } else do { } while (0)
9311#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9312 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9313 { \
9314 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9315 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9316 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9317 } else do { } while (0)
9318#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9319 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9320 { \
9321 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9322 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9323 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9324 } else do { } while (0)
9325
9326 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9327 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9328 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9329 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9330 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9331
9332 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9333 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9334 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9335 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9336 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
9337 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9338 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9339 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9340 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
9341 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9342 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
9343 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9344 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
9345 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9346 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9347 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9348 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9349 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9350 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9351 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9352 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9353 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9354 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9355 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9356 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9357 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9358 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9359 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9360 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9361 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9362 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9363 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9364 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9365 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9366 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9367 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9368
9369 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9370 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9371 {
9372 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_APIC_TPR);
9373 AssertRC(rc);
9374
9375#if 0 /** @todo fix me */
9376 pDbgState->fClearCr0Mask = true;
9377 pDbgState->fClearCr4Mask = true;
9378#endif
9379 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9380 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_STORE_EXIT | VMX_PROC_CTLS_CR8_STORE_EXIT;
9381 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9382 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_LOAD_EXIT | VMX_PROC_CTLS_CR8_LOAD_EXIT;
9383 pDbgState->fCpe1Unwanted |= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
9384 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9385 require clearing here and in the loop if we start using it. */
9386 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9387 }
9388 else
9389 {
9390 if (pDbgState->fClearCr0Mask)
9391 {
9392 pDbgState->fClearCr0Mask = false;
9393 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
9394 }
9395 if (pDbgState->fClearCr4Mask)
9396 {
9397 pDbgState->fClearCr4Mask = false;
9398 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
9399 }
9400 }
9401 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9402 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9403
9404 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9405 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9406 {
9407 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9408 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9409 }
9410 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9411 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9412
9413 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
9414 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9415 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
9416 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9417 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
9418 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9419 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
9420 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9421#if 0 /** @todo too slow, fix handler. */
9422 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
9423#endif
9424 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9425
9426 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9427 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9428 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9429 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9430 {
9431 pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
9432 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
9433 }
9434 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9435 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9436 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9437 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9438
9439 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9440 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9441 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9442 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9443 {
9444 pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
9445 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
9446 }
9447 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
9448 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
9449 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
9450 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
9451
9452 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9453 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9454 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
9455 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9456 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9457 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9458 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
9459 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9460 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9461 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9462 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
9463 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9464 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
9465 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9466 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9467 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9468 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
9469 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9470 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9471 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9472 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9473 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9474
9475#undef IS_EITHER_ENABLED
9476#undef SET_ONLY_XBM_IF_EITHER_EN
9477#undef SET_CPE1_XBM_IF_EITHER_EN
9478#undef SET_CPEU_XBM_IF_EITHER_EN
9479#undef SET_CPE2_XBM_IF_EITHER_EN
9480
9481 /*
9482 * Sanitize the control stuff.
9483 */
9484 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
9485 if (pDbgState->fCpe2Extra)
9486 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
9487 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
9488 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0;
9489 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
9490 {
9491 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9492 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9493 }
9494
9495 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9496 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9497 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9498 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9499}
9500
9501
9502/**
9503 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9504 * appropriate.
9505 *
9506 * The caller has checked the VM-exit against the
9507 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9508 * already, so we don't have to do that either.
9509 *
9510 * @returns Strict VBox status code (i.e. informational status codes too).
9511 * @param pVCpu The cross context virtual CPU structure.
9512 * @param pVmxTransient Pointer to the VMX-transient structure.
9513 * @param uExitReason The VM-exit reason.
9514 *
9515 * @remarks The name of this function is displayed by dtrace, so keep it short
9516 * and to the point. No longer than 33 chars long, please.
9517 */
9518static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9519{
9520 /*
9521 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9522 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9523 *
9524 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9525 * does. Must add/change/remove both places. Same ordering, please.
9526 *
9527 * Added/removed events must also be reflected in the next section
9528 * where we dispatch dtrace events.
9529 */
9530 bool fDtrace1 = false;
9531 bool fDtrace2 = false;
9532 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9533 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9534 uint32_t uEventArg = 0;
9535#define SET_EXIT(a_EventSubName) \
9536 do { \
9537 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9538 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9539 } while (0)
9540#define SET_BOTH(a_EventSubName) \
9541 do { \
9542 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9543 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9544 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9545 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9546 } while (0)
9547 switch (uExitReason)
9548 {
9549 case VMX_EXIT_MTF:
9550 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9551
9552 case VMX_EXIT_XCPT_OR_NMI:
9553 {
9554 uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9555 switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
9556 {
9557 case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
9558 case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
9559 case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
9560 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9561 {
9562 if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
9563 {
9564 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9565 uEventArg = pVmxTransient->uExitIntErrorCode;
9566 }
9567 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9568 switch (enmEvent1)
9569 {
9570 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9571 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9572 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9573 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9574 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9575 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9576 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9577 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9578 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9579 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9580 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9581 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9582 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9583 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9584 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9585 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9586 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9587 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9588 default: break;
9589 }
9590 }
9591 else
9592 AssertFailed();
9593 break;
9594
9595 case VMX_EXIT_INT_INFO_TYPE_SW_INT:
9596 uEventArg = idxVector;
9597 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9598 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9599 break;
9600 }
9601 break;
9602 }
9603
9604 case VMX_EXIT_TRIPLE_FAULT:
9605 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9606 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9607 break;
9608 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9609 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9610 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9611 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9612 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9613
9614 /* Instruction specific VM-exits: */
9615 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9616 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9617 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9618 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9619 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9620 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9621 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9622 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9623 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9624 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9625 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9626 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9627 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9628 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9629 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9630 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9631 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9632 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9633 case VMX_EXIT_MOV_CRX:
9634 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9635 if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
9636 SET_BOTH(CRX_READ);
9637 else
9638 SET_BOTH(CRX_WRITE);
9639 uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
9640 break;
9641 case VMX_EXIT_MOV_DRX:
9642 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9643 if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
9644 == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
9645 SET_BOTH(DRX_READ);
9646 else
9647 SET_BOTH(DRX_WRITE);
9648 uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
9649 break;
9650 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9651 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9652 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9653 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9654 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9655 case VMX_EXIT_GDTR_IDTR_ACCESS:
9656 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9657 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
9658 {
9659 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9660 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9661 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9662 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9663 }
9664 break;
9665
9666 case VMX_EXIT_LDTR_TR_ACCESS:
9667 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9668 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
9669 {
9670 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9671 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9672 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9673 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9674 }
9675 break;
9676
9677 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9678 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9679 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9680 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9681 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9682 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9683 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9684 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9685 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9686 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9687 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9688
9689 /* Events that aren't relevant at this point. */
9690 case VMX_EXIT_EXT_INT:
9691 case VMX_EXIT_INT_WINDOW:
9692 case VMX_EXIT_NMI_WINDOW:
9693 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9694 case VMX_EXIT_PREEMPT_TIMER:
9695 case VMX_EXIT_IO_INSTR:
9696 break;
9697
9698 /* Errors and unexpected events. */
9699 case VMX_EXIT_INIT_SIGNAL:
9700 case VMX_EXIT_SIPI:
9701 case VMX_EXIT_IO_SMI:
9702 case VMX_EXIT_SMI:
9703 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9704 case VMX_EXIT_ERR_MSR_LOAD:
9705 case VMX_EXIT_ERR_MACHINE_CHECK:
9706 break;
9707
9708 default:
9709 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9710 break;
9711 }
9712#undef SET_BOTH
9713#undef SET_EXIT
9714
9715 /*
9716 * Dtrace tracepoints go first. We do them here at once so we don't
9717 * have to copy the guest state saving and stuff a few dozen times.
9718 * Down side is that we've got to repeat the switch, though this time
9719 * we use enmEvent since the probes are a subset of what DBGF does.
9720 */
9721 if (fDtrace1 || fDtrace2)
9722 {
9723 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9724 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9725 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9726 switch (enmEvent1)
9727 {
9728 /** @todo consider which extra parameters would be helpful for each probe. */
9729 case DBGFEVENT_END: break;
9730 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
9731 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
9732 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
9733 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
9734 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
9735 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
9736 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
9737 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
9738 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
9739 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
9740 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
9741 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
9742 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
9743 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
9744 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
9745 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
9746 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
9747 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
9748 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9749 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9750 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
9751 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
9752 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
9753 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
9754 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
9755 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
9756 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
9757 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9758 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9759 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9760 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9761 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9762 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
9763 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9764 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
9765 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
9766 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
9767 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
9768 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
9769 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
9770 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
9771 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
9772 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
9773 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
9774 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
9775 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
9776 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
9777 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
9778 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
9779 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
9780 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
9781 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
9782 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
9783 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
9784 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
9785 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
9786 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
9787 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
9788 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
9789 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
9790 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
9791 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
9792 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
9793 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
9794 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
9795 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
9796 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9797 }
9798 switch (enmEvent2)
9799 {
9800 /** @todo consider which extra parameters would be helpful for each probe. */
9801 case DBGFEVENT_END: break;
9802 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
9803 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9804 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
9805 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
9806 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
9807 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
9808 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
9809 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
9810 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
9811 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9812 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9813 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9814 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9815 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9816 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
9817 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9818 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
9819 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
9820 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
9821 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
9822 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
9823 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
9824 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
9825 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
9826 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
9827 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
9828 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
9829 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
9830 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
9831 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
9832 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
9833 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
9834 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
9835 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
9836 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
9837 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
9838 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
9839 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
9840 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
9841 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
9842 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
9843 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
9844 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
9845 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
9846 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
9847 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
9848 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
9849 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
9850 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
9851 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
9852 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
9853 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
9854 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9855 }
9856 }
9857
9858 /*
9859 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9860 * the DBGF call will do a full check).
9861 *
9862 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9863 * Note! If we have to events, we prioritize the first, i.e. the instruction
9864 * one, in order to avoid event nesting.
9865 */
9866 PVM pVM = pVCpu->CTX_SUFF(pVM);
9867 if ( enmEvent1 != DBGFEVENT_END
9868 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9869 {
9870 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
9871 if (rcStrict != VINF_SUCCESS)
9872 return rcStrict;
9873 }
9874 else if ( enmEvent2 != DBGFEVENT_END
9875 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9876 {
9877 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
9878 if (rcStrict != VINF_SUCCESS)
9879 return rcStrict;
9880 }
9881
9882 return VINF_SUCCESS;
9883}
9884
9885
9886/**
9887 * Single-stepping VM-exit filtering.
9888 *
9889 * This is preprocessing the VM-exits and deciding whether we've gotten far
9890 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9891 * handling is performed.
9892 *
9893 * @returns Strict VBox status code (i.e. informational status codes too).
9894 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9895 * @param pVmxTransient Pointer to the VMX-transient structure.
9896 * @param pDbgState The debug state.
9897 */
9898DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
9899{
9900 /*
9901 * Expensive (saves context) generic dtrace VM-exit probe.
9902 */
9903 uint32_t const uExitReason = pVmxTransient->uExitReason;
9904 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9905 { /* more likely */ }
9906 else
9907 {
9908 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9909 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9910 AssertRC(rc);
9911 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
9912 }
9913
9914 /*
9915 * Check for host NMI, just to get that out of the way.
9916 */
9917 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9918 { /* normally likely */ }
9919 else
9920 {
9921 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9922 AssertRCReturn(rc2, rc2);
9923 uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
9924 if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
9925 return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
9926 }
9927
9928 /*
9929 * Check for single stepping event if we're stepping.
9930 */
9931 if (pVCpu->hm.s.fSingleInstruction)
9932 {
9933 switch (uExitReason)
9934 {
9935 case VMX_EXIT_MTF:
9936 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9937
9938 /* Various events: */
9939 case VMX_EXIT_XCPT_OR_NMI:
9940 case VMX_EXIT_EXT_INT:
9941 case VMX_EXIT_TRIPLE_FAULT:
9942 case VMX_EXIT_INT_WINDOW:
9943 case VMX_EXIT_NMI_WINDOW:
9944 case VMX_EXIT_TASK_SWITCH:
9945 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9946 case VMX_EXIT_APIC_ACCESS:
9947 case VMX_EXIT_EPT_VIOLATION:
9948 case VMX_EXIT_EPT_MISCONFIG:
9949 case VMX_EXIT_PREEMPT_TIMER:
9950
9951 /* Instruction specific VM-exits: */
9952 case VMX_EXIT_CPUID:
9953 case VMX_EXIT_GETSEC:
9954 case VMX_EXIT_HLT:
9955 case VMX_EXIT_INVD:
9956 case VMX_EXIT_INVLPG:
9957 case VMX_EXIT_RDPMC:
9958 case VMX_EXIT_RDTSC:
9959 case VMX_EXIT_RSM:
9960 case VMX_EXIT_VMCALL:
9961 case VMX_EXIT_VMCLEAR:
9962 case VMX_EXIT_VMLAUNCH:
9963 case VMX_EXIT_VMPTRLD:
9964 case VMX_EXIT_VMPTRST:
9965 case VMX_EXIT_VMREAD:
9966 case VMX_EXIT_VMRESUME:
9967 case VMX_EXIT_VMWRITE:
9968 case VMX_EXIT_VMXOFF:
9969 case VMX_EXIT_VMXON:
9970 case VMX_EXIT_MOV_CRX:
9971 case VMX_EXIT_MOV_DRX:
9972 case VMX_EXIT_IO_INSTR:
9973 case VMX_EXIT_RDMSR:
9974 case VMX_EXIT_WRMSR:
9975 case VMX_EXIT_MWAIT:
9976 case VMX_EXIT_MONITOR:
9977 case VMX_EXIT_PAUSE:
9978 case VMX_EXIT_GDTR_IDTR_ACCESS:
9979 case VMX_EXIT_LDTR_TR_ACCESS:
9980 case VMX_EXIT_INVEPT:
9981 case VMX_EXIT_RDTSCP:
9982 case VMX_EXIT_INVVPID:
9983 case VMX_EXIT_WBINVD:
9984 case VMX_EXIT_XSETBV:
9985 case VMX_EXIT_RDRAND:
9986 case VMX_EXIT_INVPCID:
9987 case VMX_EXIT_VMFUNC:
9988 case VMX_EXIT_RDSEED:
9989 case VMX_EXIT_XSAVES:
9990 case VMX_EXIT_XRSTORS:
9991 {
9992 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
9993 AssertRCReturn(rc, rc);
9994 if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
9995 || pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
9996 return VINF_EM_DBG_STEPPED;
9997 break;
9998 }
9999
10000 /* Errors and unexpected events: */
10001 case VMX_EXIT_INIT_SIGNAL:
10002 case VMX_EXIT_SIPI:
10003 case VMX_EXIT_IO_SMI:
10004 case VMX_EXIT_SMI:
10005 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10006 case VMX_EXIT_ERR_MSR_LOAD:
10007 case VMX_EXIT_ERR_MACHINE_CHECK:
10008 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
10009 break;
10010
10011 default:
10012 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10013 break;
10014 }
10015 }
10016
10017 /*
10018 * Check for debugger event breakpoints and dtrace probes.
10019 */
10020 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
10021 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
10022 {
10023 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
10024 if (rcStrict != VINF_SUCCESS)
10025 return rcStrict;
10026 }
10027
10028 /*
10029 * Normal processing.
10030 */
10031#ifdef HMVMX_USE_FUNCTION_TABLE
10032 return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
10033#else
10034 return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
10035#endif
10036}
10037
10038
10039/**
10040 * Single steps guest code using VT-x.
10041 *
10042 * @returns Strict VBox status code (i.e. informational status codes too).
10043 * @param pVCpu The cross context virtual CPU structure.
10044 *
10045 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10046 */
10047static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu)
10048{
10049 VMXTRANSIENT VmxTransient;
10050 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10051
10052 /* Set HMCPU indicators. */
10053 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10054 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
10055 pVCpu->hm.s.fDebugWantRdTscExit = false;
10056 pVCpu->hm.s.fUsingDebugLoop = true;
10057
10058 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10059 VMXRUNDBGSTATE DbgState;
10060 hmR0VmxRunDebugStateInit(pVCpu, &DbgState);
10061 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
10062
10063 /*
10064 * The loop.
10065 */
10066 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10067 for (uint32_t cLoops = 0; ; cLoops++)
10068 {
10069 Assert(!HMR0SuspendPending());
10070 HMVMX_ASSERT_CPU_SAFE(pVCpu);
10071 bool fStepping = pVCpu->hm.s.fSingleInstruction;
10072
10073 /*
10074 * Preparatory work for running guest code, this may force us to return
10075 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10076 */
10077 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10078 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10079 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
10080 if (rcStrict != VINF_SUCCESS)
10081 break;
10082
10083 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10084 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10085
10086 /*
10087 * Now we can run the guest code.
10088 */
10089 int rcRun = hmR0VmxRunGuest(pVCpu);
10090
10091 /*
10092 * Restore any residual host-state and save any bits shared between host
10093 * and guest into the guest-CPU state. Re-enables interrupts!
10094 */
10095 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10096
10097 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10098 if (RT_SUCCESS(rcRun))
10099 { /* very likely */ }
10100 else
10101 {
10102 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10103 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10104 return rcRun;
10105 }
10106
10107 /* Profile the VM-exit. */
10108 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10109 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10110 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10111 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10112 HMVMX_START_EXIT_DISPATCH_PROF();
10113
10114 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10115
10116 /*
10117 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10118 */
10119 rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
10120 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10121 if (rcStrict != VINF_SUCCESS)
10122 break;
10123 if (cLoops > pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
10124 {
10125 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10126 rcStrict = VINF_EM_RAW_INTERRUPT;
10127 break;
10128 }
10129
10130 /*
10131 * Stepping: Did the RIP change, if so, consider it a single step.
10132 * Otherwise, make sure one of the TFs gets set.
10133 */
10134 if (fStepping)
10135 {
10136 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
10137 AssertRC(rc);
10138 if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
10139 || pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
10140 {
10141 rcStrict = VINF_EM_DBG_STEPPED;
10142 break;
10143 }
10144 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
10145 }
10146
10147 /*
10148 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
10149 */
10150 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10151 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
10152 }
10153
10154 /*
10155 * Clear the X86_EFL_TF if necessary.
10156 */
10157 if (pVCpu->hm.s.fClearTrapFlag)
10158 {
10159 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
10160 AssertRC(rc);
10161 pVCpu->hm.s.fClearTrapFlag = false;
10162 pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
10163 }
10164 /** @todo there seems to be issues with the resume flag when the monitor trap
10165 * flag is pending without being used. Seen early in bios init when
10166 * accessing APIC page in protected mode. */
10167
10168 /*
10169 * Restore VM-exit control settings as we may not reenter this function the
10170 * next time around.
10171 */
10172 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10173
10174 /* Restore HMCPU indicators. */
10175 pVCpu->hm.s.fUsingDebugLoop = false;
10176 pVCpu->hm.s.fDebugWantRdTscExit = false;
10177 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10178
10179 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10180 return rcStrict;
10181}
10182
10183
10184/** @} */
10185
10186
10187/**
10188 * Checks if any expensive dtrace probes are enabled and we should go to the
10189 * debug loop.
10190 *
10191 * @returns true if we should use debug loop, false if not.
10192 */
10193static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10194{
10195 /* It's probably faster to OR the raw 32-bit counter variables together.
10196 Since the variables are in an array and the probes are next to one
10197 another (more or less), we have good locality. So, better read
10198 eight-nine cache lines ever time and only have one conditional, than
10199 128+ conditionals, right? */
10200 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10201 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10202 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10203 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10204 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10205 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10206 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10207 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10208 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10209 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10210 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10211 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10212 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10213 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10214 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10215 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10216 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10217 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10218 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10219 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10220 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10221 ) != 0
10222 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10223 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10224 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10225 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10226 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10227 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10228 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10229 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10230 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10231 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10232 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10233 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10234 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10235 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10236 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10237 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10238 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10239 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10240 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10241 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10242 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10243 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10244 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10245 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10246 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10247 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10248 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10249 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10250 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10251 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10252 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10253 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10254 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10255 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10256 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10257 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10258 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10259 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10260 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10261 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10262 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10263 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10264 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10265 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10266 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10267 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10268 ) != 0
10269 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10270 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10271 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10272 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10273 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10274 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10275 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10276 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10277 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10278 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10279 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10280 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10281 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10282 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10283 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10284 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10285 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10286 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10287 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10288 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10289 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10290 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10291 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10292 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10293 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10294 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10295 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10296 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10297 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10298 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10299 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10300 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10301 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10302 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10303 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10304 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10305 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10306 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10307 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10308 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10309 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10310 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10311 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10312 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10313 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10314 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10315 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10316 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10317 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10318 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10319 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10320 ) != 0;
10321}
10322
10323
10324/**
10325 * Runs the guest code using VT-x.
10326 *
10327 * @returns Strict VBox status code (i.e. informational status codes too).
10328 * @param pVCpu The cross context virtual CPU structure.
10329 */
10330VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
10331{
10332 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10333 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10334 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10335 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
10336
10337 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10338
10339 VBOXSTRICTRC rcStrict;
10340 if ( !pVCpu->hm.s.fUseDebugLoop
10341 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10342 && !DBGFIsStepping(pVCpu)
10343 && !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
10344 rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu);
10345 else
10346 rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu);
10347
10348 if (rcStrict == VERR_EM_INTERPRETER)
10349 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10350 else if (rcStrict == VINF_EM_RESET)
10351 rcStrict = VINF_EM_TRIPLE_FAULT;
10352
10353 int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
10354 if (RT_FAILURE(rc2))
10355 {
10356 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10357 rcStrict = rc2;
10358 }
10359 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10360 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10361 return rcStrict;
10362}
10363
10364
10365#ifndef HMVMX_USE_FUNCTION_TABLE
10366DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10367{
10368#ifdef DEBUG_ramshankar
10369#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
10370 do { \
10371 if (a_fSave != 0) \
10372 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
10373 VBOXSTRICTRC rcStrict = a_CallExpr; \
10374 if (a_fSave != 0) \
10375 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
10376 return rcStrict; \
10377 } while (0)
10378#else
10379# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
10380#endif
10381 switch (rcReason)
10382 {
10383 case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
10384 case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
10385 case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
10386 case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
10387 case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
10388 case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
10389 case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
10390 case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
10391 case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
10392 case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
10393 case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
10394 case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
10395 case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
10396 case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
10397 case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
10398 case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
10399 case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
10400 case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
10401 case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
10402 case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
10403 case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
10404 case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
10405 case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
10406 case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pVmxTransient));
10407 case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
10408 case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
10409 case VMX_EXIT_GDTR_IDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10410 case VMX_EXIT_LDTR_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10411 case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
10412 case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
10413 case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
10414 case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
10415 case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
10416 case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
10417
10418 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
10419 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
10420 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pVmxTransient);
10421 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pVmxTransient);
10422 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pVmxTransient);
10423 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pVmxTransient);
10424 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pVmxTransient);
10425 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
10426 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pVmxTransient);
10427
10428 case VMX_EXIT_VMCLEAR:
10429 case VMX_EXIT_VMLAUNCH:
10430 case VMX_EXIT_VMPTRLD:
10431 case VMX_EXIT_VMPTRST:
10432 case VMX_EXIT_VMREAD:
10433 case VMX_EXIT_VMRESUME:
10434 case VMX_EXIT_VMWRITE:
10435 case VMX_EXIT_VMXOFF:
10436 case VMX_EXIT_VMXON:
10437 case VMX_EXIT_INVEPT:
10438 case VMX_EXIT_INVVPID:
10439 case VMX_EXIT_VMFUNC:
10440 case VMX_EXIT_XSAVES:
10441 case VMX_EXIT_XRSTORS:
10442 return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
10443
10444 case VMX_EXIT_ENCLS:
10445 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10446 case VMX_EXIT_PML_FULL:
10447 default:
10448 return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
10449 }
10450#undef VMEXIT_CALL_RET
10451}
10452#endif /* !HMVMX_USE_FUNCTION_TABLE */
10453
10454
10455#ifdef VBOX_STRICT
10456/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10457# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10458 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10459
10460# define HMVMX_ASSERT_PREEMPT_CPUID() \
10461 do { \
10462 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10463 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10464 } while (0)
10465
10466# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10467 do { \
10468 AssertPtr((a_pVCpu)); \
10469 AssertPtr((a_pVmxTransient)); \
10470 Assert((a_pVmxTransient)->fVMEntryFailed == false); \
10471 Assert(ASMIntAreEnabled()); \
10472 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10473 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10474 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
10475 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10476 if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
10477 HMVMX_ASSERT_PREEMPT_CPUID(); \
10478 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10479 } while (0)
10480
10481# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10482 do { \
10483 Log4Func(("\n")); \
10484 } while (0)
10485#else
10486# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10487 do { \
10488 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10489 NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
10490 } while (0)
10491# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
10492#endif
10493
10494
10495/**
10496 * Advances the guest RIP by the specified number of bytes.
10497 *
10498 * @param pVCpu The cross context virtual CPU structure.
10499 * @param cbInstr Number of bytes to advance the RIP by.
10500 *
10501 * @remarks No-long-jump zone!!!
10502 */
10503DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
10504{
10505 /* Advance the RIP. */
10506 pVCpu->cpum.GstCtx.rip += cbInstr;
10507 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
10508
10509 /* Update interrupt inhibition. */
10510 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10511 && pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
10512 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10513}
10514
10515
10516/**
10517 * Advances the guest RIP after reading it from the VMCS.
10518 *
10519 * @returns VBox status code, no informational status codes.
10520 * @param pVCpu The cross context virtual CPU structure.
10521 * @param pVmxTransient Pointer to the VMX transient structure.
10522 *
10523 * @remarks No-long-jump zone!!!
10524 */
10525static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10526{
10527 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10528 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS);
10529 AssertRCReturn(rc, rc);
10530
10531 hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
10532
10533 /*
10534 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10535 * pending debug exception field as it takes care of priority of events.
10536 *
10537 * See Intel spec. 32.2.1 "Debug Exceptions".
10538 */
10539 if ( !pVCpu->hm.s.fSingleInstruction
10540 && pVCpu->cpum.GstCtx.eflags.Bits.u1TF)
10541 {
10542 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10543 AssertRCReturn(rc, rc);
10544 }
10545
10546 return VINF_SUCCESS;
10547}
10548
10549
10550/**
10551 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10552 * and update error record fields accordingly.
10553 *
10554 * @return VMX_IGS_* return codes.
10555 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10556 * wrong with the guest state.
10557 *
10558 * @param pVCpu The cross context virtual CPU structure.
10559 *
10560 * @remarks This function assumes our cache of the VMCS controls
10561 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10562 */
10563static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu)
10564{
10565#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10566#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10567 uError = (err); \
10568 break; \
10569 } else do { } while (0)
10570
10571 int rc;
10572 PVM pVM = pVCpu->CTX_SUFF(pVM);
10573 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10574 uint32_t uError = VMX_IGS_ERROR;
10575 uint32_t u32Val;
10576 bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10577
10578 do
10579 {
10580 /*
10581 * CR0.
10582 */
10583 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10584 uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10585 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10586 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10587 if (fUnrestrictedGuest)
10588 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
10589
10590 uint32_t u32GuestCr0;
10591 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10592 AssertRCBreak(rc);
10593 HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10594 HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10595 if ( !fUnrestrictedGuest
10596 && (u32GuestCr0 & X86_CR0_PG)
10597 && !(u32GuestCr0 & X86_CR0_PE))
10598 {
10599 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10600 }
10601
10602 /*
10603 * CR4.
10604 */
10605 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10606 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10607
10608 uint32_t u32GuestCr4;
10609 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10610 AssertRCBreak(rc);
10611 HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10612 HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10613
10614 /*
10615 * IA32_DEBUGCTL MSR.
10616 */
10617 uint64_t u64Val;
10618 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10619 AssertRCBreak(rc);
10620 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10621 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10622 {
10623 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10624 }
10625 uint64_t u64DebugCtlMsr = u64Val;
10626
10627#ifdef VBOX_STRICT
10628 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10629 AssertRCBreak(rc);
10630 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10631#endif
10632 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
10633
10634 /*
10635 * RIP and RFLAGS.
10636 */
10637 uint32_t u32Eflags;
10638#if HC_ARCH_BITS == 64
10639 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10640 AssertRCBreak(rc);
10641 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10642 if ( !fLongModeGuest
10643 || !pCtx->cs.Attr.n.u1Long)
10644 {
10645 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10646 }
10647 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10648 * must be identical if the "IA-32e mode guest" VM-entry
10649 * control is 1 and CS.L is 1. No check applies if the
10650 * CPU supports 64 linear-address bits. */
10651
10652 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10653 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10654 AssertRCBreak(rc);
10655 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10656 VMX_IGS_RFLAGS_RESERVED);
10657 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10658 u32Eflags = u64Val;
10659#else
10660 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10661 AssertRCBreak(rc);
10662 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10663 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10664#endif
10665
10666 if ( fLongModeGuest
10667 || ( fUnrestrictedGuest
10668 && !(u32GuestCr0 & X86_CR0_PE)))
10669 {
10670 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10671 }
10672
10673 uint32_t u32EntryInfo;
10674 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10675 AssertRCBreak(rc);
10676 if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
10677 && VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
10678 {
10679 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10680 }
10681
10682 /*
10683 * 64-bit checks.
10684 */
10685#if HC_ARCH_BITS == 64
10686 if (fLongModeGuest)
10687 {
10688 HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10689 HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10690 }
10691
10692 if ( !fLongModeGuest
10693 && (u32GuestCr4 & X86_CR4_PCIDE))
10694 {
10695 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10696 }
10697
10698 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10699 * 51:32 beyond the processor's physical-address width are 0. */
10700
10701 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10702 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10703 {
10704 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10705 }
10706
10707 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10708 AssertRCBreak(rc);
10709 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10710
10711 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10712 AssertRCBreak(rc);
10713 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10714#endif
10715
10716 /*
10717 * PERF_GLOBAL MSR.
10718 */
10719 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
10720 {
10721 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10722 AssertRCBreak(rc);
10723 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10724 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10725 }
10726
10727 /*
10728 * PAT MSR.
10729 */
10730 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
10731 {
10732 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10733 AssertRCBreak(rc);
10734 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10735 for (unsigned i = 0; i < 8; i++)
10736 {
10737 uint8_t u8Val = (u64Val & 0xff);
10738 if ( u8Val != 0 /* UC */
10739 && u8Val != 1 /* WC */
10740 && u8Val != 4 /* WT */
10741 && u8Val != 5 /* WP */
10742 && u8Val != 6 /* WB */
10743 && u8Val != 7 /* UC- */)
10744 {
10745 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10746 }
10747 u64Val >>= 8;
10748 }
10749 }
10750
10751 /*
10752 * EFER MSR.
10753 */
10754 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
10755 {
10756 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10757 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10758 AssertRCBreak(rc);
10759 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10760 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10761 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10762 & VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
10763 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10764 /** @todo r=ramshankar: Unrestricted check here is probably wrong, see
10765 * iemVmxVmentryCheckGuestState(). */
10766 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10767 || !(u32GuestCr0 & X86_CR0_PG)
10768 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10769 VMX_IGS_EFER_LMA_LME_MISMATCH);
10770 }
10771
10772 /*
10773 * Segment registers.
10774 */
10775 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10776 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10777 if (!(u32Eflags & X86_EFL_VM))
10778 {
10779 /* CS */
10780 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10781 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10782 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10783 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10784 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10785 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10786 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10787 /* CS cannot be loaded with NULL in protected mode. */
10788 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10789 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10790 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10791 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10792 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10793 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10794 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10795 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10796 else
10797 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10798
10799 /* SS */
10800 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10801 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10802 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10803 if ( !(pCtx->cr0 & X86_CR0_PE)
10804 || pCtx->cs.Attr.n.u4Type == 3)
10805 {
10806 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10807 }
10808 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10809 {
10810 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10811 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10812 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10813 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10814 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10815 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10816 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10817 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10818 }
10819
10820 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmenReg(). */
10821 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10822 {
10823 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10824 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10825 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10826 || pCtx->ds.Attr.n.u4Type > 11
10827 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10828 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10829 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10830 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10831 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10832 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10833 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10834 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10835 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10836 }
10837 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10838 {
10839 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10840 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10841 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10842 || pCtx->es.Attr.n.u4Type > 11
10843 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10844 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10845 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10846 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10847 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10848 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10849 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10850 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10851 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10852 }
10853 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10854 {
10855 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10856 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10857 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10858 || pCtx->fs.Attr.n.u4Type > 11
10859 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10860 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10861 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10862 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10863 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10864 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10865 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10866 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10867 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10868 }
10869 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10870 {
10871 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10872 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10873 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10874 || pCtx->gs.Attr.n.u4Type > 11
10875 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10876 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10877 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10878 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10879 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10880 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10881 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10882 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10883 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10884 }
10885 /* 64-bit capable CPUs. */
10886#if HC_ARCH_BITS == 64
10887 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10888 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10889 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10890 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10891 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10892 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10893 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10894 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10895 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10896 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10897 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10898#endif
10899 }
10900 else
10901 {
10902 /* V86 mode checks. */
10903 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10904 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10905 {
10906 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10907 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10908 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10909 }
10910 else
10911 {
10912 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10913 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10914 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10915 }
10916
10917 /* CS */
10918 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10919 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10920 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10921 /* SS */
10922 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10923 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10924 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10925 /* DS */
10926 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10927 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10928 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10929 /* ES */
10930 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10931 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10932 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10933 /* FS */
10934 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10935 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10936 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10937 /* GS */
10938 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10939 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10940 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10941 /* 64-bit capable CPUs. */
10942#if HC_ARCH_BITS == 64
10943 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10944 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10945 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10946 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10947 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10948 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10949 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10950 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10951 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10952 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10953 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10954#endif
10955 }
10956
10957 /*
10958 * TR.
10959 */
10960 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10961 /* 64-bit capable CPUs. */
10962#if HC_ARCH_BITS == 64
10963 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10964#endif
10965 if (fLongModeGuest)
10966 {
10967 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10968 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10969 }
10970 else
10971 {
10972 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10973 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10974 VMX_IGS_TR_ATTR_TYPE_INVALID);
10975 }
10976 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10977 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10978 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10979 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10980 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10981 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10982 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10983 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10984
10985 /*
10986 * GDTR and IDTR.
10987 */
10988#if HC_ARCH_BITS == 64
10989 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10990 AssertRCBreak(rc);
10991 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10992
10993 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10994 AssertRCBreak(rc);
10995 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10996#endif
10997
10998 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10999 AssertRCBreak(rc);
11000 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11001
11002 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
11003 AssertRCBreak(rc);
11004 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11005
11006 /*
11007 * Guest Non-Register State.
11008 */
11009 /* Activity State. */
11010 uint32_t u32ActivityState;
11011 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
11012 AssertRCBreak(rc);
11013 HMVMX_CHECK_BREAK( !u32ActivityState
11014 || (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
11015 VMX_IGS_ACTIVITY_STATE_INVALID);
11016 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
11017 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
11018 uint32_t u32IntrState;
11019 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
11020 AssertRCBreak(rc);
11021 if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
11022 || u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
11023 {
11024 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11025 }
11026
11027 /** @todo Activity state and injecting interrupts. Left as a todo since we
11028 * currently don't use activity states but ACTIVE. */
11029
11030 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
11031 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11032
11033 /* Guest interruptibility-state. */
11034 HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11035 HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
11036 != (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
11037 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11038 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11039 || !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
11040 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11041 if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
11042 {
11043 if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
11044 {
11045 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
11046 && !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
11047 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11048 }
11049 else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
11050 {
11051 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
11052 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11053 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
11054 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11055 }
11056 }
11057 /** @todo Assumes the processor is not in SMM. */
11058 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
11059 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11060 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
11061 || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
11062 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11063 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
11064 && VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
11065 && VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
11066 {
11067 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
11068 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11069 }
11070
11071 /* Pending debug exceptions. */
11072#if HC_ARCH_BITS == 64
11073 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
11074 AssertRCBreak(rc);
11075 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11076 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11077 u32Val = u64Val; /* For pending debug exceptions checks below. */
11078#else
11079 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
11080 AssertRCBreak(rc);
11081 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11082 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11083#endif
11084
11085 if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
11086 || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
11087 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11088 {
11089 if ( (u32Eflags & X86_EFL_TF)
11090 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11091 {
11092 /* Bit 14 is PendingDebug.BS. */
11093 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11094 }
11095 if ( !(u32Eflags & X86_EFL_TF)
11096 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11097 {
11098 /* Bit 14 is PendingDebug.BS. */
11099 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11100 }
11101 }
11102
11103 /* VMCS link pointer. */
11104 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11105 AssertRCBreak(rc);
11106 if (u64Val != UINT64_C(0xffffffffffffffff))
11107 {
11108 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11109 /** @todo Bits beyond the processor's physical-address width MBZ. */
11110 /** @todo 32-bit located in memory referenced by value of this field (as a
11111 * physical address) must contain the processor's VMCS revision ID. */
11112 /** @todo SMM checks. */
11113 }
11114
11115 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11116 * not using Nested Paging? */
11117 if ( pVM->hm.s.fNestedPaging
11118 && !fLongModeGuest
11119 && CPUMIsGuestInPAEModeEx(pCtx))
11120 {
11121 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11122 AssertRCBreak(rc);
11123 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11124
11125 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11126 AssertRCBreak(rc);
11127 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11128
11129 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11130 AssertRCBreak(rc);
11131 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11132
11133 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11134 AssertRCBreak(rc);
11135 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11136 }
11137
11138 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11139 if (uError == VMX_IGS_ERROR)
11140 uError = VMX_IGS_REASON_NOT_FOUND;
11141 } while (0);
11142
11143 pVCpu->hm.s.u32HMError = uError;
11144 return uError;
11145
11146#undef HMVMX_ERROR_BREAK
11147#undef HMVMX_CHECK_BREAK
11148}
11149
11150
11151/** @name VM-exit handlers.
11152 * @{
11153 */
11154/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11155/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11156/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11157
11158/**
11159 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11160 */
11161HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11162{
11163 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11164 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11165 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11166 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11167 return VINF_SUCCESS;
11168 return VINF_EM_RAW_INTERRUPT;
11169}
11170
11171
11172/**
11173 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11174 */
11175HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11176{
11177 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11178 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11179
11180 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11181 AssertRCReturn(rc, rc);
11182
11183 uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
11184 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
11185 && uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
11186 Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11187
11188 if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
11189 {
11190 /*
11191 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
11192 * injected it ourselves and anything we inject is not going to cause a VM-exit directly
11193 * for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
11194 *
11195 * [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11196 * [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
11197 */
11198 VMXDispatchHostNmi();
11199 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11200 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11201 return VINF_SUCCESS;
11202 }
11203
11204 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11205 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
11206 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11207 { /* likely */ }
11208 else
11209 {
11210 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11211 rcStrictRc1 = VINF_SUCCESS;
11212 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11213 return rcStrictRc1;
11214 }
11215
11216 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11217 uint32_t uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
11218 switch (uIntType)
11219 {
11220 case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11221 Assert(uVector == X86_XCPT_DB);
11222 RT_FALL_THRU();
11223 case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11224 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
11225 RT_FALL_THRU();
11226 case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11227 {
11228 /*
11229 * If there's any exception caused as a result of event injection, the resulting
11230 * secondary/final execption will be pending, we shall continue guest execution
11231 * after injecting the event. The page-fault case is complicated and we manually
11232 * handle any currently pending event in hmR0VmxExitXcptPF.
11233 */
11234 if (!pVCpu->hm.s.Event.fPending)
11235 { /* likely */ }
11236 else if (uVector != X86_XCPT_PF)
11237 {
11238 rc = VINF_SUCCESS;
11239 break;
11240 }
11241
11242 switch (uVector)
11243 {
11244 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
11245 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
11246 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
11247 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
11248 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
11249 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
11250
11251 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11252 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11253 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11254 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11255 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11256 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11257 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11258 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11259 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11260 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11261 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11262 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11263 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11264 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11265 default:
11266 {
11267 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11268 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11269 {
11270 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11271 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11272 Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
11273
11274 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
11275 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11276 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11277 AssertRCReturn(rc, rc);
11278 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11279 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11280 0 /* GCPtrFaultAddress */);
11281 }
11282 else
11283 {
11284 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11285 pVCpu->hm.s.u32HMError = uVector;
11286 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11287 }
11288 break;
11289 }
11290 }
11291 break;
11292 }
11293
11294 default:
11295 {
11296 pVCpu->hm.s.u32HMError = uExitIntInfo;
11297 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11298 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
11299 break;
11300 }
11301 }
11302 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11303 return rc;
11304}
11305
11306
11307/**
11308 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11309 */
11310HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11311{
11312 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11313
11314 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11315 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11316
11317 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11318 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11319 return VINF_SUCCESS;
11320}
11321
11322
11323/**
11324 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11325 */
11326HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11327{
11328 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11329 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)))
11330 {
11331 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11332 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11333 }
11334
11335 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11336
11337 /*
11338 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11339 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11340 */
11341 uint32_t fIntrState = 0;
11342 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
11343 AssertRCReturn(rc, rc);
11344
11345 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
11346 if ( fBlockSti
11347 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11348 {
11349 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11350 }
11351
11352 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11353 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11354
11355 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11356 return VINF_SUCCESS;
11357}
11358
11359
11360/**
11361 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11362 */
11363HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11364{
11365 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11366 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11367}
11368
11369
11370/**
11371 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11372 */
11373HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11374{
11375 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11376 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11377}
11378
11379
11380/**
11381 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11382 */
11383HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11384{
11385 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11386
11387 /*
11388 * Get the state we need and update the exit history entry.
11389 */
11390 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11391 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
11392 AssertRCReturn(rc, rc);
11393
11394 VBOXSTRICTRC rcStrict;
11395 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11396 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
11397 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11398 if (!pExitRec)
11399 {
11400 /*
11401 * Regular CPUID instruction execution.
11402 */
11403 rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
11404 if (rcStrict == VINF_SUCCESS)
11405 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11406 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11407 {
11408 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11409 rcStrict = VINF_SUCCESS;
11410 }
11411 }
11412 else
11413 {
11414 /*
11415 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
11416 */
11417 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11418 AssertRCReturn(rc2, rc2);
11419
11420 Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11421 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11422
11423 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11424 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
11425
11426 Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11427 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11428 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11429 }
11430 return rcStrict;
11431}
11432
11433
11434/**
11435 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11436 */
11437HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11438{
11439 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11440 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4);
11441 AssertRCReturn(rc, rc);
11442
11443 if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
11444 return VINF_EM_RAW_EMULATE_INSTR;
11445
11446 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11447 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11448}
11449
11450
11451/**
11452 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11453 */
11454HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11455{
11456 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11457 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11458 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11459 AssertRCReturn(rc, rc);
11460
11461 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11462 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11463 {
11464 /* If we get a spurious VM-exit when offsetting is enabled,
11465 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11466 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
11467 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11468 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11469 }
11470 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11471 {
11472 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11473 rcStrict = VINF_SUCCESS;
11474 }
11475 return rcStrict;
11476}
11477
11478
11479/**
11480 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11481 */
11482HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11483{
11484 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11485 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_TSC_AUX);
11486 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11487 AssertRCReturn(rc, rc);
11488
11489 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11490 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11491 {
11492 /* If we get a spurious VM-exit when offsetting is enabled,
11493 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11494 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
11495 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11496 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11497 }
11498 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11499 {
11500 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11501 rcStrict = VINF_SUCCESS;
11502 }
11503 return rcStrict;
11504}
11505
11506
11507/**
11508 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11509 */
11510HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11511{
11512 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11513 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11514 AssertRCReturn(rc, rc);
11515
11516 PVM pVM = pVCpu->CTX_SUFF(pVM);
11517 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11518 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11519 if (RT_LIKELY(rc == VINF_SUCCESS))
11520 {
11521 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11522 Assert(pVmxTransient->cbInstr == 2);
11523 }
11524 else
11525 {
11526 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11527 rc = VERR_EM_INTERPRETER;
11528 }
11529 return rc;
11530}
11531
11532
11533/**
11534 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11535 */
11536HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11537{
11538 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11539
11540 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11541 if (EMAreHypercallInstructionsEnabled(pVCpu))
11542 {
11543 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_SS
11544 | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_EFER);
11545 AssertRCReturn(rc, rc);
11546
11547 /* Perform the hypercall. */
11548 rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
11549 if (rcStrict == VINF_SUCCESS)
11550 {
11551 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11552 AssertRCReturn(rc, rc);
11553 }
11554 else
11555 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11556 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11557 || RT_FAILURE(rcStrict));
11558
11559 /* If the hypercall changes anything other than guest's general-purpose registers,
11560 we would need to reload the guest changed bits here before VM-entry. */
11561 }
11562 else
11563 Log4Func(("Hypercalls not enabled\n"));
11564
11565 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11566 if (RT_FAILURE(rcStrict))
11567 {
11568 hmR0VmxSetPendingXcptUD(pVCpu);
11569 rcStrict = VINF_SUCCESS;
11570 }
11571
11572 return rcStrict;
11573}
11574
11575
11576/**
11577 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11578 */
11579HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11580{
11581 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11582 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11583
11584 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11585 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11586 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
11587 AssertRCReturn(rc, rc);
11588
11589 VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
11590
11591 if (rcStrict == VINF_SUCCESS || rcStrict == VINF_PGM_SYNC_CR3)
11592 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11593 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11594 {
11595 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11596 rcStrict = VINF_SUCCESS;
11597 }
11598 else
11599 AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) sttus: %Rrc\n", pVmxTransient->uExitQual,
11600 VBOXSTRICTRC_VAL(rcStrict)));
11601 return rcStrict;
11602}
11603
11604
11605/**
11606 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11607 */
11608HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11609{
11610 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11611 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11612 AssertRCReturn(rc, rc);
11613
11614 PVM pVM = pVCpu->CTX_SUFF(pVM);
11615 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11616 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11617 if (RT_LIKELY(rc == VINF_SUCCESS))
11618 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11619 else
11620 {
11621 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11622 rc = VERR_EM_INTERPRETER;
11623 }
11624 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11625 return rc;
11626}
11627
11628
11629/**
11630 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11631 */
11632HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11633{
11634 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11635 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11636 AssertRCReturn(rc, rc);
11637
11638 PVM pVM = pVCpu->CTX_SUFF(pVM);
11639 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11640 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11641 rc = VBOXSTRICTRC_VAL(rc2);
11642 if (RT_LIKELY( rc == VINF_SUCCESS
11643 || rc == VINF_EM_HALT))
11644 {
11645 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11646 AssertRCReturn(rc3, rc3);
11647
11648 if ( rc == VINF_EM_HALT
11649 && EMMonitorWaitShouldContinue(pVCpu, pCtx))
11650 rc = VINF_SUCCESS;
11651 }
11652 else
11653 {
11654 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11655 rc = VERR_EM_INTERPRETER;
11656 }
11657 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11658 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11659 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11660 return rc;
11661}
11662
11663
11664/**
11665 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11666 */
11667HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11668{
11669 /*
11670 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
11671 * mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
11672 * treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
11673 * VMX root operation. If we get here, something funny is going on.
11674 *
11675 * See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
11676 */
11677 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11678 AssertMsgFailed(("Unexpected RSM VM-exit\n"));
11679 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11680}
11681
11682
11683/**
11684 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11685 */
11686HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11687{
11688 /*
11689 * This can only happen if we support dual-monitor treatment of SMI, which can be activated
11690 * by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
11691 * this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
11692 * an SMI. If we get here, something funny is going on.
11693 *
11694 * See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11695 * See Intel spec. 25.3 "Other Causes of VM-Exits"
11696 */
11697 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11698 AssertMsgFailed(("Unexpected SMI VM-exit\n"));
11699 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11700}
11701
11702
11703/**
11704 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11705 */
11706HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11707{
11708 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11709 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11710 AssertMsgFailed(("Unexpected IO SMI VM-exit\n"));
11711 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11712}
11713
11714
11715/**
11716 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11717 */
11718HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11719{
11720 /*
11721 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
11722 * We don't make use of it as our guests don't have direct access to the host LAPIC.
11723 * See Intel spec. 25.3 "Other Causes of VM-exits".
11724 */
11725 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11726 AssertMsgFailed(("Unexpected SIPI VM-exit\n"));
11727 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11728}
11729
11730
11731/**
11732 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11733 * VM-exit.
11734 */
11735HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11736{
11737 /*
11738 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11739 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11740 *
11741 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11742 * See Intel spec. "23.8 Restrictions on VMX operation".
11743 */
11744 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11745 return VINF_SUCCESS;
11746}
11747
11748
11749/**
11750 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11751 * VM-exit.
11752 */
11753HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11754{
11755 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11756 return VINF_EM_RESET;
11757}
11758
11759
11760/**
11761 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11762 */
11763HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11764{
11765 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11766 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_HLT_EXIT);
11767
11768 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11769 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
11770 AssertRCReturn(rc, rc);
11771
11772 if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
11773 rc = VINF_SUCCESS;
11774 else
11775 rc = VINF_EM_HALT;
11776
11777 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11778 if (rc != VINF_SUCCESS)
11779 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11780 return rc;
11781}
11782
11783
11784/**
11785 * VM-exit handler for instructions that result in a \#UD exception delivered to
11786 * the guest.
11787 */
11788HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11789{
11790 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11791 hmR0VmxSetPendingXcptUD(pVCpu);
11792 return VINF_SUCCESS;
11793}
11794
11795
11796/**
11797 * VM-exit handler for expiry of the VMX preemption timer.
11798 */
11799HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11800{
11801 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11802
11803 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11804 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11805
11806 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11807 PVM pVM = pVCpu->CTX_SUFF(pVM);
11808 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11809 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11810 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11811}
11812
11813
11814/**
11815 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11816 */
11817HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11818{
11819 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11820
11821 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11822 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_CR4);
11823 AssertRCReturn(rc, rc);
11824
11825 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11826 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11827 : HM_CHANGED_RAISED_XCPT_MASK);
11828
11829 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11830 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
11831
11832 return rcStrict;
11833}
11834
11835
11836/**
11837 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11838 */
11839HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11840{
11841 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11842 /** @todo Use VM-exit instruction information. */
11843 return VERR_EM_INTERPRETER;
11844}
11845
11846
11847/**
11848 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11849 * Error VM-exit.
11850 */
11851HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11852{
11853 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11854 AssertRCReturn(rc, rc);
11855 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11856 if (RT_FAILURE(rc))
11857 return rc;
11858
11859 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
11860 NOREF(uInvalidReason);
11861
11862#ifdef VBOX_STRICT
11863 uint32_t fIntrState;
11864 RTHCUINTREG uHCReg;
11865 uint64_t u64Val;
11866 uint32_t u32Val;
11867
11868 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11869 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11870 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11871 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
11872 AssertRCReturn(rc, rc);
11873
11874 Log4(("uInvalidReason %u\n", uInvalidReason));
11875 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11876 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11877 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11878 Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
11879
11880 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11881 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11882 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11883 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11884 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11885 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11886 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11887 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11888 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11889 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11890 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11891 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11892
11893 hmR0DumpRegs(pVCpu);
11894#else
11895 NOREF(pVmxTransient);
11896#endif
11897
11898 return VERR_VMX_INVALID_GUEST_STATE;
11899}
11900
11901
11902/**
11903 * VM-exit handler for VM-entry failure due to an MSR-load
11904 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11905 */
11906HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11907{
11908 AssertMsgFailed(("Unexpected MSR-load exit\n"));
11909 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11910}
11911
11912
11913/**
11914 * VM-exit handler for VM-entry failure due to a machine-check event
11915 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11916 */
11917HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11918{
11919 AssertMsgFailed(("Unexpected machine-check event exit\n"));
11920 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11921}
11922
11923
11924/**
11925 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11926 * theory.
11927 */
11928HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11929{
11930 RT_NOREF2(pVCpu, pVmxTransient);
11931 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d\n", pVmxTransient->uExitReason));
11932 return VERR_VMX_UNDEFINED_EXIT_CODE;
11933}
11934
11935
11936/**
11937 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11938 * (VMX_EXIT_GDTR_IDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11939 * Conditional VM-exit.
11940 */
11941HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11942{
11943 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11944
11945 /* By default, we don't enable VMX_PROC_CTLS2_DESCRIPTOR_TABLE_EXIT. */
11946 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11947 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_DESC_TABLE_EXIT)
11948 return VERR_EM_INTERPRETER;
11949 AssertMsgFailed(("Unexpected XDTR access\n"));
11950 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11951}
11952
11953
11954/**
11955 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11956 */
11957HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11958{
11959 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11960
11961 /* By default, we don't enable VMX_PROC_CTLS2_RDRAND_EXIT. */
11962 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_RDRAND_EXIT)
11963 return VERR_EM_INTERPRETER;
11964 AssertMsgFailed(("Unexpected RDRAND exit\n"));
11965 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11966}
11967
11968
11969/**
11970 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11971 */
11972HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11973{
11974 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11975
11976 /** @todo Optimize this: We currently drag in in the whole MSR state
11977 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11978 * MSRs required. That would require changes to IEM and possibly CPUM too.
11979 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11980 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; NOREF(idMsr); /* Save it. */
11981 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11982 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
11983 AssertRCReturn(rc, rc);
11984
11985 Log4Func(("ecx=%#RX32\n", idMsr));
11986
11987#ifdef VBOX_STRICT
11988 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
11989 {
11990 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr)
11991 && idMsr != MSR_K6_EFER)
11992 {
11993 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
11994 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11995 }
11996 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11997 {
11998 VMXMSREXITREAD enmRead;
11999 VMXMSREXITWRITE enmWrite;
12000 int rc2 = hmR0VmxGetMsrPermission(pVCpu, idMsr, &enmRead, &enmWrite);
12001 AssertRCReturn(rc2, rc2);
12002 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
12003 {
12004 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
12005 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12006 }
12007 }
12008 }
12009#endif
12010
12011 VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
12012 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
12013 if (rcStrict == VINF_SUCCESS)
12014 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
12015 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX);
12016 else if (rcStrict == VINF_IEM_RAISED_XCPT)
12017 {
12018 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
12019 rcStrict = VINF_SUCCESS;
12020 }
12021 else
12022 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12023
12024 return rcStrict;
12025}
12026
12027
12028/**
12029 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12030 */
12031HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12032{
12033 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12034
12035 /** @todo Optimize this: We currently drag in in the whole MSR state
12036 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
12037 * MSRs required. That would require changes to IEM and possibly CPUM too.
12038 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
12039 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; /* Save it. */
12040 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12041 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
12042 AssertRCReturn(rc, rc);
12043
12044 Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
12045
12046 VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
12047 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12048
12049 if (rcStrict == VINF_SUCCESS)
12050 {
12051 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12052
12053 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
12054 if ( idMsr == MSR_IA32_APICBASE
12055 || ( idMsr >= MSR_IA32_X2APIC_START
12056 && idMsr <= MSR_IA32_X2APIC_END))
12057 {
12058 /*
12059 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12060 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before IEM changes it.
12061 */
12062 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
12063 }
12064 else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12065 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12066 else if (idMsr == MSR_K6_EFER)
12067 {
12068 /*
12069 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12070 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12071 * the other bits as well, SCE and NXE. See @bugref{7368}.
12072 */
12073 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS
12074 | HM_CHANGED_VMX_EXIT_CTLS);
12075 }
12076
12077 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12078 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
12079 {
12080 switch (idMsr)
12081 {
12082 case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
12083 case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
12084 case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
12085 case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
12086 case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
12087 case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
12088 default:
12089 {
12090 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
12091 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12092 else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
12093 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
12094 break;
12095 }
12096 }
12097 }
12098#ifdef VBOX_STRICT
12099 else
12100 {
12101 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12102 switch (idMsr)
12103 {
12104 case MSR_IA32_SYSENTER_CS:
12105 case MSR_IA32_SYSENTER_EIP:
12106 case MSR_IA32_SYSENTER_ESP:
12107 case MSR_K8_FS_BASE:
12108 case MSR_K8_GS_BASE:
12109 {
12110 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
12111 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12112 }
12113
12114 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12115 default:
12116 {
12117 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
12118 {
12119 /* EFER writes are always intercepted, see hmR0VmxExportGuestMsrs(). */
12120 if (idMsr != MSR_K6_EFER)
12121 {
12122 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12123 idMsr));
12124 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12125 }
12126 }
12127
12128 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
12129 {
12130 VMXMSREXITREAD enmRead;
12131 VMXMSREXITWRITE enmWrite;
12132 int rc2 = hmR0VmxGetMsrPermission(pVCpu, idMsr, &enmRead, &enmWrite);
12133 AssertRCReturn(rc2, rc2);
12134 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12135 {
12136 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
12137 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12138 }
12139 }
12140 break;
12141 }
12142 }
12143 }
12144#endif /* VBOX_STRICT */
12145 }
12146 else if (rcStrict == VINF_IEM_RAISED_XCPT)
12147 {
12148 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
12149 rcStrict = VINF_SUCCESS;
12150 }
12151 else
12152 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12153
12154 return rcStrict;
12155}
12156
12157
12158/**
12159 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12160 */
12161HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12162{
12163 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12164 /** @todo The guest has likely hit a contended spinlock. We might want to
12165 * poke a schedule different guest VCPU. */
12166 return VINF_EM_RAW_INTERRUPT;
12167}
12168
12169
12170/**
12171 * VM-exit handler for when the TPR value is lowered below the specified
12172 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12173 */
12174HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12175{
12176 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12177 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
12178
12179 /*
12180 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12181 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12182 */
12183 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12184 return VINF_SUCCESS;
12185}
12186
12187
12188/**
12189 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12190 * VM-exit.
12191 *
12192 * @retval VINF_SUCCESS when guest execution can continue.
12193 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12194 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12195 * interpreter.
12196 */
12197HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12198{
12199 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12200 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12201
12202 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12203 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12204 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12205 AssertRCReturn(rc, rc);
12206
12207 VBOXSTRICTRC rcStrict;
12208 PVM pVM = pVCpu->CTX_SUFF(pVM);
12209 RTGCUINTPTR const uExitQual = pVmxTransient->uExitQual;
12210 uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
12211 switch (uAccessType)
12212 {
12213 case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
12214 {
12215 uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
12216 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
12217 VMX_EXIT_QUAL_CRX_GENREG(uExitQual));
12218 AssertMsg( rcStrict == VINF_SUCCESS
12219 || rcStrict == VINF_IEM_RAISED_XCPT
12220 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12221
12222 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
12223 {
12224 case 0:
12225 {
12226 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12227 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12228 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
12229 Log4Func(("CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
12230
12231 /*
12232 * This is a kludge for handling switches back to real mode when we try to use
12233 * V86 mode to run real mode code directly. Problem is that V86 mode cannot
12234 * deal with special selector values, so we have to return to ring-3 and run
12235 * there till the selector values are V86 mode compatible.
12236 *
12237 * Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
12238 * latter is an alias for VINF_IEM_RAISED_XCPT which is converted to VINF_SUCCESs
12239 * at the end of this function.
12240 */
12241 if ( rc == VINF_SUCCESS
12242 && !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
12243 && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
12244 && (uOldCr0 & X86_CR0_PE)
12245 && !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
12246 {
12247 /** @todo check selectors rather than returning all the time. */
12248 Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
12249 rcStrict = VINF_EM_RESCHEDULE_REM;
12250 }
12251 break;
12252 }
12253
12254 case 2:
12255 {
12256 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
12257 /* Nothing to do here, CR2 it's not part of the VMCS. */
12258 break;
12259 }
12260
12261 case 3:
12262 {
12263 Assert( !pVM->hm.s.fNestedPaging
12264 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
12265 || pVCpu->hm.s.fUsingDebugLoop);
12266 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
12267 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12268 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR3);
12269 Log4Func(("CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
12270 break;
12271 }
12272
12273 case 4:
12274 {
12275 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
12276 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12277 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR4);
12278 Log4Func(("CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
12279 pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12280 break;
12281 }
12282
12283 case 8:
12284 {
12285 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
12286 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
12287 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12288 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_APIC_TPR);
12289 break;
12290 }
12291 default:
12292 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual)));
12293 break;
12294 }
12295 break;
12296 }
12297
12298 case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
12299 {
12300 Assert( !pVM->hm.s.fNestedPaging
12301 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
12302 || pVCpu->hm.s.fUsingDebugLoop
12303 || VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 3);
12304 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12305 Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 8
12306 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
12307
12308 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_GENREG(uExitQual),
12309 VMX_EXIT_QUAL_CRX_REGISTER(uExitQual));
12310 AssertMsg( rcStrict == VINF_SUCCESS
12311 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12312#ifdef VBOX_WITH_STATISTICS
12313 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
12314 {
12315 case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
12316 case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
12317 case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
12318 case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
12319 case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
12320 }
12321#endif
12322 Log4Func(("CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
12323 VBOXSTRICTRC_VAL(rcStrict)));
12324 if (VMX_EXIT_QUAL_CRX_GENREG(uExitQual) == X86_GREG_xSP)
12325 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_RSP);
12326 else
12327 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12328 break;
12329 }
12330
12331 case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12332 {
12333 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12334 AssertMsg( rcStrict == VINF_SUCCESS
12335 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12336
12337 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12338 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12339 Log4Func(("CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12340 break;
12341 }
12342
12343 case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12344 {
12345 /* Note! LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here. */
12346 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual));
12347 AssertMsg( rcStrict == VINF_SUCCESS
12348 || rcStrict == VINF_IEM_RAISED_XCPT
12349 , ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12350
12351 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12352 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12353 Log4Func(("LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12354 break;
12355 }
12356
12357 default:
12358 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12359 VERR_VMX_UNEXPECTED_EXCEPTION);
12360 }
12361
12362 Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS))
12363 == (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS));
12364 if (rcStrict == VINF_IEM_RAISED_XCPT)
12365 {
12366 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
12367 rcStrict = VINF_SUCCESS;
12368 }
12369
12370 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12371 NOREF(pVM);
12372 return rcStrict;
12373}
12374
12375
12376/**
12377 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12378 * VM-exit.
12379 */
12380HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12381{
12382 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12383 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12384
12385 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12386 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12387 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12388 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_EFER);
12389 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12390 AssertRCReturn(rc, rc);
12391
12392 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12393 uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
12394 uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQual);
12395 bool fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12396 bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
12397 bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
12398 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12399 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12400
12401 /*
12402 * Update exit history to see if this exit can be optimized.
12403 */
12404 VBOXSTRICTRC rcStrict;
12405 PCEMEXITREC pExitRec = NULL;
12406 if ( !fGstStepping
12407 && !fDbgStepping)
12408 pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12409 !fIOString
12410 ? !fIOWrite
12411 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12412 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12413 : !fIOWrite
12414 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12415 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12416 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12417 if (!pExitRec)
12418 {
12419 /* I/O operation lookup arrays. */
12420 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12421 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12422 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12423 uint32_t const cbInstr = pVmxTransient->cbInstr;
12424 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12425 PVM pVM = pVCpu->CTX_SUFF(pVM);
12426 if (fIOString)
12427 {
12428 /*
12429 * INS/OUTS - I/O String instruction.
12430 *
12431 * Use instruction-information if available, otherwise fall back on
12432 * interpreting the instruction.
12433 */
12434 Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12435 AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
12436 bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
12437 if (fInsOutsInfo)
12438 {
12439 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12440 AssertRCReturn(rc2, rc2);
12441 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12442 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12443 IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12444 bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
12445 if (fIOWrite)
12446 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12447 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12448 else
12449 {
12450 /*
12451 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12452 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12453 * See Intel Instruction spec. for "INS".
12454 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12455 */
12456 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12457 }
12458 }
12459 else
12460 rcStrict = IEMExecOne(pVCpu);
12461
12462 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12463 fUpdateRipAlready = true;
12464 }
12465 else
12466 {
12467 /*
12468 * IN/OUT - I/O instruction.
12469 */
12470 Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12471 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12472 Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
12473 if (fIOWrite)
12474 {
12475 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
12476 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12477 if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
12478 && !pCtx->eflags.Bits.u1TF)
12479 rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
12480 }
12481 else
12482 {
12483 uint32_t u32Result = 0;
12484 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12485 if (IOM_SUCCESS(rcStrict))
12486 {
12487 /* Save result of I/O IN instr. in AL/AX/EAX. */
12488 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12489 }
12490 if ( rcStrict == VINF_IOM_R3_IOPORT_READ
12491 && !pCtx->eflags.Bits.u1TF)
12492 rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
12493 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12494 }
12495 }
12496
12497 if (IOM_SUCCESS(rcStrict))
12498 {
12499 if (!fUpdateRipAlready)
12500 {
12501 hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
12502 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12503 }
12504
12505 /*
12506 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12507 * while booting Fedora 17 64-bit guest.
12508 *
12509 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12510 */
12511 if (fIOString)
12512 {
12513 /** @todo Single-step for INS/OUTS with REP prefix? */
12514 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12515 }
12516 else if ( !fDbgStepping
12517 && fGstStepping)
12518 {
12519 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12520 AssertRCReturn(rc, rc);
12521 }
12522
12523 /*
12524 * If any I/O breakpoints are armed, we need to check if one triggered
12525 * and take appropriate action.
12526 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12527 */
12528 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12529 AssertRCReturn(rc, rc);
12530
12531 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12532 * execution engines about whether hyper BPs and such are pending. */
12533 uint32_t const uDr7 = pCtx->dr[7];
12534 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12535 && X86_DR7_ANY_RW_IO(uDr7)
12536 && (pCtx->cr4 & X86_CR4_DE))
12537 || DBGFBpIsHwIoArmed(pVM)))
12538 {
12539 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12540
12541 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12542 VMMRZCallRing3Disable(pVCpu);
12543 HM_DISABLE_PREEMPT(pVCpu);
12544
12545 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12546
12547 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
12548 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12549 {
12550 /* Raise #DB. */
12551 if (fIsGuestDbgActive)
12552 ASMSetDR6(pCtx->dr[6]);
12553 if (pCtx->dr[7] != uDr7)
12554 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_DR7;
12555
12556 hmR0VmxSetPendingXcptDB(pVCpu);
12557 }
12558 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12559 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12560 else if ( rcStrict2 != VINF_SUCCESS
12561 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12562 rcStrict = rcStrict2;
12563 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12564
12565 HM_RESTORE_PREEMPT();
12566 VMMRZCallRing3Enable(pVCpu);
12567 }
12568 }
12569
12570#ifdef VBOX_STRICT
12571 if ( rcStrict == VINF_IOM_R3_IOPORT_READ
12572 || rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
12573 Assert(!fIOWrite);
12574 else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
12575 || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
12576 || rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
12577 Assert(fIOWrite);
12578 else
12579 {
12580# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12581 * statuses, that the VMM device and some others may return. See
12582 * IOM_SUCCESS() for guidance. */
12583 AssertMsg( RT_FAILURE(rcStrict)
12584 || rcStrict == VINF_SUCCESS
12585 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12586 || rcStrict == VINF_EM_DBG_BREAKPOINT
12587 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12588 || rcStrict == VINF_EM_RAW_TO_R3
12589 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12590# endif
12591 }
12592#endif
12593 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12594 }
12595 else
12596 {
12597 /*
12598 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12599 */
12600 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12601 AssertRCReturn(rc2, rc2);
12602 STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
12603 : fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
12604 Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12605 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12606 VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
12607 fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
12608
12609 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12610 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12611
12612 Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12613 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12614 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12615 }
12616 return rcStrict;
12617}
12618
12619
12620/**
12621 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12622 * VM-exit.
12623 */
12624HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12625{
12626 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12627
12628 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12629 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12630 AssertRCReturn(rc, rc);
12631 if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12632 {
12633 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12634 AssertRCReturn(rc, rc);
12635 if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
12636 {
12637 uint32_t uErrCode;
12638 RTGCUINTPTR GCPtrFaultAddress;
12639 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12640 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12641 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
12642 if (fErrorCodeValid)
12643 {
12644 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12645 AssertRCReturn(rc, rc);
12646 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12647 }
12648 else
12649 uErrCode = 0;
12650
12651 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12652 && uVector == X86_XCPT_PF)
12653 GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
12654 else
12655 GCPtrFaultAddress = 0;
12656
12657 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12658 AssertRCReturn(rc, rc);
12659
12660 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12661 pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
12662
12663 Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
12664 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12665 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12666 }
12667 }
12668
12669 /* Fall back to the interpreter to emulate the task-switch. */
12670 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12671 return VERR_EM_INTERPRETER;
12672}
12673
12674
12675/**
12676 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12677 */
12678HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12679{
12680 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12681 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_MONITOR_TRAP_FLAG);
12682 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
12683 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12684 AssertRCReturn(rc, rc);
12685 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12686 return VINF_EM_DBG_STEPPED;
12687}
12688
12689
12690/**
12691 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12692 */
12693HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12694{
12695 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12696
12697 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12698
12699 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12700 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12701 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12702 {
12703 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12704 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12705 {
12706 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12707 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12708 }
12709 }
12710 else
12711 {
12712 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12713 rcStrict1 = VINF_SUCCESS;
12714 return rcStrict1;
12715 }
12716
12717 /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12718 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12719 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12720 AssertRCReturn(rc, rc);
12721
12722 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12723 uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
12724 VBOXSTRICTRC rcStrict2;
12725 switch (uAccessType)
12726 {
12727 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12728 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12729 {
12730 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
12731 || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
12732 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12733
12734 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12735 GCPhys &= PAGE_BASE_GC_MASK;
12736 GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
12737 PVM pVM = pVCpu->CTX_SUFF(pVM);
12738 Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12739 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
12740
12741 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12742 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12743 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12744 CPUMCTX2CORE(pCtx), GCPhys);
12745 Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12746 if ( rcStrict2 == VINF_SUCCESS
12747 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12748 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12749 {
12750 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12751 | HM_CHANGED_GUEST_APIC_TPR);
12752 rcStrict2 = VINF_SUCCESS;
12753 }
12754 break;
12755 }
12756
12757 default:
12758 Log4Func(("uAccessType=%#x\n", uAccessType));
12759 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12760 break;
12761 }
12762
12763 if (rcStrict2 != VINF_SUCCESS)
12764 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12765 return rcStrict2;
12766}
12767
12768
12769/**
12770 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12771 * VM-exit.
12772 */
12773HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12774{
12775 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12776
12777 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12778 if (pVmxTransient->fWasGuestDebugStateActive)
12779 {
12780 AssertMsgFailed(("Unexpected MOV DRx exit\n"));
12781 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12782 }
12783
12784 if ( !pVCpu->hm.s.fSingleInstruction
12785 && !pVmxTransient->fWasHyperDebugStateActive)
12786 {
12787 Assert(!DBGFIsStepping(pVCpu));
12788 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12789
12790 /* Don't intercept MOV DRx any more. */
12791 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
12792 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12793 AssertRCReturn(rc, rc);
12794
12795 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12796 VMMRZCallRing3Disable(pVCpu);
12797 HM_DISABLE_PREEMPT(pVCpu);
12798
12799 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12800 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12801 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12802
12803 HM_RESTORE_PREEMPT();
12804 VMMRZCallRing3Enable(pVCpu);
12805
12806#ifdef VBOX_WITH_STATISTICS
12807 rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12808 AssertRCReturn(rc, rc);
12809 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12810 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12811 else
12812 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12813#endif
12814 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12815 return VINF_SUCCESS;
12816 }
12817
12818 /*
12819 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12820 * Update the segment registers and DR7 from the CPU.
12821 */
12822 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12823 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12824 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_DR7);
12825 AssertRCReturn(rc, rc);
12826 Log4Func(("CS:RIP=%04x:%08RX64\n", pCtx->cs.Sel, pCtx->rip));
12827
12828 PVM pVM = pVCpu->CTX_SUFF(pVM);
12829 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12830 {
12831 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12832 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
12833 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
12834 if (RT_SUCCESS(rc))
12835 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12836 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12837 }
12838 else
12839 {
12840 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12841 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
12842 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
12843 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12844 }
12845
12846 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12847 if (RT_SUCCESS(rc))
12848 {
12849 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12850 AssertRCReturn(rc2, rc2);
12851 return VINF_SUCCESS;
12852 }
12853 return rc;
12854}
12855
12856
12857/**
12858 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12859 * Conditional VM-exit.
12860 */
12861HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12862{
12863 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12864 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12865
12866 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12867 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12868 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12869 {
12870 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12871 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12872 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12873 {
12874 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12875 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12876 }
12877 }
12878 else
12879 {
12880 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12881 rcStrict1 = VINF_SUCCESS;
12882 return rcStrict1;
12883 }
12884
12885 /*
12886 * Get sufficent state and update the exit history entry.
12887 */
12888 RTGCPHYS GCPhys;
12889 int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
12890 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12891 AssertRCReturn(rc, rc);
12892
12893 VBOXSTRICTRC rcStrict;
12894 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12895 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
12896 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12897 if (!pExitRec)
12898 {
12899 /*
12900 * If we succeed, resume guest execution.
12901 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12902 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12903 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12904 * weird case. See @bugref{6043}.
12905 */
12906 PVM pVM = pVCpu->CTX_SUFF(pVM);
12907 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12908 rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
12909 Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12910 if ( rcStrict == VINF_SUCCESS
12911 || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12912 || rcStrict == VERR_PAGE_NOT_PRESENT)
12913 {
12914 /* Successfully handled MMIO operation. */
12915 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12916 | HM_CHANGED_GUEST_APIC_TPR);
12917 rcStrict = VINF_SUCCESS;
12918 }
12919 }
12920 else
12921 {
12922 /*
12923 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12924 */
12925 int rc2 = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12926 AssertRCReturn(rc2, rc2);
12927
12928 Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12929 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12930
12931 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12932 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12933
12934 Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12935 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12936 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12937 }
12938 return VBOXSTRICTRC_TODO(rcStrict);
12939}
12940
12941
12942/**
12943 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12944 * VM-exit.
12945 */
12946HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12947{
12948 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12949 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12950
12951 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12952 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12953 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12954 {
12955 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12956 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12957 Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12958 }
12959 else
12960 {
12961 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12962 rcStrict1 = VINF_SUCCESS;
12963 return rcStrict1;
12964 }
12965
12966 RTGCPHYS GCPhys;
12967 int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
12968 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12969 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12970 AssertRCReturn(rc, rc);
12971
12972 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12973 AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
12974
12975 RTGCUINT uErrorCode = 0;
12976 if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
12977 uErrorCode |= X86_TRAP_PF_ID;
12978 if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
12979 uErrorCode |= X86_TRAP_PF_RW;
12980 if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
12981 uErrorCode |= X86_TRAP_PF_P;
12982
12983 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12984
12985
12986 /* Handle the pagefault trap for the nested shadow table. */
12987 PVM pVM = pVCpu->CTX_SUFF(pVM);
12988 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12989
12990 Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
12991 pCtx->cs.Sel, pCtx->rip));
12992
12993 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
12994 TRPMResetTrap(pVCpu);
12995
12996 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12997 if ( rcStrict2 == VINF_SUCCESS
12998 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12999 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
13000 {
13001 /* Successfully synced our nested page tables. */
13002 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
13003 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
13004 return VINF_SUCCESS;
13005 }
13006
13007 Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
13008 return rcStrict2;
13009}
13010
13011/** @} */
13012
13013/** @name VM-exit exception handlers.
13014 * @{
13015 */
13016/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13017/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
13018/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13019
13020/**
13021 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
13022 */
13023static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13024{
13025 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13026 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13027
13028 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
13029 AssertRCReturn(rc, rc);
13030
13031 if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
13032 {
13033 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13034 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13035
13036 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13037 * provides VM-exit instruction length. If this causes problem later,
13038 * disassemble the instruction like it's done on AMD-V. */
13039 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13040 AssertRCReturn(rc2, rc2);
13041 return rc;
13042 }
13043
13044 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13045 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13046 return rc;
13047}
13048
13049
13050/**
13051 * VM-exit exception handler for \#BP (Breakpoint exception).
13052 */
13053static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13054{
13055 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13056 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13057
13058 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13059 AssertRCReturn(rc, rc);
13060
13061 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13062 rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13063 if (rc == VINF_EM_RAW_GUEST_TRAP)
13064 {
13065 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13066 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13067 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13068 AssertRCReturn(rc, rc);
13069
13070 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13071 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13072 }
13073
13074 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
13075 return rc;
13076}
13077
13078
13079/**
13080 * VM-exit exception handler for \#AC (alignment check exception).
13081 */
13082static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13083{
13084 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13085
13086 /*
13087 * Re-inject it. We'll detect any nesting before getting here.
13088 */
13089 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13090 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13091 AssertRCReturn(rc, rc);
13092 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13093
13094 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13095 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13096 return VINF_SUCCESS;
13097}
13098
13099
13100/**
13101 * VM-exit exception handler for \#DB (Debug exception).
13102 */
13103static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13104{
13105 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13106 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13107
13108 /*
13109 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13110 * for processing.
13111 */
13112 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13113
13114 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13115 uint64_t uDR6 = X86_DR6_INIT_VAL;
13116 uDR6 |= (pVmxTransient->uExitQual & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
13117
13118 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13119 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13120 Log6Func(("rc=%Rrc\n", rc));
13121 if (rc == VINF_EM_RAW_GUEST_TRAP)
13122 {
13123 /*
13124 * The exception was for the guest. Update DR6, DR7.GD and
13125 * IA32_DEBUGCTL.LBR before forwarding it.
13126 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13127 */
13128 VMMRZCallRing3Disable(pVCpu);
13129 HM_DISABLE_PREEMPT(pVCpu);
13130
13131 pCtx->dr[6] &= ~X86_DR6_B_MASK;
13132 pCtx->dr[6] |= uDR6;
13133 if (CPUMIsGuestDebugStateActive(pVCpu))
13134 ASMSetDR6(pCtx->dr[6]);
13135
13136 HM_RESTORE_PREEMPT();
13137 VMMRZCallRing3Enable(pVCpu);
13138
13139 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
13140 AssertRCReturn(rc, rc);
13141
13142 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13143 pCtx->dr[7] &= ~X86_DR7_GD;
13144
13145 /* Paranoia. */
13146 pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13147 pCtx->dr[7] |= X86_DR7_RA1_MASK;
13148
13149 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
13150 AssertRCReturn(rc, rc);
13151
13152 /*
13153 * Raise #DB in the guest.
13154 *
13155 * It is important to reflect exactly what the VM-exit gave us (preserving the
13156 * interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
13157 * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
13158 * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
13159 *
13160 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
13161 * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
13162 */
13163 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13164 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13165 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13166 AssertRCReturn(rc, rc);
13167 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13168 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13169 return VINF_SUCCESS;
13170 }
13171
13172 /*
13173 * Not a guest trap, must be a hypervisor related debug event then.
13174 * Update DR6 in case someone is interested in it.
13175 */
13176 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13177 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13178 CPUMSetHyperDR6(pVCpu, uDR6);
13179
13180 return rc;
13181}
13182
13183
13184/**
13185 * VM-exit exception handler for \#GP (General-protection exception).
13186 *
13187 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13188 */
13189static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13190{
13191 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13192 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13193
13194 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13195 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13196 { /* likely */ }
13197 else
13198 {
13199#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13200 Assert(pVCpu->hm.s.fUsingDebugLoop);
13201#endif
13202 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13203 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13204 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13205 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13206 rc |= hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13207 AssertRCReturn(rc, rc);
13208 Log4Func(("Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pCtx->cs.Sel, pCtx->rip,
13209 pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
13210 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13211 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13212 return rc;
13213 }
13214
13215 Assert(CPUMIsGuestInRealModeEx(pCtx));
13216 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13217
13218 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13219 AssertRCReturn(rc, rc);
13220
13221 VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
13222 if (rcStrict == VINF_SUCCESS)
13223 {
13224 if (!CPUMIsGuestInRealModeEx(pCtx))
13225 {
13226 /*
13227 * The guest is no longer in real-mode, check if we can continue executing the
13228 * guest using hardware-assisted VMX. Otherwise, fall back to emulation.
13229 */
13230 if (HMVmxCanExecuteGuest(pVCpu, pCtx))
13231 {
13232 Log4Func(("Mode changed but guest still suitable for executing using VT-x\n"));
13233 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
13234 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13235 }
13236 else
13237 {
13238 Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
13239 rcStrict = VINF_EM_RESCHEDULE;
13240 }
13241 }
13242 else
13243 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13244 }
13245 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13246 {
13247 rcStrict = VINF_SUCCESS;
13248 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13249 }
13250 return VBOXSTRICTRC_VAL(rcStrict);
13251}
13252
13253
13254/**
13255 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13256 * the exception reported in the VMX transient structure back into the VM.
13257 *
13258 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13259 * up-to-date.
13260 */
13261static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13262{
13263 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13264#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13265 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13266 ("uVector=%#x u32XcptBitmap=%#X32\n",
13267 VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13268#endif
13269
13270 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13271 hmR0VmxCheckExitDueToEventDelivery(). */
13272 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13273 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13274 AssertRCReturn(rc, rc);
13275 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13276
13277#ifdef DEBUG_ramshankar
13278 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
13279 uint8_t uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13280 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13281#endif
13282
13283 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13284 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13285 return VINF_SUCCESS;
13286}
13287
13288
13289/**
13290 * VM-exit exception handler for \#PF (Page-fault exception).
13291 */
13292static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13293{
13294 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13295 PVM pVM = pVCpu->CTX_SUFF(pVM);
13296 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13297 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13298 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13299 AssertRCReturn(rc, rc);
13300
13301 if (!pVM->hm.s.fNestedPaging)
13302 { /* likely */ }
13303 else
13304 {
13305#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13306 Assert(pVCpu->hm.s.fUsingDebugLoop);
13307#endif
13308 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13309 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13310 {
13311 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13312 pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
13313 }
13314 else
13315 {
13316 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13317 hmR0VmxSetPendingXcptDF(pVCpu);
13318 Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13319 }
13320 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13321 return rc;
13322 }
13323
13324 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13325 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13326 if (pVmxTransient->fVectoringPF)
13327 {
13328 Assert(pVCpu->hm.s.Event.fPending);
13329 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13330 }
13331
13332 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13333 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13334 AssertRCReturn(rc, rc);
13335
13336 Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
13337 pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
13338
13339 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13340 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
13341
13342 Log4Func(("#PF: rc=%Rrc\n", rc));
13343 if (rc == VINF_SUCCESS)
13344 {
13345 /*
13346 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13347 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13348 */
13349 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13350 TRPMResetTrap(pVCpu);
13351 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13352 return rc;
13353 }
13354
13355 if (rc == VINF_EM_RAW_GUEST_TRAP)
13356 {
13357 if (!pVmxTransient->fVectoringDoublePF)
13358 {
13359 /* It's a guest page fault and needs to be reflected to the guest. */
13360 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13361 TRPMResetTrap(pVCpu);
13362 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13363 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13364 uGstErrorCode, pVmxTransient->uExitQual);
13365 }
13366 else
13367 {
13368 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13369 TRPMResetTrap(pVCpu);
13370 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13371 hmR0VmxSetPendingXcptDF(pVCpu);
13372 Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13373 }
13374
13375 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13376 return VINF_SUCCESS;
13377 }
13378
13379 TRPMResetTrap(pVCpu);
13380 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13381 return rc;
13382}
13383
13384/** @} */
13385
13386#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13387/** @name Nested-guest VM-exit handlers.
13388 * @{
13389 */
13390/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13391/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= Nested-guest VM-exit handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13392/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13393
13394/**
13395 * VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
13396 */
13397HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13398{
13399 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13400
13401 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13402 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13403 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13404 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13405 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13406 AssertRCReturn(rc, rc);
13407
13408 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13409
13410 VMXVEXITINFO ExitInfo;
13411 RT_ZERO(ExitInfo);
13412 ExitInfo.uReason = pVmxTransient->uExitReason;
13413 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13414 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13415 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13416 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13417
13418 VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
13419 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13420 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13421 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13422 {
13423 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13424 rcStrict = VINF_SUCCESS;
13425 }
13426 return rcStrict;
13427}
13428
13429
13430/**
13431 * VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
13432 */
13433HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13434{
13435 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13436
13437 /** @todo NSTVMX: Vmlaunch. */
13438 hmR0VmxSetPendingXcptUD(pVCpu);
13439 return VINF_SUCCESS;
13440}
13441
13442
13443/**
13444 * VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
13445 */
13446HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13447{
13448 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13449
13450 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13451 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13452 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13453 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13454 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13455 AssertRCReturn(rc, rc);
13456
13457 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13458
13459 VMXVEXITINFO ExitInfo;
13460 RT_ZERO(ExitInfo);
13461 ExitInfo.uReason = pVmxTransient->uExitReason;
13462 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13463 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13464 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13465 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13466
13467 VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
13468 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13469 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13470 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13471 {
13472 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13473 rcStrict = VINF_SUCCESS;
13474 }
13475 return rcStrict;
13476}
13477
13478
13479/**
13480 * VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
13481 */
13482HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13483{
13484 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13485
13486 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13487 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13488 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13489 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13490 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13491 AssertRCReturn(rc, rc);
13492
13493 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13494
13495 VMXVEXITINFO ExitInfo;
13496 RT_ZERO(ExitInfo);
13497 ExitInfo.uReason = pVmxTransient->uExitReason;
13498 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13499 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13500 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13501 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
13502
13503 VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
13504 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13505 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13506 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13507 {
13508 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13509 rcStrict = VINF_SUCCESS;
13510 }
13511 return rcStrict;
13512}
13513
13514
13515/**
13516 * VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Unconditional VM-exit.
13517 */
13518HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13519{
13520 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13521
13522 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13523 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13524 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13525 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13526 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13527 AssertRCReturn(rc, rc);
13528
13529 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13530
13531 VMXVEXITINFO ExitInfo;
13532 RT_ZERO(ExitInfo);
13533 ExitInfo.uReason = pVmxTransient->uExitReason;
13534 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13535 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13536 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13537 if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
13538 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
13539
13540 VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
13541 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13542 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13543 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13544 {
13545 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13546 rcStrict = VINF_SUCCESS;
13547 }
13548 return rcStrict;
13549}
13550
13551
13552/**
13553 * VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
13554 */
13555HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13556{
13557 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13558
13559 /** @todo NSTVMX: Vmresume. */
13560 hmR0VmxSetPendingXcptUD(pVCpu);
13561 return VINF_SUCCESS;
13562}
13563
13564
13565/**
13566 * VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Unconditional VM-exit.
13567 */
13568HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13569{
13570 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13571
13572 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13573 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13574 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13575 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13576 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13577 AssertRCReturn(rc, rc);
13578
13579 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13580
13581 VMXVEXITINFO ExitInfo;
13582 RT_ZERO(ExitInfo);
13583 ExitInfo.uReason = pVmxTransient->uExitReason;
13584 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13585 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13586 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13587 if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
13588 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13589
13590 VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
13591 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13592 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13593 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13594 {
13595 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13596 rcStrict = VINF_SUCCESS;
13597 }
13598 return rcStrict;
13599}
13600
13601
13602/**
13603 * VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
13604 */
13605HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13606{
13607 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13608
13609 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13610 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13611 AssertRCReturn(rc, rc);
13612
13613 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13614
13615 VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
13616 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13617 {
13618 /* VMXOFF on success changes the internal hwvirt state but not anything that's visible to the guest. */
13619 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
13620 }
13621 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13622 {
13623 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13624 rcStrict = VINF_SUCCESS;
13625 }
13626 return rcStrict;
13627}
13628
13629
13630/**
13631 * VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
13632 */
13633HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13634{
13635 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13636
13637 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13638 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13639 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13640 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13641 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13642 AssertRCReturn(rc, rc);
13643
13644 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13645
13646 VMXVEXITINFO ExitInfo;
13647 RT_ZERO(ExitInfo);
13648 ExitInfo.uReason = pVmxTransient->uExitReason;
13649 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13650 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13651 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13652 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13653
13654 VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
13655 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13656 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13657 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13658 {
13659 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13660 rcStrict = VINF_SUCCESS;
13661 }
13662 return rcStrict;
13663}
13664
13665/** @} */
13666#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13667
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette