VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 74209

Last change on this file since 74209 was 74209, checked in by vboxsync, 6 years ago

IEM/VMX: Fix task switch triggered by INTn instruction (bugref:9244).
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 575.3 KB
Line 
1/* $Id: HMVMXR0.cpp 74209 2018-09-12 09:48:13Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/x86.h>
25#include <iprt/asm-amd64-x86.h>
26#include <iprt/thread.h>
27
28#include <VBox/vmm/pdmapi.h>
29#include <VBox/vmm/dbgf.h>
30#include <VBox/vmm/iem.h>
31#include <VBox/vmm/iom.h>
32#include <VBox/vmm/selm.h>
33#include <VBox/vmm/tm.h>
34#include <VBox/vmm/em.h>
35#include <VBox/vmm/gim.h>
36#include <VBox/vmm/apic.h>
37#ifdef VBOX_WITH_REM
38# include <VBox/vmm/rem.h>
39#endif
40#include "HMInternal.h"
41#include <VBox/vmm/vm.h>
42#include "HMVMXR0.h"
43#include "dtrace/VBoxVMM.h"
44
45#ifdef DEBUG_ramshankar
46# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
47# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
49# define HMVMX_ALWAYS_CHECK_GUEST_STATE
50# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
51# define HMVMX_ALWAYS_TRAP_PF
52# define HMVMX_ALWAYS_FLUSH_TLB
53# define HMVMX_ALWAYS_SWAP_EFER
54#endif
55
56
57/*********************************************************************************************************************************
58* Defined Constants And Macros *
59*********************************************************************************************************************************/
60/** Use the function table. */
61#define HMVMX_USE_FUNCTION_TABLE
62
63/** Determine which tagged-TLB flush handler to use. */
64#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
65#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
66#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
67#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
68
69/** @name HMVMX_READ_XXX
70 * Flags to skip redundant reads of some common VMCS fields that are not part of
71 * the guest-CPU or VCPU state but are needed while handling VM-exits.
72 */
73#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
74#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
75#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
76#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
77#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
78#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
79#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
80/** @} */
81
82/**
83 * States of the VMCS.
84 *
85 * This does not reflect all possible VMCS states but currently only those
86 * needed for maintaining the VMCS consistently even when thread-context hooks
87 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
88 */
89#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
90#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
91#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
92
93/**
94 * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
95 * guest using hardware-assisted VMX.
96 *
97 * This excludes state like GPRs (other than RSP) which are always are
98 * swapped and restored across the world-switch and also registers like EFER,
99 * MSR which cannot be modified by the guest without causing a VM-exit.
100 */
101#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
102 | CPUMCTX_EXTRN_RFLAGS \
103 | CPUMCTX_EXTRN_RSP \
104 | CPUMCTX_EXTRN_SREG_MASK \
105 | CPUMCTX_EXTRN_TABLE_MASK \
106 | CPUMCTX_EXTRN_KERNEL_GS_BASE \
107 | CPUMCTX_EXTRN_SYSCALL_MSRS \
108 | CPUMCTX_EXTRN_SYSENTER_MSRS \
109 | CPUMCTX_EXTRN_TSC_AUX \
110 | CPUMCTX_EXTRN_OTHER_MSRS \
111 | CPUMCTX_EXTRN_CR0 \
112 | CPUMCTX_EXTRN_CR3 \
113 | CPUMCTX_EXTRN_CR4 \
114 | CPUMCTX_EXTRN_DR7 \
115 | CPUMCTX_EXTRN_HM_VMX_MASK)
116
117/**
118 * Exception bitmap mask for real-mode guests (real-on-v86).
119 *
120 * We need to intercept all exceptions manually except:
121 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
122 * due to bugs in Intel CPUs.
123 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
124 * support.
125 */
126#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
127 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
128 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
129 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
130 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
131 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
132 | RT_BIT(X86_XCPT_XF))
133
134/** Maximum VM-instruction error number. */
135#define HMVMX_INSTR_ERROR_MAX 28
136
137/** Profiling macro. */
138#ifdef HM_PROFILE_EXIT_DISPATCH
139# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
140# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
141#else
142# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
143# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
144#endif
145
146/** Assert that preemption is disabled or covered by thread-context hooks. */
147#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
148 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
149
150/** Assert that we haven't migrated CPUs when thread-context hooks are not
151 * used. */
152#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
153 || (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
154 ("Illegal migration! Entered on CPU %u Current %u\n", \
155 (a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
156
157/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
158 * context. */
159#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
160 ("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
161 (a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
162
163/** Helper macro for VM-exit handlers called unexpectedly. */
164#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
165 do { \
166 (a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
167 return VERR_VMX_UNEXPECTED_EXIT; \
168 } while (0)
169
170/** Macro for importing segment registers to the VMCS from the guest-CPU context. */
171#ifdef VMX_USE_CACHED_VMCS_ACCESSES
172# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
173 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
174 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
175#else
176# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
177 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
178 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
179#endif
180
181/** Macro for exporting segment registers to the VMCS from the guest-CPU context. */
182#define HMVMX_EXPORT_SREG(Sel, a_pCtxSelReg) \
183 hmR0VmxExportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
184 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
185
186#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
187/** Macro that does the necessary privilege checks and intercepted VM-exits for
188 * guests that attempted to execute a VMX instruction. */
189# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
190 do \
191 { \
192 VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
193 if (rcStrictTmp == VINF_SUCCESS) \
194 { /* likely */ } \
195 else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
196 { \
197 Assert((a_pVCpu)->hm.s.Event.fPending); \
198 Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
199 return VINF_SUCCESS; \
200 } \
201 else \
202 { \
203 int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
204 AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
205 } \
206 } while (0)
207
208/** Macro that decodes a memory operand for an instruction VM-exit. */
209# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
210 do \
211 { \
212 VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
213 (a_pGCPtrEffAddr)); \
214 if (rcStrictTmp == VINF_SUCCESS) \
215 { /* likely */ } \
216 else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
217 { \
218 uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
219 Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
220 return VINF_SUCCESS; \
221 } \
222 else \
223 { \
224 Log4Func(("hmR0VmxCheckExitDueToVmxInstr failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
225 return rcStrictTmp; \
226 } \
227 } while (0)
228
229#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
230
231
232/*********************************************************************************************************************************
233* Structures and Typedefs *
234*********************************************************************************************************************************/
235/**
236 * VMX transient state.
237 *
238 * A state structure for holding miscellaneous information across
239 * VMX non-root operation and restored after the transition.
240 */
241typedef struct VMXTRANSIENT
242{
243 /** The host's rflags/eflags. */
244 RTCCUINTREG fEFlags;
245#if HC_ARCH_BITS == 32
246 uint32_t u32Alignment0;
247#endif
248 /** The guest's TPR value used for TPR shadowing. */
249 uint8_t u8GuestTpr;
250 /** Alignment. */
251 uint8_t abAlignment0[7];
252
253 /** The basic VM-exit reason. */
254 uint16_t uExitReason;
255 /** Alignment. */
256 uint16_t u16Alignment0;
257 /** The VM-exit interruption error code. */
258 uint32_t uExitIntErrorCode;
259 /** The VM-exit exit code qualification. */
260 uint64_t uExitQual;
261
262 /** The VM-exit interruption-information field. */
263 uint32_t uExitIntInfo;
264 /** The VM-exit instruction-length field. */
265 uint32_t cbInstr;
266 /** The VM-exit instruction-information field. */
267 VMXEXITINSTRINFO ExitInstrInfo;
268 /** Whether the VM-entry failed or not. */
269 bool fVMEntryFailed;
270 /** Alignment. */
271 uint8_t abAlignment1[3];
272
273 /** The VM-entry interruption-information field. */
274 uint32_t uEntryIntInfo;
275 /** The VM-entry exception error code field. */
276 uint32_t uEntryXcptErrorCode;
277 /** The VM-entry instruction length field. */
278 uint32_t cbEntryInstr;
279
280 /** IDT-vectoring information field. */
281 uint32_t uIdtVectoringInfo;
282 /** IDT-vectoring error code. */
283 uint32_t uIdtVectoringErrorCode;
284
285 /** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
286 uint32_t fVmcsFieldsRead;
287
288 /** Whether the guest debug state was active at the time of VM-exit. */
289 bool fWasGuestDebugStateActive;
290 /** Whether the hyper debug state was active at the time of VM-exit. */
291 bool fWasHyperDebugStateActive;
292 /** Whether TSC-offsetting should be setup before VM-entry. */
293 bool fUpdateTscOffsettingAndPreemptTimer;
294 /** Whether the VM-exit was caused by a page-fault during delivery of a
295 * contributory exception or a page-fault. */
296 bool fVectoringDoublePF;
297 /** Whether the VM-exit was caused by a page-fault during delivery of an
298 * external interrupt or NMI. */
299 bool fVectoringPF;
300} VMXTRANSIENT;
301AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
302AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
303AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
304AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
305AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
306/** Pointer to VMX transient state. */
307typedef VMXTRANSIENT *PVMXTRANSIENT;
308
309/**
310 * MSR-bitmap read permissions.
311 */
312typedef enum VMXMSREXITREAD
313{
314 /** Reading this MSR causes a VM-exit. */
315 VMXMSREXIT_INTERCEPT_READ = 0xb,
316 /** Reading this MSR does not cause a VM-exit. */
317 VMXMSREXIT_PASSTHRU_READ
318} VMXMSREXITREAD;
319/** Pointer to MSR-bitmap read permissions. */
320typedef VMXMSREXITREAD* PVMXMSREXITREAD;
321
322/**
323 * MSR-bitmap write permissions.
324 */
325typedef enum VMXMSREXITWRITE
326{
327 /** Writing to this MSR causes a VM-exit. */
328 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
329 /** Writing to this MSR does not cause a VM-exit. */
330 VMXMSREXIT_PASSTHRU_WRITE
331} VMXMSREXITWRITE;
332/** Pointer to MSR-bitmap write permissions. */
333typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
334
335/**
336 * Memory operand read or write access.
337 */
338typedef enum VMXMEMACCESS
339{
340 VMXMEMACCESS_READ = 0,
341 VMXMEMACCESS_WRITE = 1
342} VMXMEMACCESS;
343
344/**
345 * VMX VM-exit handler.
346 *
347 * @returns Strict VBox status code (i.e. informational status codes too).
348 * @param pVCpu The cross context virtual CPU structure.
349 * @param pVmxTransient Pointer to the VMX-transient structure.
350 */
351#ifndef HMVMX_USE_FUNCTION_TABLE
352typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
353#else
354typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
355/** Pointer to VM-exit handler. */
356typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
357#endif
358
359/**
360 * VMX VM-exit handler, non-strict status code.
361 *
362 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
363 *
364 * @returns VBox status code, no informational status code returned.
365 * @param pVCpu The cross context virtual CPU structure.
366 * @param pVmxTransient Pointer to the VMX-transient structure.
367 *
368 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
369 * use of that status code will be replaced with VINF_EM_SOMETHING
370 * later when switching over to IEM.
371 */
372#ifndef HMVMX_USE_FUNCTION_TABLE
373typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
374#else
375typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
376#endif
377
378
379/*********************************************************************************************************************************
380* Internal Functions *
381*********************************************************************************************************************************/
382static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush);
383static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr);
384static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
385static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat);
386static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
387 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState);
388#if HC_ARCH_BITS == 32
389static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
390#endif
391#ifndef HMVMX_USE_FUNCTION_TABLE
392DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
393# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
394# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
395#else
396# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
397# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
398#endif
399
400/** @name VM-exit handlers.
401 * @{
402 */
403static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
404static FNVMXEXITHANDLER hmR0VmxExitExtInt;
405static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
406static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
407static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
408static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
409static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
410static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
411static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
412static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
413static FNVMXEXITHANDLER hmR0VmxExitCpuid;
414static FNVMXEXITHANDLER hmR0VmxExitGetsec;
415static FNVMXEXITHANDLER hmR0VmxExitHlt;
416static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
417static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
418static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
419static FNVMXEXITHANDLER hmR0VmxExitVmcall;
420#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
421static FNVMXEXITHANDLER hmR0VmxExitVmclear;
422static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
423static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
424static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
425static FNVMXEXITHANDLER hmR0VmxExitVmread;
426static FNVMXEXITHANDLER hmR0VmxExitVmresume;
427static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
428static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
429static FNVMXEXITHANDLER hmR0VmxExitVmxon;
430#endif
431static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
432static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
433static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
434static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
435static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
436static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
437static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
438static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
439static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
440static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
441static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
442static FNVMXEXITHANDLER hmR0VmxExitMwait;
443static FNVMXEXITHANDLER hmR0VmxExitMtf;
444static FNVMXEXITHANDLER hmR0VmxExitMonitor;
445static FNVMXEXITHANDLER hmR0VmxExitPause;
446static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
447static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
448static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
449static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
450static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
451static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
452static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
453static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
454static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
455static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
456static FNVMXEXITHANDLER hmR0VmxExitRdrand;
457static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
458/** @} */
459
460static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
461static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
462static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
463static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
464static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
465static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
466static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
467static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu);
468
469
470/*********************************************************************************************************************************
471* Global Variables *
472*********************************************************************************************************************************/
473#ifdef HMVMX_USE_FUNCTION_TABLE
474
475/**
476 * VMX_EXIT dispatch table.
477 */
478static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
479{
480 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
481 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
482 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
483 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
484 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
485 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
486 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
487 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
488 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
489 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
490 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
491 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
492 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
493 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
494 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
495 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
496 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
497 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
498 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
499#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
500 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
501 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
502 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
503 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
504 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
505 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
506 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
507 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
508 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
509#else
510 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
511 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
512 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
513 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
514 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
515 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
516 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
517 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
518 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
519#endif
520 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
521 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
522 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
523 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
524 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
525 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
526 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
527 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
528 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
529 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
530 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
531 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
532 /* 40 UNDEFINED */ hmR0VmxExitPause,
533 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
534 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
535 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
536 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
537 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
538 /* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitXdtrAccess,
539 /* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitXdtrAccess,
540 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
541 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
542 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
543 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
544 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
545 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
546 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
547 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
548 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
549 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
550 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
551 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
552 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
553 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
554 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
555 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
556 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
557};
558#endif /* HMVMX_USE_FUNCTION_TABLE */
559
560#ifdef VBOX_STRICT
561static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
562{
563 /* 0 */ "(Not Used)",
564 /* 1 */ "VMCALL executed in VMX root operation.",
565 /* 2 */ "VMCLEAR with invalid physical address.",
566 /* 3 */ "VMCLEAR with VMXON pointer.",
567 /* 4 */ "VMLAUNCH with non-clear VMCS.",
568 /* 5 */ "VMRESUME with non-launched VMCS.",
569 /* 6 */ "VMRESUME after VMXOFF",
570 /* 7 */ "VM-entry with invalid control fields.",
571 /* 8 */ "VM-entry with invalid host state fields.",
572 /* 9 */ "VMPTRLD with invalid physical address.",
573 /* 10 */ "VMPTRLD with VMXON pointer.",
574 /* 11 */ "VMPTRLD with incorrect revision identifier.",
575 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
576 /* 13 */ "VMWRITE to read-only VMCS component.",
577 /* 14 */ "(Not Used)",
578 /* 15 */ "VMXON executed in VMX root operation.",
579 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
580 /* 17 */ "VM-entry with non-launched executing VMCS.",
581 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
582 /* 19 */ "VMCALL with non-clear VMCS.",
583 /* 20 */ "VMCALL with invalid VM-exit control fields.",
584 /* 21 */ "(Not Used)",
585 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
586 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
587 /* 24 */ "VMCALL with invalid SMM-monitor features.",
588 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
589 /* 26 */ "VM-entry with events blocked by MOV SS.",
590 /* 27 */ "(Not Used)",
591 /* 28 */ "Invalid operand to INVEPT/INVVPID."
592};
593#endif /* VBOX_STRICT */
594
595
596/**
597 * Updates the VM's last error record.
598 *
599 * If there was a VMX instruction error, reads the error data from the VMCS and
600 * updates VCPU's last error record as well.
601 *
602 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
603 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
604 * VERR_VMX_INVALID_VMCS_FIELD.
605 * @param rc The error code.
606 */
607static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
608{
609 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
610 || rc == VERR_VMX_UNABLE_TO_START_VM)
611 {
612 AssertPtrReturnVoid(pVCpu);
613 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
614 }
615 pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
616}
617
618
619/**
620 * Reads the VM-entry interruption-information field from the VMCS into the VMX
621 * transient structure.
622 *
623 * @returns VBox status code.
624 * @param pVmxTransient Pointer to the VMX transient structure.
625 *
626 * @remarks No-long-jump zone!!!
627 */
628DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
629{
630 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
631 AssertRCReturn(rc, rc);
632 return VINF_SUCCESS;
633}
634
635#ifdef VBOX_STRICT
636/**
637 * Reads the VM-entry exception error code field from the VMCS into
638 * the VMX transient structure.
639 *
640 * @returns VBox status code.
641 * @param pVmxTransient Pointer to the VMX transient structure.
642 *
643 * @remarks No-long-jump zone!!!
644 */
645DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
646{
647 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
648 AssertRCReturn(rc, rc);
649 return VINF_SUCCESS;
650}
651
652
653/**
654 * Reads the VM-entry exception error code field from the VMCS into
655 * the VMX transient structure.
656 *
657 * @returns VBox status code.
658 * @param pVmxTransient Pointer to the VMX transient structure.
659 *
660 * @remarks No-long-jump zone!!!
661 */
662DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
663{
664 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
665 AssertRCReturn(rc, rc);
666 return VINF_SUCCESS;
667}
668#endif /* VBOX_STRICT */
669
670
671/**
672 * Reads the VM-exit interruption-information field from the VMCS into the VMX
673 * transient structure.
674 *
675 * @returns VBox status code.
676 * @param pVmxTransient Pointer to the VMX transient structure.
677 */
678DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
679{
680 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
681 {
682 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
683 AssertRCReturn(rc,rc);
684 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_INFO;
685 }
686 return VINF_SUCCESS;
687}
688
689
690/**
691 * Reads the VM-exit interruption error code from the VMCS into the VMX
692 * transient structure.
693 *
694 * @returns VBox status code.
695 * @param pVmxTransient Pointer to the VMX transient structure.
696 */
697DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
698{
699 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
700 {
701 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
702 AssertRCReturn(rc, rc);
703 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
704 }
705 return VINF_SUCCESS;
706}
707
708
709/**
710 * Reads the VM-exit instruction length field from the VMCS into the VMX
711 * transient structure.
712 *
713 * @returns VBox status code.
714 * @param pVmxTransient Pointer to the VMX transient structure.
715 */
716DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
717{
718 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
719 {
720 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
721 AssertRCReturn(rc, rc);
722 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_LEN;
723 }
724 return VINF_SUCCESS;
725}
726
727
728/**
729 * Reads the VM-exit instruction-information field from the VMCS into
730 * the VMX transient structure.
731 *
732 * @returns VBox status code.
733 * @param pVmxTransient Pointer to the VMX transient structure.
734 */
735DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
736{
737 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
738 {
739 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
740 AssertRCReturn(rc, rc);
741 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_INFO;
742 }
743 return VINF_SUCCESS;
744}
745
746
747/**
748 * Reads the exit code qualification from the VMCS into the VMX transient
749 * structure.
750 *
751 * @returns VBox status code.
752 * @param pVCpu The cross context virtual CPU structure of the
753 * calling EMT. (Required for the VMCS cache case.)
754 * @param pVmxTransient Pointer to the VMX transient structure.
755 */
756DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
757{
758 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
759 {
760 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
761 AssertRCReturn(rc, rc);
762 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION;
763 }
764 return VINF_SUCCESS;
765}
766
767
768/**
769 * Reads the IDT-vectoring information field from the VMCS into the VMX
770 * transient structure.
771 *
772 * @returns VBox status code.
773 * @param pVmxTransient Pointer to the VMX transient structure.
774 *
775 * @remarks No-long-jump zone!!!
776 */
777DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
778{
779 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
780 {
781 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
782 AssertRCReturn(rc, rc);
783 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_INFO;
784 }
785 return VINF_SUCCESS;
786}
787
788
789/**
790 * Reads the IDT-vectoring error code from the VMCS into the VMX
791 * transient structure.
792 *
793 * @returns VBox status code.
794 * @param pVmxTransient Pointer to the VMX transient structure.
795 */
796DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
797{
798 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
799 {
800 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
801 AssertRCReturn(rc, rc);
802 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
803 }
804 return VINF_SUCCESS;
805}
806
807
808/**
809 * Enters VMX root mode operation on the current CPU.
810 *
811 * @returns VBox status code.
812 * @param pVM The cross context VM structure. Can be
813 * NULL, after a resume.
814 * @param HCPhysCpuPage Physical address of the VMXON region.
815 * @param pvCpuPage Pointer to the VMXON region.
816 */
817static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
818{
819 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
820 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
821 Assert(pvCpuPage);
822 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
823
824 if (pVM)
825 {
826 /* Write the VMCS revision dword to the VMXON region. */
827 *(uint32_t *)pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
828 }
829
830 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
831 RTCCUINTREG fEFlags = ASMIntDisableFlags();
832
833 /* Enable the VMX bit in CR4 if necessary. */
834 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
835
836 /* Enter VMX root mode. */
837 int rc = VMXEnable(HCPhysCpuPage);
838 if (RT_FAILURE(rc))
839 {
840 if (!(uOldCr4 & X86_CR4_VMXE))
841 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
842
843 if (pVM)
844 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
845 }
846
847 /* Restore interrupts. */
848 ASMSetFlags(fEFlags);
849 return rc;
850}
851
852
853/**
854 * Exits VMX root mode operation on the current CPU.
855 *
856 * @returns VBox status code.
857 */
858static int hmR0VmxLeaveRootMode(void)
859{
860 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
861
862 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
863 RTCCUINTREG fEFlags = ASMIntDisableFlags();
864
865 /* If we're for some reason not in VMX root mode, then don't leave it. */
866 RTCCUINTREG uHostCR4 = ASMGetCR4();
867
868 int rc;
869 if (uHostCR4 & X86_CR4_VMXE)
870 {
871 /* Exit VMX root mode and clear the VMX bit in CR4. */
872 VMXDisable();
873 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
874 rc = VINF_SUCCESS;
875 }
876 else
877 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
878
879 /* Restore interrupts. */
880 ASMSetFlags(fEFlags);
881 return rc;
882}
883
884
885/**
886 * Allocates and maps one physically contiguous page. The allocated page is
887 * zero'd out. (Used by various VT-x structures).
888 *
889 * @returns IPRT status code.
890 * @param pMemObj Pointer to the ring-0 memory object.
891 * @param ppVirt Where to store the virtual address of the
892 * allocation.
893 * @param pHCPhys Where to store the physical address of the
894 * allocation.
895 */
896static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
897{
898 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
899 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
900 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
901
902 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
903 if (RT_FAILURE(rc))
904 return rc;
905 *ppVirt = RTR0MemObjAddress(*pMemObj);
906 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
907 ASMMemZero32(*ppVirt, PAGE_SIZE);
908 return VINF_SUCCESS;
909}
910
911
912/**
913 * Frees and unmaps an allocated physical page.
914 *
915 * @param pMemObj Pointer to the ring-0 memory object.
916 * @param ppVirt Where to re-initialize the virtual address of
917 * allocation as 0.
918 * @param pHCPhys Where to re-initialize the physical address of the
919 * allocation as 0.
920 */
921static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
922{
923 AssertPtr(pMemObj);
924 AssertPtr(ppVirt);
925 AssertPtr(pHCPhys);
926 if (*pMemObj != NIL_RTR0MEMOBJ)
927 {
928 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
929 AssertRC(rc);
930 *pMemObj = NIL_RTR0MEMOBJ;
931 *ppVirt = 0;
932 *pHCPhys = 0;
933 }
934}
935
936
937/**
938 * Worker function to free VT-x related structures.
939 *
940 * @returns IPRT status code.
941 * @param pVM The cross context VM structure.
942 */
943static void hmR0VmxStructsFree(PVM pVM)
944{
945 for (VMCPUID i = 0; i < pVM->cCpus; i++)
946 {
947 PVMCPU pVCpu = &pVM->aCpus[i];
948 AssertPtr(pVCpu);
949
950 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
951 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
952
953 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
954 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
955
956 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
957 }
958
959 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
960#ifdef VBOX_WITH_CRASHDUMP_MAGIC
961 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
962#endif
963}
964
965
966/**
967 * Worker function to allocate VT-x related VM structures.
968 *
969 * @returns IPRT status code.
970 * @param pVM The cross context VM structure.
971 */
972static int hmR0VmxStructsAlloc(PVM pVM)
973{
974 /*
975 * Initialize members up-front so we can cleanup properly on allocation failure.
976 */
977#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
978 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
979 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
980 pVM->hm.s.vmx.HCPhys##a_Name = 0;
981
982#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
983 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
984 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
985 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
986
987#ifdef VBOX_WITH_CRASHDUMP_MAGIC
988 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
989#endif
990 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
991
992 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
993 for (VMCPUID i = 0; i < pVM->cCpus; i++)
994 {
995 PVMCPU pVCpu = &pVM->aCpus[i];
996 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
997 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
998 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
999 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
1000 }
1001#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
1002#undef VMXLOCAL_INIT_VM_MEMOBJ
1003
1004 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
1005 AssertReturnStmt(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE) <= PAGE_SIZE,
1006 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
1007 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
1008
1009 /*
1010 * Allocate all the VT-x structures.
1011 */
1012 int rc = VINF_SUCCESS;
1013#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1014 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1015 if (RT_FAILURE(rc))
1016 goto cleanup;
1017 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1018 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1019#endif
1020
1021 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1022 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1023 {
1024 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1025 &pVM->hm.s.vmx.HCPhysApicAccess);
1026 if (RT_FAILURE(rc))
1027 goto cleanup;
1028 }
1029
1030 /*
1031 * Initialize per-VCPU VT-x structures.
1032 */
1033 for (VMCPUID i = 0; i < pVM->cCpus; i++)
1034 {
1035 PVMCPU pVCpu = &pVM->aCpus[i];
1036 AssertPtr(pVCpu);
1037
1038 /* Allocate the VM control structure (VMCS). */
1039 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1040 if (RT_FAILURE(rc))
1041 goto cleanup;
1042
1043 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1044 if ( PDMHasApic(pVM)
1045 && (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1046 {
1047 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1048 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1049 if (RT_FAILURE(rc))
1050 goto cleanup;
1051 }
1052
1053 /*
1054 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1055 * transparent accesses of specific MSRs.
1056 *
1057 * If the condition for enabling MSR bitmaps changes here, don't forget to
1058 * update HMAreMsrBitmapsAvailable().
1059 */
1060 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1061 {
1062 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1063 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1064 if (RT_FAILURE(rc))
1065 goto cleanup;
1066 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1067 }
1068
1069 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1070 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1071 if (RT_FAILURE(rc))
1072 goto cleanup;
1073
1074 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1075 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1076 if (RT_FAILURE(rc))
1077 goto cleanup;
1078 }
1079
1080 return VINF_SUCCESS;
1081
1082cleanup:
1083 hmR0VmxStructsFree(pVM);
1084 return rc;
1085}
1086
1087
1088/**
1089 * Does global VT-x initialization (called during module initialization).
1090 *
1091 * @returns VBox status code.
1092 */
1093VMMR0DECL(int) VMXR0GlobalInit(void)
1094{
1095#ifdef HMVMX_USE_FUNCTION_TABLE
1096 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1097# ifdef VBOX_STRICT
1098 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1099 Assert(g_apfnVMExitHandlers[i]);
1100# endif
1101#endif
1102 return VINF_SUCCESS;
1103}
1104
1105
1106/**
1107 * Does global VT-x termination (called during module termination).
1108 */
1109VMMR0DECL(void) VMXR0GlobalTerm()
1110{
1111 /* Nothing to do currently. */
1112}
1113
1114
1115/**
1116 * Sets up and activates VT-x on the current CPU.
1117 *
1118 * @returns VBox status code.
1119 * @param pHostCpu Pointer to the global CPU info struct.
1120 * @param pVM The cross context VM structure. Can be
1121 * NULL after a host resume operation.
1122 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1123 * fEnabledByHost is @c true).
1124 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1125 * @a fEnabledByHost is @c true).
1126 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1127 * enable VT-x on the host.
1128 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1129 */
1130VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1131 void *pvMsrs)
1132{
1133 Assert(pHostCpu);
1134 Assert(pvMsrs);
1135 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1136
1137 /* Enable VT-x if it's not already enabled by the host. */
1138 if (!fEnabledByHost)
1139 {
1140 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1141 if (RT_FAILURE(rc))
1142 return rc;
1143 }
1144
1145 /*
1146 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
1147 * using EPTPs) so we don't retain any stale guest-physical mappings which won't get
1148 * invalidated when flushing by VPID.
1149 */
1150 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1151 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1152 {
1153 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
1154 pHostCpu->fFlushAsidBeforeUse = false;
1155 }
1156 else
1157 pHostCpu->fFlushAsidBeforeUse = true;
1158
1159 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1160 ++pHostCpu->cTlbFlushes;
1161
1162 return VINF_SUCCESS;
1163}
1164
1165
1166/**
1167 * Deactivates VT-x on the current CPU.
1168 *
1169 * @returns VBox status code.
1170 * @param pHostCpu Pointer to the global CPU info struct.
1171 * @param pvCpuPage Pointer to the VMXON region.
1172 * @param HCPhysCpuPage Physical address of the VMXON region.
1173 *
1174 * @remarks This function should never be called when SUPR0EnableVTx() or
1175 * similar was used to enable VT-x on the host.
1176 */
1177VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1178{
1179 RT_NOREF3(pHostCpu, pvCpuPage, HCPhysCpuPage);
1180
1181 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1182 return hmR0VmxLeaveRootMode();
1183}
1184
1185
1186/**
1187 * Sets the permission bits for the specified MSR in the MSR bitmap.
1188 *
1189 * @param pVCpu The cross context virtual CPU structure.
1190 * @param uMsr The MSR value.
1191 * @param enmRead Whether reading this MSR causes a VM-exit.
1192 * @param enmWrite Whether writing this MSR causes a VM-exit.
1193 */
1194static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1195{
1196 int32_t iBit;
1197 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1198
1199 /*
1200 * Layout:
1201 * 0x000 - 0x3ff - Low MSR read bits
1202 * 0x400 - 0x7ff - High MSR read bits
1203 * 0x800 - 0xbff - Low MSR write bits
1204 * 0xc00 - 0xfff - High MSR write bits
1205 */
1206 if (uMsr <= 0x00001fff)
1207 iBit = uMsr;
1208 else if (uMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1209 {
1210 iBit = uMsr - UINT32_C(0xc0000000);
1211 pbMsrBitmap += 0x400;
1212 }
1213 else
1214 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1215
1216 Assert(iBit <= 0x1fff);
1217 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1218 ASMBitSet(pbMsrBitmap, iBit);
1219 else
1220 ASMBitClear(pbMsrBitmap, iBit);
1221
1222 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1223 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1224 else
1225 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1226}
1227
1228
1229#ifdef VBOX_STRICT
1230/**
1231 * Gets the permission bits for the specified MSR in the MSR bitmap.
1232 *
1233 * @returns VBox status code.
1234 * @retval VINF_SUCCESS if the specified MSR is found.
1235 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1236 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1237 *
1238 * @param pVCpu The cross context virtual CPU structure.
1239 * @param uMsr The MSR.
1240 * @param penmRead Where to store the read permissions.
1241 * @param penmWrite Where to store the write permissions.
1242 */
1243static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1244{
1245 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1246 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1247 int32_t iBit;
1248 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1249
1250 /* See hmR0VmxSetMsrPermission() for the layout. */
1251 if (uMsr <= 0x00001fff)
1252 iBit = uMsr;
1253 else if ( uMsr >= 0xc0000000
1254 && uMsr <= 0xc0001fff)
1255 {
1256 iBit = (uMsr - 0xc0000000);
1257 pbMsrBitmap += 0x400;
1258 }
1259 else
1260 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1261
1262 Assert(iBit <= 0x1fff);
1263 if (ASMBitTest(pbMsrBitmap, iBit))
1264 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1265 else
1266 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1267
1268 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1269 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1270 else
1271 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1272 return VINF_SUCCESS;
1273}
1274#endif /* VBOX_STRICT */
1275
1276
1277/**
1278 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1279 * area.
1280 *
1281 * @returns VBox status code.
1282 * @param pVCpu The cross context virtual CPU structure.
1283 * @param cMsrs The number of MSRs.
1284 */
1285static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1286{
1287 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1288 uint64_t const uVmxMiscMsr = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc;
1289 uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(uVmxMiscMsr);
1290 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1291 {
1292 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1293 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1294 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1295 }
1296
1297 /* Update number of guest MSRs to load/store across the world-switch. */
1298 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1299 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1300
1301 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1302 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1303 AssertRCReturn(rc, rc);
1304
1305 /* Update the VCPU's copy of the MSR count. */
1306 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1307
1308 return VINF_SUCCESS;
1309}
1310
1311
1312/**
1313 * Adds a new (or updates the value of an existing) guest/host MSR
1314 * pair to be swapped during the world-switch as part of the
1315 * auto-load/store MSR area in the VMCS.
1316 *
1317 * @returns VBox status code.
1318 * @param pVCpu The cross context virtual CPU structure.
1319 * @param uMsr The MSR.
1320 * @param uGuestMsrValue Value of the guest MSR.
1321 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1322 * necessary.
1323 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1324 * its value was updated. Optional, can be NULL.
1325 */
1326static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1327 bool *pfAddedAndUpdated)
1328{
1329 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1330 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1331 uint32_t i;
1332 for (i = 0; i < cMsrs; i++)
1333 {
1334 if (pGuestMsr->u32Msr == uMsr)
1335 break;
1336 pGuestMsr++;
1337 }
1338
1339 bool fAdded = false;
1340 if (i == cMsrs)
1341 {
1342 ++cMsrs;
1343 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1344 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1345
1346 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1347 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1348 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1349
1350 fAdded = true;
1351 }
1352
1353 /* Update the MSR values in the auto-load/store MSR area. */
1354 pGuestMsr->u32Msr = uMsr;
1355 pGuestMsr->u64Value = uGuestMsrValue;
1356
1357 /* Create/update the MSR slot in the host MSR area. */
1358 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1359 pHostMsr += i;
1360 pHostMsr->u32Msr = uMsr;
1361
1362 /*
1363 * Update the host MSR only when requested by the caller AND when we're
1364 * adding it to the auto-load/store area. Otherwise, it would have been
1365 * updated by hmR0VmxExportHostMsrs(). We do this for performance reasons.
1366 */
1367 bool fUpdatedMsrValue = false;
1368 if ( fAdded
1369 && fUpdateHostMsr)
1370 {
1371 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1372 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1373 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1374 fUpdatedMsrValue = true;
1375 }
1376
1377 if (pfAddedAndUpdated)
1378 *pfAddedAndUpdated = fUpdatedMsrValue;
1379 return VINF_SUCCESS;
1380}
1381
1382
1383/**
1384 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1385 * auto-load/store MSR area in the VMCS.
1386 *
1387 * @returns VBox status code.
1388 * @param pVCpu The cross context virtual CPU structure.
1389 * @param uMsr The MSR.
1390 */
1391static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1392{
1393 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1394 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1395 for (uint32_t i = 0; i < cMsrs; i++)
1396 {
1397 /* Find the MSR. */
1398 if (pGuestMsr->u32Msr == uMsr)
1399 {
1400 /* If it's the last MSR, simply reduce the count. */
1401 if (i == cMsrs - 1)
1402 {
1403 --cMsrs;
1404 break;
1405 }
1406
1407 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1408 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1409 pLastGuestMsr += cMsrs - 1;
1410 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1411 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1412
1413 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1414 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1415 pLastHostMsr += cMsrs - 1;
1416 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1417 pHostMsr->u64Value = pLastHostMsr->u64Value;
1418 --cMsrs;
1419 break;
1420 }
1421 pGuestMsr++;
1422 }
1423
1424 /* Update the VMCS if the count changed (meaning the MSR was found). */
1425 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1426 {
1427 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1428 AssertRCReturn(rc, rc);
1429
1430 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1431 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1432 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1433
1434 Log4Func(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1435 return VINF_SUCCESS;
1436 }
1437
1438 return VERR_NOT_FOUND;
1439}
1440
1441
1442/**
1443 * Checks if the specified guest MSR is part of the auto-load/store area in
1444 * the VMCS.
1445 *
1446 * @returns true if found, false otherwise.
1447 * @param pVCpu The cross context virtual CPU structure.
1448 * @param uMsr The MSR to find.
1449 */
1450static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1451{
1452 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1453 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1454
1455 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1456 {
1457 if (pGuestMsr->u32Msr == uMsr)
1458 return true;
1459 }
1460 return false;
1461}
1462
1463
1464/**
1465 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1466 *
1467 * @param pVCpu The cross context virtual CPU structure.
1468 *
1469 * @remarks No-long-jump zone!!!
1470 */
1471static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1472{
1473 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1474 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1475 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1476 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1477
1478 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1479 {
1480 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1481
1482 /*
1483 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1484 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1485 */
1486 if (pHostMsr->u32Msr == MSR_K6_EFER)
1487 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1488 else
1489 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1490 }
1491
1492 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1493}
1494
1495
1496/**
1497 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1498 * perform lazy restoration of the host MSRs while leaving VT-x.
1499 *
1500 * @param pVCpu The cross context virtual CPU structure.
1501 *
1502 * @remarks No-long-jump zone!!!
1503 */
1504static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1505{
1506 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1507
1508 /*
1509 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1510 */
1511 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1512 {
1513 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1514#if HC_ARCH_BITS == 64
1515 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1516 {
1517 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1518 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1519 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1520 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1521 }
1522#endif
1523 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1524 }
1525}
1526
1527
1528/**
1529 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1530 * lazily while leaving VT-x.
1531 *
1532 * @returns true if it does, false otherwise.
1533 * @param pVCpu The cross context virtual CPU structure.
1534 * @param uMsr The MSR to check.
1535 */
1536static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1537{
1538 NOREF(pVCpu);
1539#if HC_ARCH_BITS == 64
1540 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1541 {
1542 switch (uMsr)
1543 {
1544 case MSR_K8_LSTAR:
1545 case MSR_K6_STAR:
1546 case MSR_K8_SF_MASK:
1547 case MSR_K8_KERNEL_GS_BASE:
1548 return true;
1549 }
1550 }
1551#else
1552 RT_NOREF(pVCpu, uMsr);
1553#endif
1554 return false;
1555}
1556
1557
1558/**
1559 * Loads a set of guests MSRs to allow read/passthru to the guest.
1560 *
1561 * The name of this function is slightly confusing. This function does NOT
1562 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1563 * common prefix for functions dealing with "lazy restoration" of the shared
1564 * MSRs.
1565 *
1566 * @param pVCpu The cross context virtual CPU structure.
1567 *
1568 * @remarks No-long-jump zone!!!
1569 */
1570static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1571{
1572 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1573 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1574
1575 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1576#if HC_ARCH_BITS == 64
1577 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1578 {
1579 /*
1580 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1581 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1582 * we can skip a few MSR writes.
1583 *
1584 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1585 * guest MSR values in the guest-CPU context might be different to what's currently
1586 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1587 * CPU, see @bugref{8728}.
1588 */
1589 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1590 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1591 && pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1592 && pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1593 && pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1594 && pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1595 {
1596#ifdef VBOX_STRICT
1597 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1598 Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1599 Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
1600 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
1601#endif
1602 }
1603 else
1604 {
1605 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1606 ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
1607 ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
1608 ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
1609 }
1610 }
1611#endif
1612 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1613}
1614
1615
1616/**
1617 * Performs lazy restoration of the set of host MSRs if they were previously
1618 * loaded with guest MSR values.
1619 *
1620 * @param pVCpu The cross context virtual CPU structure.
1621 *
1622 * @remarks No-long-jump zone!!!
1623 * @remarks The guest MSRs should have been saved back into the guest-CPU
1624 * context by hmR0VmxImportGuestState()!!!
1625 */
1626static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1627{
1628 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1629 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1630
1631 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1632 {
1633 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1634#if HC_ARCH_BITS == 64
1635 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1636 {
1637 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1638 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1639 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1640 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1641 }
1642#endif
1643 }
1644 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1645}
1646
1647
1648/**
1649 * Verifies that our cached values of the VMCS fields are all consistent with
1650 * what's actually present in the VMCS.
1651 *
1652 * @returns VBox status code.
1653 * @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
1654 * @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
1655 * VMCS content. HMCPU error-field is
1656 * updated, see VMX_VCI_XXX.
1657 * @param pVCpu The cross context virtual CPU structure.
1658 */
1659static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1660{
1661 uint32_t u32Val;
1662 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1663 AssertRCReturn(rc, rc);
1664 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32EntryCtls == u32Val,
1665 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1666 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
1667 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1668
1669 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1670 AssertRCReturn(rc, rc);
1671 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ExitCtls == u32Val,
1672 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1673 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
1674 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1675
1676 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1677 AssertRCReturn(rc, rc);
1678 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32PinCtls == u32Val,
1679 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1680 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
1681 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1682
1683 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1684 AssertRCReturn(rc, rc);
1685 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls == u32Val,
1686 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1687 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
1688 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1689
1690 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
1691 {
1692 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1693 AssertRCReturn(rc, rc);
1694 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1695 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1696 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
1697 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1698 }
1699
1700 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
1701 AssertRCReturn(rc, rc);
1702 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32XcptBitmap == u32Val,
1703 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap, u32Val),
1704 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
1705 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1706
1707 uint64_t u64Val;
1708 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
1709 AssertRCReturn(rc, rc);
1710 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u64TscOffset == u64Val,
1711 ("Cache=%#RX64 VMCS=%#RX64\n", pVCpu->hm.s.vmx.u64TscOffset, u64Val),
1712 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
1713 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1714
1715 return VINF_SUCCESS;
1716}
1717
1718
1719#ifdef VBOX_STRICT
1720/**
1721 * Verifies that our cached host EFER value has not changed
1722 * since we cached it.
1723 *
1724 * @param pVCpu The cross context virtual CPU structure.
1725 */
1726static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1727{
1728 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1729
1730 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
1731 {
1732 uint64_t u64Val;
1733 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1734 AssertRC(rc);
1735
1736 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1737 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1738 }
1739}
1740
1741
1742/**
1743 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1744 * VMCS are correct.
1745 *
1746 * @param pVCpu The cross context virtual CPU structure.
1747 */
1748static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1749{
1750 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1751
1752 /* Verify MSR counts in the VMCS are what we think it should be. */
1753 uint32_t cMsrs;
1754 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1755 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1756
1757 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1758 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1759
1760 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1761 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1762
1763 PCVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1764 PCVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1765 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1766 {
1767 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1768 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1769 pGuestMsr->u32Msr, cMsrs));
1770
1771 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1772 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1773 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1774
1775 /* Verify that the permissions are as expected in the MSR bitmap. */
1776 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1777 {
1778 VMXMSREXITREAD enmRead;
1779 VMXMSREXITWRITE enmWrite;
1780 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1781 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1782 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1783 {
1784 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1785 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1786 }
1787 else
1788 {
1789 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1790 pGuestMsr->u32Msr, cMsrs));
1791 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1792 pGuestMsr->u32Msr, cMsrs));
1793 }
1794 }
1795 }
1796}
1797#endif /* VBOX_STRICT */
1798
1799
1800/**
1801 * Flushes the TLB using EPT.
1802 *
1803 * @returns VBox status code.
1804 * @param pVCpu The cross context virtual CPU structure of the calling
1805 * EMT. Can be NULL depending on @a enmTlbFlush.
1806 * @param enmTlbFlush Type of flush.
1807 *
1808 * @remarks Caller is responsible for making sure this function is called only
1809 * when NestedPaging is supported and providing @a enmTlbFlush that is
1810 * supported by the CPU.
1811 * @remarks Can be called with interrupts disabled.
1812 */
1813static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush)
1814{
1815 uint64_t au64Descriptor[2];
1816 if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
1817 au64Descriptor[0] = 0;
1818 else
1819 {
1820 Assert(pVCpu);
1821 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1822 }
1823 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1824
1825 int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
1826 AssertMsg(rc == VINF_SUCCESS,
1827 ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0, rc));
1828
1829 if ( RT_SUCCESS(rc)
1830 && pVCpu)
1831 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1832}
1833
1834
1835/**
1836 * Flushes the TLB using VPID.
1837 *
1838 * @returns VBox status code.
1839 * @param pVCpu The cross context virtual CPU structure of the calling
1840 * EMT. Can be NULL depending on @a enmTlbFlush.
1841 * @param enmTlbFlush Type of flush.
1842 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1843 * on @a enmTlbFlush).
1844 *
1845 * @remarks Can be called with interrupts disabled.
1846 */
1847static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
1848{
1849 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
1850
1851 uint64_t au64Descriptor[2];
1852 if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
1853 {
1854 au64Descriptor[0] = 0;
1855 au64Descriptor[1] = 0;
1856 }
1857 else
1858 {
1859 AssertPtr(pVCpu);
1860 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1861 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1862 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1863 au64Descriptor[1] = GCPtr;
1864 }
1865
1866 int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
1867 AssertMsg(rc == VINF_SUCCESS,
1868 ("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1869
1870 if ( RT_SUCCESS(rc)
1871 && pVCpu)
1872 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1873 NOREF(rc);
1874}
1875
1876
1877/**
1878 * Invalidates a guest page by guest virtual address. Only relevant for
1879 * EPT/VPID, otherwise there is nothing really to invalidate.
1880 *
1881 * @returns VBox status code.
1882 * @param pVCpu The cross context virtual CPU structure.
1883 * @param GCVirt Guest virtual address of the page to invalidate.
1884 */
1885VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
1886{
1887 AssertPtr(pVCpu);
1888 LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
1889
1890 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1891 if (!fFlushPending)
1892 {
1893 /*
1894 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1895 * the EPT case. See @bugref{6043} and @bugref{6177}.
1896 *
1897 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1898 * as this function maybe called in a loop with individual addresses.
1899 */
1900 PVM pVM = pVCpu->CTX_SUFF(pVM);
1901 if (pVM->hm.s.vmx.fVpid)
1902 {
1903 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1904
1905#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1906 /*
1907 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1908 * where executing INVVPID outside 64-bit mode does not flush translations of
1909 * 64-bit linear addresses, see @bugref{6208#c72}.
1910 */
1911 if (RT_HI_U32(GCVirt))
1912 fVpidFlush = false;
1913#endif
1914
1915 if (fVpidFlush)
1916 {
1917 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
1918 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1919 }
1920 else
1921 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1922 }
1923 else if (pVM->hm.s.fNestedPaging)
1924 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1925 }
1926
1927 return VINF_SUCCESS;
1928}
1929
1930
1931/**
1932 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1933 * case where neither EPT nor VPID is supported by the CPU.
1934 *
1935 * @param pVCpu The cross context virtual CPU structure.
1936 * @param pCpu Pointer to the global HM struct.
1937 *
1938 * @remarks Called with interrupts disabled.
1939 */
1940static void hmR0VmxFlushTaggedTlbNone(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1941{
1942 AssertPtr(pVCpu);
1943 AssertPtr(pCpu);
1944
1945 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1946
1947 Assert(pCpu->idCpu != NIL_RTCPUID);
1948 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1949 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1950 pVCpu->hm.s.fForceTLBFlush = false;
1951 return;
1952}
1953
1954
1955/**
1956 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1957 *
1958 * @param pVCpu The cross context virtual CPU structure.
1959 * @param pCpu Pointer to the global HM CPU struct.
1960 *
1961 * @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
1962 * nomenclature. The reason is, to avoid confusion in compare statements
1963 * since the host-CPU copies are named "ASID".
1964 *
1965 * @remarks Called with interrupts disabled.
1966 */
1967static void hmR0VmxFlushTaggedTlbBoth(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1968{
1969#ifdef VBOX_WITH_STATISTICS
1970 bool fTlbFlushed = false;
1971# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1972# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1973 if (!fTlbFlushed) \
1974 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1975 } while (0)
1976#else
1977# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1978# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1979#endif
1980
1981 AssertPtr(pCpu);
1982 AssertPtr(pVCpu);
1983 Assert(pCpu->idCpu != NIL_RTCPUID);
1984
1985 PVM pVM = pVCpu->CTX_SUFF(pVM);
1986 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1987 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1988 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1989
1990 /*
1991 * Force a TLB flush for the first world-switch if the current CPU differs from the one we
1992 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1993 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1994 * cannot reuse the current ASID anymore.
1995 */
1996 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1997 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1998 {
1999 ++pCpu->uCurrentAsid;
2000 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2001 {
2002 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2003 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2004 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2005 }
2006
2007 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2008 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2009 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2010
2011 /*
2012 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2013 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2014 */
2015 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
2016 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2017 HMVMX_SET_TAGGED_TLB_FLUSHED();
2018 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2019 }
2020 else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2021 {
2022 /*
2023 * Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2024 * creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2025 * tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2026 * VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2027 * mappings, see @bugref{6568}.
2028 *
2029 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2030 */
2031 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
2032 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2033 HMVMX_SET_TAGGED_TLB_FLUSHED();
2034 }
2035
2036 pVCpu->hm.s.fForceTLBFlush = false;
2037 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2038
2039 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2040 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2041 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2042 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2043 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2044 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2045 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2046 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2047 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2048
2049 /* Update VMCS with the VPID. */
2050 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2051 AssertRC(rc);
2052
2053#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2054}
2055
2056
2057/**
2058 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2059 *
2060 * @returns VBox status code.
2061 * @param pVCpu The cross context virtual CPU structure.
2062 * @param pCpu Pointer to the global HM CPU struct.
2063 *
2064 * @remarks Called with interrupts disabled.
2065 */
2066static void hmR0VmxFlushTaggedTlbEpt(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2067{
2068 AssertPtr(pVCpu);
2069 AssertPtr(pCpu);
2070 Assert(pCpu->idCpu != NIL_RTCPUID);
2071 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2072 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2073
2074 /*
2075 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2076 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2077 */
2078 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2079 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2080 {
2081 pVCpu->hm.s.fForceTLBFlush = true;
2082 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2083 }
2084
2085 /* Check for explicit TLB flushes. */
2086 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2087 {
2088 pVCpu->hm.s.fForceTLBFlush = true;
2089 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2090 }
2091
2092 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2093 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2094
2095 if (pVCpu->hm.s.fForceTLBFlush)
2096 {
2097 hmR0VmxFlushEpt(pVCpu, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2098 pVCpu->hm.s.fForceTLBFlush = false;
2099 }
2100}
2101
2102
2103/**
2104 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2105 *
2106 * @returns VBox status code.
2107 * @param pVCpu The cross context virtual CPU structure.
2108 * @param pCpu Pointer to the global HM CPU struct.
2109 *
2110 * @remarks Called with interrupts disabled.
2111 */
2112static void hmR0VmxFlushTaggedTlbVpid(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2113{
2114 AssertPtr(pVCpu);
2115 AssertPtr(pCpu);
2116 Assert(pCpu->idCpu != NIL_RTCPUID);
2117 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2118 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2119
2120 /*
2121 * Force a TLB flush for the first world switch if the current CPU differs from the one we
2122 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2123 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2124 * cannot reuse the current ASID anymore.
2125 */
2126 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2127 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2128 {
2129 pVCpu->hm.s.fForceTLBFlush = true;
2130 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2131 }
2132
2133 /* Check for explicit TLB flushes. */
2134 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2135 {
2136 /*
2137 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2138 * hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2139 * fExplicitFlush = true here and change the pCpu->fFlushAsidBeforeUse check below to
2140 * include fExplicitFlush's too) - an obscure corner case.
2141 */
2142 pVCpu->hm.s.fForceTLBFlush = true;
2143 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2144 }
2145
2146 PVM pVM = pVCpu->CTX_SUFF(pVM);
2147 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2148 if (pVCpu->hm.s.fForceTLBFlush)
2149 {
2150 ++pCpu->uCurrentAsid;
2151 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2152 {
2153 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2154 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2155 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2156 }
2157
2158 pVCpu->hm.s.fForceTLBFlush = false;
2159 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2160 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2161 if (pCpu->fFlushAsidBeforeUse)
2162 {
2163 if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2164 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2165 else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2166 {
2167 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2168 pCpu->fFlushAsidBeforeUse = false;
2169 }
2170 else
2171 {
2172 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2173 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2174 }
2175 }
2176 }
2177
2178 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2179 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2180 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2181 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2182 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2183 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2184 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2185
2186 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2187 AssertRC(rc);
2188}
2189
2190
2191/**
2192 * Flushes the guest TLB entry based on CPU capabilities.
2193 *
2194 * @param pVCpu The cross context virtual CPU structure.
2195 * @param pCpu Pointer to the global HM CPU struct.
2196 */
2197DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2198{
2199#ifdef HMVMX_ALWAYS_FLUSH_TLB
2200 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2201#endif
2202 PVM pVM = pVCpu->CTX_SUFF(pVM);
2203 switch (pVM->hm.s.vmx.enmTlbFlushType)
2204 {
2205 case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVCpu, pCpu); break;
2206 case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pVCpu, pCpu); break;
2207 case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pVCpu, pCpu); break;
2208 case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pVCpu, pCpu); break;
2209 default:
2210 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2211 break;
2212 }
2213 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2214}
2215
2216
2217/**
2218 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2219 * TLB entries from the host TLB before VM-entry.
2220 *
2221 * @returns VBox status code.
2222 * @param pVM The cross context VM structure.
2223 */
2224static int hmR0VmxSetupTaggedTlb(PVM pVM)
2225{
2226 /*
2227 * Determine optimal flush type for Nested Paging.
2228 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2229 * guest execution (see hmR3InitFinalizeR0()).
2230 */
2231 if (pVM->hm.s.fNestedPaging)
2232 {
2233 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2234 {
2235 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2236 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2237 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2238 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2239 else
2240 {
2241 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2242 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2243 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2244 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2245 }
2246
2247 /* Make sure the write-back cacheable memory type for EPT is supported. */
2248 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2249 {
2250 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2251 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2252 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2253 }
2254
2255 /* EPT requires a page-walk length of 4. */
2256 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2257 {
2258 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2259 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2260 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2261 }
2262 }
2263 else
2264 {
2265 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2266 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2267 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2268 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2269 }
2270 }
2271
2272 /*
2273 * Determine optimal flush type for VPID.
2274 */
2275 if (pVM->hm.s.vmx.fVpid)
2276 {
2277 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2278 {
2279 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2280 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2281 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2282 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2283 else
2284 {
2285 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2286 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2287 LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2288 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2289 LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2290 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2291 pVM->hm.s.vmx.fVpid = false;
2292 }
2293 }
2294 else
2295 {
2296 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2297 Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2298 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2299 pVM->hm.s.vmx.fVpid = false;
2300 }
2301 }
2302
2303 /*
2304 * Setup the handler for flushing tagged-TLBs.
2305 */
2306 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2307 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2308 else if (pVM->hm.s.fNestedPaging)
2309 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2310 else if (pVM->hm.s.vmx.fVpid)
2311 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2312 else
2313 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2314 return VINF_SUCCESS;
2315}
2316
2317
2318/**
2319 * Sets up pin-based VM-execution controls in the VMCS.
2320 *
2321 * @returns VBox status code.
2322 * @param pVCpu The cross context virtual CPU structure.
2323 *
2324 * @remarks We don't really care about optimizing vmwrites here as it's done only
2325 * once per VM and hence we don't care about VMCS-field cache comparisons.
2326 */
2327static int hmR0VmxSetupPinCtls(PVMCPU pVCpu)
2328{
2329 PVM pVM = pVCpu->CTX_SUFF(pVM);
2330 uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.disallowed0; /* Bits set here must always be set. */
2331 uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2332
2333 fVal |= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2334 | VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2335
2336 if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2337 fVal |= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2338
2339 /* Enable the VMX preemption timer. */
2340 if (pVM->hm.s.vmx.fUsePreemptTimer)
2341 {
2342 Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2343 fVal |= VMX_PIN_CTLS_PREEMPT_TIMER;
2344 }
2345
2346#if 0
2347 /* Enable posted-interrupt processing. */
2348 if (pVM->hm.s.fPostedIntrs)
2349 {
2350 Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2351 Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2352 fVal |= VMX_PIN_CTL_POSTED_INT;
2353 }
2354#endif
2355
2356 if ((fVal & fZap) != fVal)
2357 {
2358 LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2359 pVM->hm.s.vmx.Msrs.PinCtls.n.disallowed0, fVal, fZap));
2360 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2361 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2362 }
2363
2364 /* Commit it to the VMCS and update our cache. */
2365 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2366 AssertRCReturn(rc, rc);
2367 pVCpu->hm.s.vmx.u32PinCtls = fVal;
2368
2369 return VINF_SUCCESS;
2370}
2371
2372
2373/**
2374 * Sets up secondary processor-based VM-execution controls in the VMCS.
2375 *
2376 * @returns VBox status code.
2377 * @param pVCpu The cross context virtual CPU structure.
2378 *
2379 * @remarks We don't really care about optimizing vmwrites here as it's done only
2380 * once per VM and hence we don't care about VMCS-field cache comparisons.
2381 */
2382static int hmR0VmxSetupProcCtls2(PVMCPU pVCpu)
2383{
2384 PVM pVM = pVCpu->CTX_SUFF(pVM);
2385 uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2386 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2387
2388 /* WBINVD causes a VM-exit. */
2389 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2390 fVal |= VMX_PROC_CTLS2_WBINVD_EXIT;
2391
2392 /* Enable EPT (aka nested-paging). */
2393 if (pVM->hm.s.fNestedPaging)
2394 fVal |= VMX_PROC_CTLS2_EPT;
2395
2396 /*
2397 * Enable the INVPCID instruction if supported by the hardware and we expose
2398 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2399 */
2400 if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID)
2401 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2402 fVal |= VMX_PROC_CTLS2_INVPCID;
2403
2404 /* Enable VPID. */
2405 if (pVM->hm.s.vmx.fVpid)
2406 fVal |= VMX_PROC_CTLS2_VPID;
2407
2408 /* Enable Unrestricted guest execution. */
2409 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2410 fVal |= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
2411
2412#if 0
2413 if (pVM->hm.s.fVirtApicRegs)
2414 {
2415 /* Enable APIC-register virtualization. */
2416 Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
2417 fVal |= VMX_PROC_CTLS2_APIC_REG_VIRT;
2418
2419 /* Enable virtual-interrupt delivery. */
2420 Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
2421 fVal |= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
2422 }
2423#endif
2424
2425 /* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is where the TPR shadow resides. */
2426 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2427 * done dynamically. */
2428 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
2429 {
2430 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2431 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2432 fVal |= VMX_PROC_CTLS2_VIRT_APIC_ACCESS; /* Virtualize APIC accesses. */
2433 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2434 AssertRCReturn(rc, rc);
2435 }
2436
2437 /* Enable RDTSCP. */
2438 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP)
2439 fVal |= VMX_PROC_CTLS2_RDTSCP;
2440
2441 /* Enable Pause-Loop exiting. */
2442 if ( pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT
2443 && pVM->hm.s.vmx.cPleGapTicks
2444 && pVM->hm.s.vmx.cPleWindowTicks)
2445 {
2446 fVal |= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
2447
2448 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2449 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2450 AssertRCReturn(rc, rc);
2451 }
2452
2453 if ((fVal & fZap) != fVal)
2454 {
2455 LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2456 pVM->hm.s.vmx.Msrs.ProcCtls2.n.disallowed0, fVal, fZap));
2457 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2458 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2459 }
2460
2461 /* Commit it to the VMCS and update our cache. */
2462 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2463 AssertRCReturn(rc, rc);
2464 pVCpu->hm.s.vmx.u32ProcCtls2 = fVal;
2465
2466 return VINF_SUCCESS;
2467}
2468
2469
2470/**
2471 * Sets up processor-based VM-execution controls in the VMCS.
2472 *
2473 * @returns VBox status code.
2474 * @param pVCpu The cross context virtual CPU structure.
2475 *
2476 * @remarks We don't really care about optimizing vmwrites here as it's done only
2477 * once per VM and hence we don't care about VMCS-field cache comparisons.
2478 */
2479static int hmR0VmxSetupProcCtls(PVMCPU pVCpu)
2480{
2481 PVM pVM = pVCpu->CTX_SUFF(pVM);
2482 uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2483 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2484
2485 fVal |= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
2486 | VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2487 | VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2488 | VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2489 | VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2490 | VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2491 | VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2492
2493 /* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2494 if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
2495 || (pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
2496 {
2497 LogRelFunc(("Unsupported VMX_PROC_CTLS_MOV_DR_EXIT combo!"));
2498 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2499 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2500 }
2501
2502 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2503 if (!pVM->hm.s.fNestedPaging)
2504 {
2505 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2506 fVal |= VMX_PROC_CTLS_INVLPG_EXIT
2507 | VMX_PROC_CTLS_CR3_LOAD_EXIT
2508 | VMX_PROC_CTLS_CR3_STORE_EXIT;
2509 }
2510
2511 /* Use TPR shadowing if supported by the CPU. */
2512 if ( PDMHasApic(pVM)
2513 && pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
2514 {
2515 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2516 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2517 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2518 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2519 AssertRCReturn(rc, rc);
2520
2521 fVal |= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2522 /* CR8 writes cause a VM-exit based on TPR threshold. */
2523 Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
2524 Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
2525 }
2526 else
2527 {
2528 /*
2529 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2530 * Set this control only for 64-bit guests.
2531 */
2532 if (pVM->hm.s.fAllow64BitGuests)
2533 {
2534 fVal |= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2535 | VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2536 }
2537 }
2538
2539 /* Use MSR-bitmaps if supported by the CPU. */
2540 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2541 {
2542 fVal |= VMX_PROC_CTLS_USE_MSR_BITMAPS;
2543
2544 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2545 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2546 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2547 AssertRCReturn(rc, rc);
2548
2549 /*
2550 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2551 * automatically using dedicated fields in the VMCS.
2552 */
2553 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2554 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2555 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2556 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2557 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2558#if HC_ARCH_BITS == 64
2559 /*
2560 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2561 */
2562 if (pVM->hm.s.fAllow64BitGuests)
2563 {
2564 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2565 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2566 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2567 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2568 }
2569#endif
2570 /*
2571 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2572 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2573 */
2574 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2575 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2576
2577 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2578 }
2579
2580 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2581 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2582 fVal |= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
2583
2584 if ((fVal & fZap) != fVal)
2585 {
2586 LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2587 pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0, fVal, fZap));
2588 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2589 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2590 }
2591
2592 /* Commit it to the VMCS and update our cache. */
2593 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2594 AssertRCReturn(rc, rc);
2595 pVCpu->hm.s.vmx.u32ProcCtls = fVal;
2596
2597 /* Set up secondary processor-based VM-execution controls if the CPU supports it. */
2598 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2599 return hmR0VmxSetupProcCtls2(pVCpu);
2600
2601 /* Sanity check, should not really happen. */
2602 if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2603 {
2604 LogRelFunc(("Unrestricted Guest enabled when secondary processor-based VM-execution controls not available\n"));
2605 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2606 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2607 }
2608
2609 /* Old CPUs without secondary processor-based VM-execution controls would end up here. */
2610 return VINF_SUCCESS;
2611}
2612
2613
2614/**
2615 * Sets up miscellaneous (everything other than Pin & Processor-based
2616 * VM-execution) control fields in the VMCS.
2617 *
2618 * @returns VBox status code.
2619 * @param pVCpu The cross context virtual CPU structure.
2620 */
2621static int hmR0VmxSetupMiscCtls(PVMCPU pVCpu)
2622{
2623 AssertPtr(pVCpu);
2624
2625 int rc = VERR_GENERAL_FAILURE;
2626
2627 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2628#if 0
2629 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxExportGuestCR3AndCR4())*/
2630 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2631 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2632
2633 /*
2634 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2635 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2636 * We thus use the exception bitmap to control it rather than use both.
2637 */
2638 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2639 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2640
2641 /* All IO & IOIO instructions cause VM-exits. */
2642 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2643 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2644
2645 /* Initialize the MSR-bitmap area. */
2646 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2647 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2648 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2649 AssertRCReturn(rc, rc);
2650#endif
2651
2652 /* Setup MSR auto-load/store area. */
2653 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2654 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2655 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2656 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2657 AssertRCReturn(rc, rc);
2658
2659 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2660 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2661 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2662 AssertRCReturn(rc, rc);
2663
2664 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2665 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2666 AssertRCReturn(rc, rc);
2667
2668 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2669#if 0
2670 /* Setup debug controls */
2671 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2672 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2673 AssertRCReturn(rc, rc);
2674#endif
2675
2676 return rc;
2677}
2678
2679
2680/**
2681 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2682 *
2683 * We shall setup those exception intercepts that don't change during the
2684 * lifetime of the VM here. The rest are done dynamically while loading the
2685 * guest state.
2686 *
2687 * @returns VBox status code.
2688 * @param pVCpu The cross context virtual CPU structure.
2689 */
2690static int hmR0VmxInitXcptBitmap(PVMCPU pVCpu)
2691{
2692 AssertPtr(pVCpu);
2693
2694 uint32_t uXcptBitmap;
2695
2696 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2697 uXcptBitmap = RT_BIT_32(X86_XCPT_AC);
2698
2699 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2700 and writes, and because recursive #DBs can cause the CPU hang, we must always
2701 intercept #DB. */
2702 uXcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2703
2704 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2705 if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
2706 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
2707
2708 /* Commit it to the VMCS. */
2709 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
2710 AssertRCReturn(rc, rc);
2711
2712 /* Update our cache of the exception bitmap. */
2713 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
2714 return VINF_SUCCESS;
2715}
2716
2717
2718/**
2719 * Does per-VM VT-x initialization.
2720 *
2721 * @returns VBox status code.
2722 * @param pVM The cross context VM structure.
2723 */
2724VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2725{
2726 LogFlowFunc(("pVM=%p\n", pVM));
2727
2728 int rc = hmR0VmxStructsAlloc(pVM);
2729 if (RT_FAILURE(rc))
2730 {
2731 LogRelFunc(("hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2732 return rc;
2733 }
2734
2735 return VINF_SUCCESS;
2736}
2737
2738
2739/**
2740 * Does per-VM VT-x termination.
2741 *
2742 * @returns VBox status code.
2743 * @param pVM The cross context VM structure.
2744 */
2745VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2746{
2747 LogFlowFunc(("pVM=%p\n", pVM));
2748
2749#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2750 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2751 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2752#endif
2753 hmR0VmxStructsFree(pVM);
2754 return VINF_SUCCESS;
2755}
2756
2757
2758/**
2759 * Sets up the VM for execution under VT-x.
2760 * This function is only called once per-VM during initialization.
2761 *
2762 * @returns VBox status code.
2763 * @param pVM The cross context VM structure.
2764 */
2765VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2766{
2767 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2768 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2769
2770 LogFlowFunc(("pVM=%p\n", pVM));
2771
2772 /*
2773 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be
2774 * allocated. We no longer support the highly unlikely case of UnrestrictedGuest without
2775 * pRealModeTSS, see hmR3InitFinalizeR0Intel().
2776 */
2777 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2778 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2779 || !pVM->hm.s.vmx.pRealModeTSS))
2780 {
2781 LogRelFunc(("Invalid real-on-v86 state.\n"));
2782 return VERR_INTERNAL_ERROR;
2783 }
2784
2785 /* Initialize these always, see hmR3InitFinalizeR0().*/
2786 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
2787 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
2788
2789 /* Setup the tagged-TLB flush handlers. */
2790 int rc = hmR0VmxSetupTaggedTlb(pVM);
2791 if (RT_FAILURE(rc))
2792 {
2793 LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2794 return rc;
2795 }
2796
2797 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2798 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2799#if HC_ARCH_BITS == 64
2800 if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
2801 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2802 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
2803 {
2804 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2805 }
2806#endif
2807
2808 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2809 RTCCUINTREG const uHostCR4 = ASMGetCR4();
2810 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2811 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2812
2813 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2814 {
2815 PVMCPU pVCpu = &pVM->aCpus[i];
2816 AssertPtr(pVCpu);
2817 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2818
2819 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2820 Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2821
2822 /* Set revision dword at the beginning of the VMCS structure. */
2823 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
2824
2825 /* Set the VMCS launch state to "clear", see Intel spec. 31.6 "Preparation and launch a virtual machine". */
2826 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2827 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc\n", rc),
2828 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2829
2830 /* Load this VMCS as the current VMCS. */
2831 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2832 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc\n", rc),
2833 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2834
2835 rc = hmR0VmxSetupPinCtls(pVCpu);
2836 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc\n", rc),
2837 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2838
2839 rc = hmR0VmxSetupProcCtls(pVCpu);
2840 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc\n", rc),
2841 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2842
2843 rc = hmR0VmxSetupMiscCtls(pVCpu);
2844 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc\n", rc),
2845 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2846
2847 rc = hmR0VmxInitXcptBitmap(pVCpu);
2848 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc\n", rc),
2849 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2850
2851#if HC_ARCH_BITS == 32
2852 rc = hmR0VmxInitVmcsReadCache(pVCpu);
2853 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc\n", rc),
2854 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2855#endif
2856
2857 /* Sync any CPU internal VMCS data back into our VMCS in memory. */
2858 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2859 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc\n", rc),
2860 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2861
2862 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2863
2864 hmR0VmxUpdateErrorRecord(pVCpu, rc);
2865 }
2866
2867 return VINF_SUCCESS;
2868}
2869
2870
2871/**
2872 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2873 * the VMCS.
2874 *
2875 * @returns VBox status code.
2876 */
2877static int hmR0VmxExportHostControlRegs(void)
2878{
2879 RTCCUINTREG uReg = ASMGetCR0();
2880 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2881 AssertRCReturn(rc, rc);
2882
2883 uReg = ASMGetCR3();
2884 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2885 AssertRCReturn(rc, rc);
2886
2887 uReg = ASMGetCR4();
2888 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2889 AssertRCReturn(rc, rc);
2890 return rc;
2891}
2892
2893
2894/**
2895 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2896 * the host-state area in the VMCS.
2897 *
2898 * @returns VBox status code.
2899 * @param pVCpu The cross context virtual CPU structure.
2900 */
2901static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
2902{
2903#if HC_ARCH_BITS == 64
2904/**
2905 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2906 * requirements. See hmR0VmxExportHostSegmentRegs().
2907 */
2908# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2909 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2910 { \
2911 bool fValidSelector = true; \
2912 if ((selValue) & X86_SEL_LDT) \
2913 { \
2914 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2915 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2916 } \
2917 if (fValidSelector) \
2918 { \
2919 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2920 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2921 } \
2922 (selValue) = 0; \
2923 }
2924
2925 /*
2926 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2927 * should -not- save the messed up state without restoring the original host-state,
2928 * see @bugref{7240}.
2929 *
2930 * This apparently can happen (most likely the FPU changes), deal with it rather than
2931 * asserting. Was observed booting Solaris 10u10 32-bit guest.
2932 */
2933 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2934 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2935 {
2936 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2937 pVCpu->idCpu));
2938 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2939 }
2940 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2941#else
2942 RT_NOREF(pVCpu);
2943#endif
2944
2945 /*
2946 * Host DS, ES, FS and GS segment registers.
2947 */
2948#if HC_ARCH_BITS == 64
2949 RTSEL uSelDS = ASMGetDS();
2950 RTSEL uSelES = ASMGetES();
2951 RTSEL uSelFS = ASMGetFS();
2952 RTSEL uSelGS = ASMGetGS();
2953#else
2954 RTSEL uSelDS = 0;
2955 RTSEL uSelES = 0;
2956 RTSEL uSelFS = 0;
2957 RTSEL uSelGS = 0;
2958#endif
2959
2960 /*
2961 * Host CS and SS segment registers.
2962 */
2963 RTSEL uSelCS = ASMGetCS();
2964 RTSEL uSelSS = ASMGetSS();
2965
2966 /*
2967 * Host TR segment register.
2968 */
2969 RTSEL uSelTR = ASMGetTR();
2970
2971#if HC_ARCH_BITS == 64
2972 /*
2973 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
2974 * gain VM-entry and restore them before we get preempted.
2975 *
2976 * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2977 */
2978 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2979 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2980 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2981 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2982# undef VMXLOCAL_ADJUST_HOST_SEG
2983#endif
2984
2985 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2986 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2987 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2988 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2989 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2990 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2991 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2992 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2993 Assert(uSelCS);
2994 Assert(uSelTR);
2995
2996 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2997#if 0
2998 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE))
2999 Assert(uSelSS != 0);
3000#endif
3001
3002 /* Write these host selector fields into the host-state area in the VMCS. */
3003 int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3004 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3005#if HC_ARCH_BITS == 64
3006 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3007 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3008 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3009 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3010#else
3011 NOREF(uSelDS);
3012 NOREF(uSelES);
3013 NOREF(uSelFS);
3014 NOREF(uSelGS);
3015#endif
3016 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3017 AssertRCReturn(rc, rc);
3018
3019 /*
3020 * Host GDTR and IDTR.
3021 */
3022 RTGDTR Gdtr;
3023 RTIDTR Idtr;
3024 RT_ZERO(Gdtr);
3025 RT_ZERO(Idtr);
3026 ASMGetGDTR(&Gdtr);
3027 ASMGetIDTR(&Idtr);
3028 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3029 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3030 AssertRCReturn(rc, rc);
3031
3032#if HC_ARCH_BITS == 64
3033 /*
3034 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
3035 * them to the maximum limit (0xffff) on every VM-exit.
3036 */
3037 if (Gdtr.cbGdt != 0xffff)
3038 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3039
3040 /*
3041 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
3042 * Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
3043 * as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
3044 * However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
3045 * uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
3046 * alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
3047 * at 0xffff on hosts where we are sure it won't cause trouble.
3048 */
3049# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3050 if (Idtr.cbIdt < 0x0fff)
3051# else
3052 if (Idtr.cbIdt != 0xffff)
3053# endif
3054 {
3055 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3056 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3057 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3058 }
3059#endif
3060
3061 /*
3062 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3063 * and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3064 * RPL should be too in most cases.
3065 */
3066 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3067 ("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3068
3069 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3070#if HC_ARCH_BITS == 64
3071 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3072
3073 /*
3074 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3075 * all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3076 * restoration if the host has something else. Task switching is not supported in 64-bit
3077 * mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3078 * limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3079 *
3080 * [1] See Intel spec. 3.5 "System Descriptor Types".
3081 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3082 */
3083 PVM pVM = pVCpu->CTX_SUFF(pVM);
3084 Assert(pDesc->System.u4Type == 11);
3085 if ( pDesc->System.u16LimitLow != 0x67
3086 || pDesc->System.u4LimitHigh)
3087 {
3088 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3089 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3090 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3091 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3092 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3093 }
3094
3095 /*
3096 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3097 */
3098 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3099 {
3100 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3101 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3102 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3103 {
3104 /* The GDT is read-only but the writable GDT is available. */
3105 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3106 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3107 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3108 AssertRCReturn(rc, rc);
3109 }
3110 }
3111#else
3112 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3113#endif
3114 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3115 AssertRCReturn(rc, rc);
3116
3117 /*
3118 * Host FS base and GS base.
3119 */
3120#if HC_ARCH_BITS == 64
3121 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3122 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3123 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3124 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3125 AssertRCReturn(rc, rc);
3126
3127 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3128 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3129 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3130 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3131 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3132#endif
3133 return VINF_SUCCESS;
3134}
3135
3136
3137/**
3138 * Exports certain host MSRs in the VM-exit MSR-load area and some in the
3139 * host-state area of the VMCS.
3140 *
3141 * Theses MSRs will be automatically restored on the host after every successful
3142 * VM-exit.
3143 *
3144 * @returns VBox status code.
3145 * @param pVCpu The cross context virtual CPU structure.
3146 *
3147 * @remarks No-long-jump zone!!!
3148 */
3149static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
3150{
3151 AssertPtr(pVCpu);
3152 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3153
3154 /*
3155 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3156 * rather than swapping them on every VM-entry.
3157 */
3158 hmR0VmxLazySaveHostMsrs(pVCpu);
3159
3160 /*
3161 * Host Sysenter MSRs.
3162 */
3163 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3164#if HC_ARCH_BITS == 32
3165 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3166 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3167#else
3168 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3169 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3170#endif
3171 AssertRCReturn(rc, rc);
3172
3173 /*
3174 * Host EFER MSR.
3175 *
3176 * If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3177 * done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3178 */
3179 PVM pVM = pVCpu->CTX_SUFF(pVM);
3180 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3181 {
3182 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3183 AssertRCReturn(rc, rc);
3184 }
3185
3186 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see hmR0VmxExportGuestExitCtls(). */
3187
3188 return VINF_SUCCESS;
3189}
3190
3191
3192/**
3193 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3194 *
3195 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3196 * these two bits are handled by VM-entry, see hmR0VmxExportGuestExitCtls() and
3197 * hmR0VMxExportGuestEntryCtls().
3198 *
3199 * @returns true if we need to load guest EFER, false otherwise.
3200 * @param pVCpu The cross context virtual CPU structure.
3201 *
3202 * @remarks Requires EFER, CR4.
3203 * @remarks No-long-jump zone!!!
3204 */
3205static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu)
3206{
3207#ifdef HMVMX_ALWAYS_SWAP_EFER
3208 return true;
3209#endif
3210
3211 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3212#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3213 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3214 if (CPUMIsGuestInLongModeEx(pCtx))
3215 return false;
3216#endif
3217
3218 PVM pVM = pVCpu->CTX_SUFF(pVM);
3219 uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3220 uint64_t const u64GuestEfer = pCtx->msrEFER;
3221
3222 /*
3223 * For 64-bit guests, if EFER.SCE bit differs, we need to swap EFER to ensure that the
3224 * guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3225 */
3226 if ( CPUMIsGuestInLongModeEx(pCtx)
3227 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3228 {
3229 return true;
3230 }
3231
3232 /*
3233 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3234 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3235 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3236 */
3237 if ( (pCtx->cr4 & X86_CR4_PAE)
3238 && (pCtx->cr0 & X86_CR0_PG)
3239 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3240 {
3241 /* Assert that host is NX capable. */
3242 Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
3243 return true;
3244 }
3245
3246 return false;
3247}
3248
3249
3250/**
3251 * Exports the guest state with appropriate VM-entry controls in the VMCS.
3252 *
3253 * These controls can affect things done on VM-exit; e.g. "load debug controls",
3254 * see Intel spec. 24.8.1 "VM-entry controls".
3255 *
3256 * @returns VBox status code.
3257 * @param pVCpu The cross context virtual CPU structure.
3258 *
3259 * @remarks Requires EFER.
3260 * @remarks No-long-jump zone!!!
3261 */
3262static int hmR0VmxExportGuestEntryCtls(PVMCPU pVCpu)
3263{
3264 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_CTLS)
3265 {
3266 PVM pVM = pVCpu->CTX_SUFF(pVM);
3267 uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
3268 uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3269
3270 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3271 fVal |= VMX_ENTRY_CTLS_LOAD_DEBUG;
3272
3273 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3274 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
3275 {
3276 fVal |= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
3277 Log4Func(("VMX_ENTRY_CTLS_IA32E_MODE_GUEST\n"));
3278 }
3279 else
3280 Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
3281
3282 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3283 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3284 && hmR0VmxShouldSwapEferMsr(pVCpu))
3285 {
3286 fVal |= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
3287 Log4Func(("VMX_ENTRY_CTLS_LOAD_EFER_MSR\n"));
3288 }
3289
3290 /*
3291 * The following should -not- be set (since we're not in SMM mode):
3292 * - VMX_ENTRY_CTLS_ENTRY_TO_SMM
3293 * - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
3294 */
3295
3296 /** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
3297 * VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
3298
3299 if ((fVal & fZap) != fVal)
3300 {
3301 Log4Func(("Invalid VM-entry controls combo! Cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3302 pVM->hm.s.vmx.Msrs.EntryCtls.n.disallowed0, fVal, fZap));
3303 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3304 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3305 }
3306
3307 /* Commit it to the VMCS and update our cache. */
3308 if (pVCpu->hm.s.vmx.u32EntryCtls != fVal)
3309 {
3310 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
3311 AssertRCReturn(rc, rc);
3312 pVCpu->hm.s.vmx.u32EntryCtls = fVal;
3313 }
3314
3315 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_CTLS);
3316 }
3317 return VINF_SUCCESS;
3318}
3319
3320
3321/**
3322 * Exports the guest state with appropriate VM-exit controls in the VMCS.
3323 *
3324 * @returns VBox status code.
3325 * @param pVCpu The cross context virtual CPU structure.
3326 *
3327 * @remarks Requires EFER.
3328 */
3329static int hmR0VmxExportGuestExitCtls(PVMCPU pVCpu)
3330{
3331 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_EXIT_CTLS)
3332 {
3333 PVM pVM = pVCpu->CTX_SUFF(pVM);
3334 uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
3335 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3336
3337 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3338 fVal |= VMX_EXIT_CTLS_SAVE_DEBUG;
3339
3340 /*
3341 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3342 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in
3343 * hmR0VmxExportHostMsrs().
3344 */
3345#if HC_ARCH_BITS == 64
3346 fVal |= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
3347 Log4Func(("VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE\n"));
3348#else
3349 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3350 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3351 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3352 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3353 {
3354 /* The switcher returns to long mode, EFER is managed by the switcher. */
3355 fVal |= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
3356 Log4Func(("VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE\n"));
3357 }
3358 else
3359 Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
3360#endif
3361
3362 /* If the newer VMCS fields for managing EFER exists, use it. */
3363 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3364 && hmR0VmxShouldSwapEferMsr(pVCpu))
3365 {
3366 fVal |= VMX_EXIT_CTLS_SAVE_EFER_MSR
3367 | VMX_EXIT_CTLS_LOAD_EFER_MSR;
3368 Log4Func(("VMX_EXIT_CTLS_SAVE_EFER_MSR and VMX_EXIT_CTLS_LOAD_EFER_MSR\n"));
3369 }
3370
3371 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3372 Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
3373
3374 /** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
3375 * VMX_EXIT_CTLS_SAVE_PAT_MSR,
3376 * VMX_EXIT_CTLS_LOAD_PAT_MSR. */
3377
3378 /* Enable saving of the VMX preemption timer value on VM-exit. */
3379 if ( pVM->hm.s.vmx.fUsePreemptTimer
3380 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
3381 fVal |= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
3382
3383 if ((fVal & fZap) != fVal)
3384 {
3385 LogRelFunc(("Invalid VM-exit controls combo! cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3386 pVM->hm.s.vmx.Msrs.ExitCtls.n.disallowed0, fVal, fZap));
3387 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3388 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3389 }
3390
3391 /* Commit it to the VMCS and update our cache. */
3392 if (pVCpu->hm.s.vmx.u32ExitCtls != fVal)
3393 {
3394 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
3395 AssertRCReturn(rc, rc);
3396 pVCpu->hm.s.vmx.u32ExitCtls = fVal;
3397 }
3398
3399 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_EXIT_CTLS);
3400 }
3401 return VINF_SUCCESS;
3402}
3403
3404
3405/**
3406 * Sets the TPR threshold in the VMCS.
3407 *
3408 * @returns VBox status code.
3409 * @param pVCpu The cross context virtual CPU structure.
3410 * @param u32TprThreshold The TPR threshold (task-priority class only).
3411 */
3412DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3413{
3414 Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
3415 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3416 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3417}
3418
3419
3420/**
3421 * Exports the guest APIC TPR state into the VMCS.
3422 *
3423 * @returns VBox status code.
3424 * @param pVCpu The cross context virtual CPU structure.
3425 *
3426 * @remarks No-long-jump zone!!!
3427 */
3428static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu)
3429{
3430 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
3431 {
3432 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
3433
3434 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3435 && APICIsEnabled(pVCpu))
3436 {
3437 /*
3438 * Setup TPR shadowing.
3439 */
3440 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
3441 {
3442 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3443
3444 bool fPendingIntr = false;
3445 uint8_t u8Tpr = 0;
3446 uint8_t u8PendingIntr = 0;
3447 int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3448 AssertRCReturn(rc, rc);
3449
3450 /*
3451 * If there are interrupts pending but masked by the TPR, instruct VT-x to
3452 * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
3453 * priority of the pending interrupt so we can deliver the interrupt. If there
3454 * are no interrupts pending, set threshold to 0 to not cause any
3455 * TPR-below-threshold VM-exits.
3456 */
3457 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3458 uint32_t u32TprThreshold = 0;
3459 if (fPendingIntr)
3460 {
3461 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3462 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3463 const uint8_t u8TprPriority = u8Tpr >> 4;
3464 if (u8PendingPriority <= u8TprPriority)
3465 u32TprThreshold = u8PendingPriority;
3466 }
3467
3468 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3469 AssertRCReturn(rc, rc);
3470 }
3471 }
3472 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
3473 }
3474 return VINF_SUCCESS;
3475}
3476
3477
3478/**
3479 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3480 *
3481 * @returns Guest's interruptibility-state.
3482 * @param pVCpu The cross context virtual CPU structure.
3483 *
3484 * @remarks No-long-jump zone!!!
3485 */
3486static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu)
3487{
3488 /*
3489 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3490 */
3491 uint32_t fIntrState = 0;
3492 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3493 {
3494 /* If inhibition is active, RIP & RFLAGS should've been accessed
3495 (i.e. read previously from the VMCS or from ring-3). */
3496 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3497#ifdef VBOX_STRICT
3498 uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
3499 AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
3500#endif
3501 if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3502 {
3503 if (pCtx->eflags.Bits.u1IF)
3504 fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
3505 else
3506 fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
3507 }
3508 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3509 {
3510 /*
3511 * We can clear the inhibit force flag as even if we go back to the recompiler
3512 * without executing guest code in VT-x, the flag's condition to be cleared is
3513 * met and thus the cleared state is correct.
3514 */
3515 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3516 }
3517 }
3518
3519 /*
3520 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3521 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3522 * setting this would block host-NMIs and IRET will not clear the blocking.
3523 *
3524 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3525 */
3526 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3527 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
3528 {
3529 fIntrState |= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
3530 }
3531
3532 return fIntrState;
3533}
3534
3535
3536/**
3537 * Exports the guest's interruptibility-state into the guest-state area in the
3538 * VMCS.
3539 *
3540 * @returns VBox status code.
3541 * @param pVCpu The cross context virtual CPU structure.
3542 * @param fIntrState The interruptibility-state to set.
3543 */
3544static int hmR0VmxExportGuestIntrState(PVMCPU pVCpu, uint32_t fIntrState)
3545{
3546 NOREF(pVCpu);
3547 AssertMsg(!(fIntrState & 0xfffffff0), ("%#x\n", fIntrState)); /* Bits 31:4 MBZ. */
3548 Assert((fIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3549 return VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
3550}
3551
3552
3553/**
3554 * Exports the exception intercepts required for guest execution in the VMCS.
3555 *
3556 * @returns VBox status code.
3557 * @param pVCpu The cross context virtual CPU structure.
3558 *
3559 * @remarks No-long-jump zone!!!
3560 */
3561static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu)
3562{
3563 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
3564 {
3565 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3566
3567 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxExportSharedCR0(). */
3568 if (pVCpu->hm.s.fGIMTrapXcptUD)
3569 uXcptBitmap |= RT_BIT(X86_XCPT_UD);
3570#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3571 else
3572 uXcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3573#endif
3574
3575 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_AC));
3576 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_DB));
3577
3578 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3579 {
3580 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3581 AssertRCReturn(rc, rc);
3582 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3583 }
3584
3585 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3586 Log4Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64\n", uXcptBitmap));
3587 }
3588 return VINF_SUCCESS;
3589}
3590
3591
3592/**
3593 * Exports the guest's RIP into the guest-state area in the VMCS.
3594 *
3595 * @returns VBox status code.
3596 * @param pVCpu The cross context virtual CPU structure.
3597 *
3598 * @remarks No-long-jump zone!!!
3599 */
3600static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
3601{
3602 int rc = VINF_SUCCESS;
3603 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
3604 {
3605 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
3606
3607 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
3608 AssertRCReturn(rc, rc);
3609
3610 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3611 Log4Func(("RIP=%#RX64\n", pVCpu->cpum.GstCtx.rip));
3612 }
3613 return rc;
3614}
3615
3616
3617/**
3618 * Exports the guest's RSP into the guest-state area in the VMCS.
3619 *
3620 * @returns VBox status code.
3621 * @param pVCpu The cross context virtual CPU structure.
3622 *
3623 * @remarks No-long-jump zone!!!
3624 */
3625static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
3626{
3627 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3628 {
3629 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
3630
3631 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
3632 AssertRCReturn(rc, rc);
3633
3634 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3635 }
3636 return VINF_SUCCESS;
3637}
3638
3639
3640/**
3641 * Exports the guest's RFLAGS into the guest-state area in the VMCS.
3642 *
3643 * @returns VBox status code.
3644 * @param pVCpu The cross context virtual CPU structure.
3645 *
3646 * @remarks No-long-jump zone!!!
3647 */
3648static int hmR0VmxExportGuestRflags(PVMCPU pVCpu)
3649{
3650 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3651 {
3652 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
3653
3654 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3655 Let us assert it as such and use 32-bit VMWRITE. */
3656 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
3657 X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
3658 Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3659 Assert(!(fEFlags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3660
3661 /*
3662 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3663 * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3664 * can run the real-mode guest code under Virtual 8086 mode.
3665 */
3666 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3667 {
3668 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3669 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3670 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3671 fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3672 fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3673 }
3674
3675 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3676 AssertRCReturn(rc, rc);
3677
3678 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3679 Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
3680 }
3681 return VINF_SUCCESS;
3682}
3683
3684
3685/**
3686 * Exports the guest CR0 control register into the guest-state area in the VMCS.
3687 *
3688 * The guest FPU state is always pre-loaded hence we don't need to bother about
3689 * sharing FPU related CR0 bits between the guest and host.
3690 *
3691 * @returns VBox status code.
3692 * @param pVCpu The cross context virtual CPU structure.
3693 *
3694 * @remarks No-long-jump zone!!!
3695 */
3696static int hmR0VmxExportGuestCR0(PVMCPU pVCpu)
3697{
3698 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
3699 {
3700 PVM pVM = pVCpu->CTX_SUFF(pVM);
3701 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
3702 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.cr0));
3703
3704 uint32_t const u32ShadowCr0 = pVCpu->cpum.GstCtx.cr0;
3705 uint32_t u32GuestCr0 = pVCpu->cpum.GstCtx.cr0;
3706
3707 /*
3708 * Setup VT-x's view of the guest CR0.
3709 * Minimize VM-exits due to CR3 changes when we have NestedPaging.
3710 */
3711 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
3712 if (pVM->hm.s.fNestedPaging)
3713 {
3714 if (CPUMIsGuestPagingEnabled(pVCpu))
3715 {
3716 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3717 uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
3718 | VMX_PROC_CTLS_CR3_STORE_EXIT);
3719 }
3720 else
3721 {
3722 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3723 uProcCtls |= VMX_PROC_CTLS_CR3_LOAD_EXIT
3724 | VMX_PROC_CTLS_CR3_STORE_EXIT;
3725 }
3726
3727 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3728 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3729 uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
3730 }
3731 else
3732 {
3733 /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3734 u32GuestCr0 |= X86_CR0_WP;
3735 }
3736
3737 /*
3738 * Guest FPU bits.
3739 *
3740 * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3741 * using CR0.TS.
3742 *
3743 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3744 * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3745 */
3746 u32GuestCr0 |= X86_CR0_NE;
3747
3748 /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3749 bool const fInterceptMF = !(u32ShadowCr0 & X86_CR0_NE);
3750
3751 /*
3752 * Update exception intercepts.
3753 */
3754 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3755 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3756 {
3757 Assert(PDMVmmDevHeapIsEnabled(pVM));
3758 Assert(pVM->hm.s.vmx.pRealModeTSS);
3759 uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3760 }
3761 else
3762 {
3763 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3764 uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3765 if (fInterceptMF)
3766 uXcptBitmap |= RT_BIT(X86_XCPT_MF);
3767 }
3768
3769 /* Additional intercepts for debugging, define these yourself explicitly. */
3770#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3771 uXcptBitmap |= 0
3772 | RT_BIT(X86_XCPT_BP)
3773 | RT_BIT(X86_XCPT_DE)
3774 | RT_BIT(X86_XCPT_NM)
3775 | RT_BIT(X86_XCPT_TS)
3776 | RT_BIT(X86_XCPT_UD)
3777 | RT_BIT(X86_XCPT_NP)
3778 | RT_BIT(X86_XCPT_SS)
3779 | RT_BIT(X86_XCPT_GP)
3780 | RT_BIT(X86_XCPT_PF)
3781 | RT_BIT(X86_XCPT_MF)
3782 ;
3783#elif defined(HMVMX_ALWAYS_TRAP_PF)
3784 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
3785#endif
3786 Assert(pVM->hm.s.fNestedPaging || (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
3787
3788 /*
3789 * Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW).
3790 */
3791 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3792 uint32_t fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3793 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3794 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
3795 else
3796 Assert((fSetCr0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3797
3798 u32GuestCr0 |= fSetCr0;
3799 u32GuestCr0 &= fZapCr0;
3800 u32GuestCr0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3801
3802 /*
3803 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3804 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3805 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3806 */
3807 uint32_t u32Cr0Mask = X86_CR0_PE
3808 | X86_CR0_NE
3809 | (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
3810 | X86_CR0_PG
3811 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3812 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3813 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3814
3815 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3816 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3817 * and @bugref{6944}. */
3818#if 0
3819 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3820 u32Cr0Mask &= ~X86_CR0_PE;
3821#endif
3822 /*
3823 * Finally, update VMCS fields with the CR0 values and the exception bitmap.
3824 */
3825 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCr0);
3826 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32ShadowCr0);
3827 if (u32Cr0Mask != pVCpu->hm.s.vmx.u32Cr0Mask)
3828 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32Cr0Mask);
3829 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
3830 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3831 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3832 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3833 AssertRCReturn(rc, rc);
3834
3835 /* Update our caches. */
3836 pVCpu->hm.s.vmx.u32Cr0Mask = u32Cr0Mask;
3837 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
3838 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3839
3840 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3841
3842 Log4Func(("u32Cr0Mask=%#RX32 u32ShadowCr0=%#RX32 u32GuestCr0=%#RX32 (fSetCr0=%#RX32 fZapCr0=%#RX32\n", u32Cr0Mask,
3843 u32ShadowCr0, u32GuestCr0, fSetCr0, fZapCr0));
3844 }
3845
3846 return VINF_SUCCESS;
3847}
3848
3849
3850/**
3851 * Exports the guest control registers (CR3, CR4) into the guest-state area
3852 * in the VMCS.
3853 *
3854 * @returns VBox strict status code.
3855 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3856 * without unrestricted guest access and the VMMDev is not presently
3857 * mapped (e.g. EFI32).
3858 *
3859 * @param pVCpu The cross context virtual CPU structure.
3860 *
3861 * @remarks No-long-jump zone!!!
3862 */
3863static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu)
3864{
3865 int rc = VINF_SUCCESS;
3866 PVM pVM = pVCpu->CTX_SUFF(pVM);
3867
3868 /*
3869 * Guest CR2.
3870 * It's always loaded in the assembler code. Nothing to do here.
3871 */
3872
3873 /*
3874 * Guest CR3.
3875 */
3876 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
3877 {
3878 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
3879
3880 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3881 if (pVM->hm.s.fNestedPaging)
3882 {
3883 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3884
3885 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3886 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3887 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3888 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3889
3890 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3891 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3892 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3893
3894 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3895 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3896 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3897 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3898 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3899 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3900 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3901
3902 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3903 AssertRCReturn(rc, rc);
3904
3905 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3906 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3907 || CPUMIsGuestPagingEnabledEx(pCtx))
3908 {
3909 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3910 if (CPUMIsGuestInPAEModeEx(pCtx))
3911 {
3912 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3913 AssertRCReturn(rc, rc);
3914 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3915 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3916 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3917 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3918 AssertRCReturn(rc, rc);
3919 }
3920
3921 /*
3922 * The guest's view of its CR3 is unblemished with Nested Paging when the
3923 * guest is using paging or we have unrestricted guest execution to handle
3924 * the guest when it's not using paging.
3925 */
3926 GCPhysGuestCR3 = pCtx->cr3;
3927 }
3928 else
3929 {
3930 /*
3931 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
3932 * thinks it accesses physical memory directly, we use our identity-mapped
3933 * page table to map guest-linear to guest-physical addresses. EPT takes care
3934 * of translating it to host-physical addresses.
3935 */
3936 RTGCPHYS GCPhys;
3937 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3938
3939 /* We obtain it here every time as the guest could have relocated this PCI region. */
3940 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3941 if (RT_SUCCESS(rc))
3942 { /* likely */ }
3943 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3944 {
3945 Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3946 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3947 }
3948 else
3949 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3950
3951 GCPhysGuestCR3 = GCPhys;
3952 }
3953
3954 Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
3955 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3956 AssertRCReturn(rc, rc);
3957 }
3958 else
3959 {
3960 /* Non-nested paging case, just use the hypervisor's CR3. */
3961 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3962
3963 Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
3964 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3965 AssertRCReturn(rc, rc);
3966 }
3967
3968 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3969 }
3970
3971 /*
3972 * Guest CR4.
3973 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3974 */
3975 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
3976 {
3977 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3978 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
3979 Assert(!RT_HI_U32(pCtx->cr4));
3980
3981 uint32_t u32GuestCr4 = pCtx->cr4;
3982 uint32_t const u32ShadowCr4 = pCtx->cr4;
3983
3984 /*
3985 * Setup VT-x's view of the guest CR4.
3986 *
3987 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3988 * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3989 * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3990 *
3991 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3992 */
3993 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3994 {
3995 Assert(pVM->hm.s.vmx.pRealModeTSS);
3996 Assert(PDMVmmDevHeapIsEnabled(pVM));
3997 u32GuestCr4 &= ~X86_CR4_VME;
3998 }
3999
4000 if (pVM->hm.s.fNestedPaging)
4001 {
4002 if ( !CPUMIsGuestPagingEnabledEx(pCtx)
4003 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4004 {
4005 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4006 u32GuestCr4 |= X86_CR4_PSE;
4007 /* Our identity mapping is a 32-bit page directory. */
4008 u32GuestCr4 &= ~X86_CR4_PAE;
4009 }
4010 /* else use guest CR4.*/
4011 }
4012 else
4013 {
4014 /*
4015 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4016 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4017 */
4018 switch (pVCpu->hm.s.enmShadowMode)
4019 {
4020 case PGMMODE_REAL: /* Real-mode. */
4021 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4022 case PGMMODE_32_BIT: /* 32-bit paging. */
4023 {
4024 u32GuestCr4 &= ~X86_CR4_PAE;
4025 break;
4026 }
4027
4028 case PGMMODE_PAE: /* PAE paging. */
4029 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4030 {
4031 u32GuestCr4 |= X86_CR4_PAE;
4032 break;
4033 }
4034
4035 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4036 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4037#ifdef VBOX_ENABLE_64_BITS_GUESTS
4038 break;
4039#endif
4040 default:
4041 AssertFailed();
4042 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4043 }
4044 }
4045
4046 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4047 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4048 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4049 u32GuestCr4 |= fSetCr4;
4050 u32GuestCr4 &= fZapCr4;
4051
4052 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them,
4053 that would cause a VM-exit. */
4054 uint32_t u32Cr4Mask = X86_CR4_VME
4055 | X86_CR4_PAE
4056 | X86_CR4_PGE
4057 | X86_CR4_PSE
4058 | X86_CR4_VMXE;
4059 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4060 u32Cr4Mask |= X86_CR4_OSXSAVE;
4061 if (pVM->cpum.ro.GuestFeatures.fPcid)
4062 u32Cr4Mask |= X86_CR4_PCIDE;
4063
4064 /* Write VT-x's view of the guest CR4, the CR4 modify mask and the read-only CR4 shadow
4065 into the VMCS and update our cache. */
4066 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCr4);
4067 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32ShadowCr4);
4068 if (pVCpu->hm.s.vmx.u32Cr4Mask != u32Cr4Mask)
4069 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32Cr4Mask);
4070 AssertRCReturn(rc, rc);
4071 pVCpu->hm.s.vmx.u32Cr4Mask = u32Cr4Mask;
4072
4073 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4074 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
4075
4076 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
4077
4078 Log4Func(("u32GuestCr4=%#RX32 u32ShadowCr4=%#RX32 (fSetCr4=%#RX32 fZapCr4=%#RX32)\n", u32GuestCr4, u32ShadowCr4, fSetCr4,
4079 fZapCr4));
4080 }
4081 return rc;
4082}
4083
4084
4085/**
4086 * Exports the guest debug registers into the guest-state area in the VMCS.
4087 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4088 *
4089 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4090 *
4091 * @returns VBox status code.
4092 * @param pVCpu The cross context virtual CPU structure.
4093 *
4094 * @remarks No-long-jump zone!!!
4095 */
4096static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu)
4097{
4098 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4099
4100#ifdef VBOX_STRICT
4101 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4102 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
4103 {
4104 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4105 Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0);
4106 Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
4107 }
4108#endif
4109
4110 bool fSteppingDB = false;
4111 bool fInterceptMovDRx = false;
4112 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
4113 if (pVCpu->hm.s.fSingleInstruction)
4114 {
4115 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4116 PVM pVM = pVCpu->CTX_SUFF(pVM);
4117 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
4118 {
4119 uProcCtls |= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
4120 Assert(fSteppingDB == false);
4121 }
4122 else
4123 {
4124 pVCpu->cpum.GstCtx.eflags.u32 |= X86_EFL_TF;
4125 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_RFLAGS;
4126 pVCpu->hm.s.fClearTrapFlag = true;
4127 fSteppingDB = true;
4128 }
4129 }
4130
4131 uint32_t u32GuestDr7;
4132 if ( fSteppingDB
4133 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4134 {
4135 /*
4136 * Use the combined guest and host DRx values found in the hypervisor register set
4137 * because the debugger has breakpoints active or someone is single stepping on the
4138 * host side without a monitor trap flag.
4139 *
4140 * Note! DBGF expects a clean DR6 state before executing guest code.
4141 */
4142#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4143 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4144 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4145 {
4146 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4147 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4148 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4149 }
4150 else
4151#endif
4152 if (!CPUMIsHyperDebugStateActive(pVCpu))
4153 {
4154 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4155 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4156 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4157 }
4158
4159 /* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
4160 u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
4161 pVCpu->hm.s.fUsingHyperDR7 = true;
4162 fInterceptMovDRx = true;
4163 }
4164 else
4165 {
4166 /*
4167 * If the guest has enabled debug registers, we need to load them prior to
4168 * executing guest code so they'll trigger at the right time.
4169 */
4170 if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD))
4171 {
4172#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4173 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4174 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4175 {
4176 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4177 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4178 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4179 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4180 }
4181 else
4182#endif
4183 if (!CPUMIsGuestDebugStateActive(pVCpu))
4184 {
4185 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4186 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4187 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4188 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4189 }
4190 Assert(!fInterceptMovDRx);
4191 }
4192 /*
4193 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4194 * must intercept #DB in order to maintain a correct DR6 guest value, and
4195 * because we need to intercept it to prevent nested #DBs from hanging the
4196 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4197 */
4198#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4199 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4200 && !CPUMIsGuestDebugStateActive(pVCpu))
4201#else
4202 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4203#endif
4204 {
4205 fInterceptMovDRx = true;
4206 }
4207
4208 /* Update DR7 with the actual guest value. */
4209 u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
4210 pVCpu->hm.s.fUsingHyperDR7 = false;
4211 }
4212
4213 if (fInterceptMovDRx)
4214 uProcCtls |= VMX_PROC_CTLS_MOV_DR_EXIT;
4215 else
4216 uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
4217
4218 /*
4219 * Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
4220 * monitor-trap flag and update our cache.
4221 */
4222 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
4223 {
4224 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4225 AssertRCReturn(rc2, rc2);
4226 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
4227 }
4228
4229 /*
4230 * Update guest DR7.
4231 */
4232 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
4233 AssertRCReturn(rc, rc);
4234
4235 return VINF_SUCCESS;
4236}
4237
4238
4239#ifdef VBOX_STRICT
4240/**
4241 * Strict function to validate segment registers.
4242 *
4243 * @param pVCpu The cross context virtual CPU structure.
4244 *
4245 * @remarks Will import guest CR0 on strict builds during validation of
4246 * segments.
4247 */
4248static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu)
4249{
4250 /*
4251 * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
4252 *
4253 * The reason we check for attribute value 0 in this function and not just the unusable bit is
4254 * because hmR0VmxExportGuestSegmentReg() only updates the VMCS' copy of the value with the unusable bit
4255 * and doesn't change the guest-context value.
4256 */
4257 PVM pVM = pVCpu->CTX_SUFF(pVM);
4258 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4259 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
4260 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4261 && ( !CPUMIsGuestInRealModeEx(pCtx)
4262 && !CPUMIsGuestInV86ModeEx(pCtx)))
4263 {
4264 /* Protected mode checks */
4265 /* CS */
4266 Assert(pCtx->cs.Attr.n.u1Present);
4267 Assert(!(pCtx->cs.Attr.u & 0xf00));
4268 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4269 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4270 || !(pCtx->cs.Attr.n.u1Granularity));
4271 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4272 || (pCtx->cs.Attr.n.u1Granularity));
4273 /* CS cannot be loaded with NULL in protected mode. */
4274 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4275 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4276 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4277 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4278 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4279 else
4280 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4281 /* SS */
4282 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4283 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4284 if ( !(pCtx->cr0 & X86_CR0_PE)
4285 || pCtx->cs.Attr.n.u4Type == 3)
4286 {
4287 Assert(!pCtx->ss.Attr.n.u2Dpl);
4288 }
4289 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4290 {
4291 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4292 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4293 Assert(pCtx->ss.Attr.n.u1Present);
4294 Assert(!(pCtx->ss.Attr.u & 0xf00));
4295 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4296 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4297 || !(pCtx->ss.Attr.n.u1Granularity));
4298 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4299 || (pCtx->ss.Attr.n.u1Granularity));
4300 }
4301 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmentReg(). */
4302 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4303 {
4304 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4305 Assert(pCtx->ds.Attr.n.u1Present);
4306 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4307 Assert(!(pCtx->ds.Attr.u & 0xf00));
4308 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4309 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4310 || !(pCtx->ds.Attr.n.u1Granularity));
4311 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4312 || (pCtx->ds.Attr.n.u1Granularity));
4313 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4314 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4315 }
4316 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4317 {
4318 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4319 Assert(pCtx->es.Attr.n.u1Present);
4320 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4321 Assert(!(pCtx->es.Attr.u & 0xf00));
4322 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4323 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4324 || !(pCtx->es.Attr.n.u1Granularity));
4325 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4326 || (pCtx->es.Attr.n.u1Granularity));
4327 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4328 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4329 }
4330 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4331 {
4332 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4333 Assert(pCtx->fs.Attr.n.u1Present);
4334 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4335 Assert(!(pCtx->fs.Attr.u & 0xf00));
4336 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4337 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4338 || !(pCtx->fs.Attr.n.u1Granularity));
4339 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4340 || (pCtx->fs.Attr.n.u1Granularity));
4341 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4342 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4343 }
4344 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4345 {
4346 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4347 Assert(pCtx->gs.Attr.n.u1Present);
4348 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4349 Assert(!(pCtx->gs.Attr.u & 0xf00));
4350 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4351 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4352 || !(pCtx->gs.Attr.n.u1Granularity));
4353 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4354 || (pCtx->gs.Attr.n.u1Granularity));
4355 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4356 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4357 }
4358 /* 64-bit capable CPUs. */
4359# if HC_ARCH_BITS == 64
4360 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4361 Assert(!pCtx->ss.Attr.u || !RT_HI_U32(pCtx->ss.u64Base));
4362 Assert(!pCtx->ds.Attr.u || !RT_HI_U32(pCtx->ds.u64Base));
4363 Assert(!pCtx->es.Attr.u || !RT_HI_U32(pCtx->es.u64Base));
4364# endif
4365 }
4366 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4367 || ( CPUMIsGuestInRealModeEx(pCtx)
4368 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4369 {
4370 /* Real and v86 mode checks. */
4371 /* hmR0VmxExportGuestSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4372 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4373 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4374 {
4375 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4376 }
4377 else
4378 {
4379 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4380 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4381 }
4382
4383 /* CS */
4384 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4385 Assert(pCtx->cs.u32Limit == 0xffff);
4386 Assert(u32CSAttr == 0xf3);
4387 /* SS */
4388 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4389 Assert(pCtx->ss.u32Limit == 0xffff);
4390 Assert(u32SSAttr == 0xf3);
4391 /* DS */
4392 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4393 Assert(pCtx->ds.u32Limit == 0xffff);
4394 Assert(u32DSAttr == 0xf3);
4395 /* ES */
4396 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4397 Assert(pCtx->es.u32Limit == 0xffff);
4398 Assert(u32ESAttr == 0xf3);
4399 /* FS */
4400 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4401 Assert(pCtx->fs.u32Limit == 0xffff);
4402 Assert(u32FSAttr == 0xf3);
4403 /* GS */
4404 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4405 Assert(pCtx->gs.u32Limit == 0xffff);
4406 Assert(u32GSAttr == 0xf3);
4407 /* 64-bit capable CPUs. */
4408# if HC_ARCH_BITS == 64
4409 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4410 Assert(!u32SSAttr || !RT_HI_U32(pCtx->ss.u64Base));
4411 Assert(!u32DSAttr || !RT_HI_U32(pCtx->ds.u64Base));
4412 Assert(!u32ESAttr || !RT_HI_U32(pCtx->es.u64Base));
4413# endif
4414 }
4415}
4416#endif /* VBOX_STRICT */
4417
4418
4419/**
4420 * Exports a guest segment register into the guest-state area in the VMCS.
4421 *
4422 * @returns VBox status code.
4423 * @param pVCpu The cross context virtual CPU structure.
4424 * @param idxSel Index of the selector in the VMCS.
4425 * @param idxLimit Index of the segment limit in the VMCS.
4426 * @param idxBase Index of the segment base in the VMCS.
4427 * @param idxAccess Index of the access rights of the segment in the VMCS.
4428 * @param pSelReg Pointer to the segment selector.
4429 *
4430 * @remarks No-long-jump zone!!!
4431 */
4432static int hmR0VmxExportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
4433 PCCPUMSELREG pSelReg)
4434{
4435 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4436 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4437 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4438 AssertRCReturn(rc, rc);
4439
4440 uint32_t u32Access = pSelReg->Attr.u;
4441 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4442 {
4443 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4444 u32Access = 0xf3;
4445 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4446 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4447 }
4448 else
4449 {
4450 /*
4451 * The way to differentiate between whether this is really a null selector or was just
4452 * a selector loaded with 0 in real-mode is using the segment attributes. A selector
4453 * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4454 * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4455 * NULL selectors loaded in protected-mode have their attribute as 0.
4456 */
4457 if (!u32Access)
4458 u32Access = X86DESCATTR_UNUSABLE;
4459 }
4460
4461 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4462 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4463 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4464
4465 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4466 AssertRCReturn(rc, rc);
4467 return rc;
4468}
4469
4470
4471/**
4472 * Exports the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4473 * into the guest-state area in the VMCS.
4474 *
4475 * @returns VBox status code.
4476 * @param pVCpu The cross context virtual CPU structure.
4477 *
4478 * @remarks Will import guest CR0 on strict builds during validation of
4479 * segments.
4480 * @remarks No-long-jump zone!!!
4481 */
4482static int hmR0VmxExportGuestSegmentRegs(PVMCPU pVCpu)
4483{
4484 int rc = VERR_INTERNAL_ERROR_5;
4485 PVM pVM = pVCpu->CTX_SUFF(pVM);
4486 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4487
4488 /*
4489 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4490 */
4491 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4492 {
4493#ifdef VBOX_WITH_REM
4494 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4495 {
4496 Assert(pVM->hm.s.vmx.pRealModeTSS);
4497 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4498 if ( pVCpu->hm.s.vmx.fWasInRealMode
4499 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4500 {
4501 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4502 in real-mode (e.g. OpenBSD 4.0) */
4503 REMFlushTBs(pVM);
4504 Log4Func(("Switch to protected mode detected!\n"));
4505 pVCpu->hm.s.vmx.fWasInRealMode = false;
4506 }
4507 }
4508#endif
4509 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
4510 {
4511 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
4512 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4513 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pCtx->cs.Attr.u;
4514 rc = HMVMX_EXPORT_SREG(CS, &pCtx->cs);
4515 AssertRCReturn(rc, rc);
4516 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
4517 }
4518
4519 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
4520 {
4521 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
4522 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4523 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pCtx->ss.Attr.u;
4524 rc = HMVMX_EXPORT_SREG(SS, &pCtx->ss);
4525 AssertRCReturn(rc, rc);
4526 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
4527 }
4528
4529 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
4530 {
4531 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
4532 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4533 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pCtx->ds.Attr.u;
4534 rc = HMVMX_EXPORT_SREG(DS, &pCtx->ds);
4535 AssertRCReturn(rc, rc);
4536 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
4537 }
4538
4539 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
4540 {
4541 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
4542 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4543 pVCpu->hm.s.vmx.RealMode.AttrES.u = pCtx->es.Attr.u;
4544 rc = HMVMX_EXPORT_SREG(ES, &pCtx->es);
4545 AssertRCReturn(rc, rc);
4546 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
4547 }
4548
4549 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
4550 {
4551 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
4552 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4553 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pCtx->fs.Attr.u;
4554 rc = HMVMX_EXPORT_SREG(FS, &pCtx->fs);
4555 AssertRCReturn(rc, rc);
4556 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
4557 }
4558
4559 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
4560 {
4561 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
4562 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4563 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pCtx->gs.Attr.u;
4564 rc = HMVMX_EXPORT_SREG(GS, &pCtx->gs);
4565 AssertRCReturn(rc, rc);
4566 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
4567 }
4568
4569#ifdef VBOX_STRICT
4570 hmR0VmxValidateSegmentRegs(pVCpu);
4571#endif
4572
4573 Log4Func(("CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pCtx->cs.Sel, pCtx->cs.u64Base,
4574 pCtx->cs.u32Limit, pCtx->cs.Attr.u));
4575 }
4576
4577 /*
4578 * Guest TR.
4579 */
4580 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
4581 {
4582 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
4583
4584 /*
4585 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
4586 * achieved using the interrupt redirection bitmap (all bits cleared to let the guest
4587 * handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4588 */
4589 uint16_t u16Sel = 0;
4590 uint32_t u32Limit = 0;
4591 uint64_t u64Base = 0;
4592 uint32_t u32AccessRights = 0;
4593
4594 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4595 {
4596 u16Sel = pCtx->tr.Sel;
4597 u32Limit = pCtx->tr.u32Limit;
4598 u64Base = pCtx->tr.u64Base;
4599 u32AccessRights = pCtx->tr.Attr.u;
4600 }
4601 else
4602 {
4603 Assert(pVM->hm.s.vmx.pRealModeTSS);
4604 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
4605
4606 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4607 RTGCPHYS GCPhys;
4608 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4609 AssertRCReturn(rc, rc);
4610
4611 X86DESCATTR DescAttr;
4612 DescAttr.u = 0;
4613 DescAttr.n.u1Present = 1;
4614 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4615
4616 u16Sel = 0;
4617 u32Limit = HM_VTX_TSS_SIZE;
4618 u64Base = GCPhys; /* in real-mode phys = virt. */
4619 u32AccessRights = DescAttr.u;
4620 }
4621
4622 /* Validate. */
4623 Assert(!(u16Sel & RT_BIT(2)));
4624 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4625 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4626 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4627 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4628 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4629 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4630 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4631 Assert( (u32Limit & 0xfff) == 0xfff
4632 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4633 Assert( !(pCtx->tr.u32Limit & 0xfff00000)
4634 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4635
4636 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4637 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4638 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4639 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4640 AssertRCReturn(rc, rc);
4641
4642 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
4643 Log4Func(("TR base=%#RX64\n", pCtx->tr.u64Base));
4644 }
4645
4646 /*
4647 * Guest GDTR.
4648 */
4649 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4650 {
4651 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
4652
4653 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
4654 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
4655 AssertRCReturn(rc, rc);
4656
4657 /* Validate. */
4658 Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4659
4660 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4661 Log4Func(("GDTR base=%#RX64\n", pCtx->gdtr.pGdt));
4662 }
4663
4664 /*
4665 * Guest LDTR.
4666 */
4667 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4668 {
4669 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
4670
4671 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4672 uint32_t u32Access = 0;
4673 if (!pCtx->ldtr.Attr.u)
4674 u32Access = X86DESCATTR_UNUSABLE;
4675 else
4676 u32Access = pCtx->ldtr.Attr.u;
4677
4678 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
4679 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
4680 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4681 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
4682 AssertRCReturn(rc, rc);
4683
4684 /* Validate. */
4685 if (!(u32Access & X86DESCATTR_UNUSABLE))
4686 {
4687 Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4688 Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4689 Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4690 Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4691 Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4692 Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4693 Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
4694 || !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4695 Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
4696 || pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4697 }
4698
4699 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4700 Log4Func(("LDTR base=%#RX64\n", pCtx->ldtr.u64Base));
4701 }
4702
4703 /*
4704 * Guest IDTR.
4705 */
4706 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4707 {
4708 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
4709
4710 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
4711 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
4712 AssertRCReturn(rc, rc);
4713
4714 /* Validate. */
4715 Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4716
4717 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4718 Log4Func(("IDTR base=%#RX64\n", pCtx->idtr.pIdt));
4719 }
4720
4721 return VINF_SUCCESS;
4722}
4723
4724
4725/**
4726 * Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4727 * areas.
4728 *
4729 * These MSRs will automatically be loaded to the host CPU on every successful
4730 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4731 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4732 * -not- updated here for performance reasons. See hmR0VmxExportHostMsrs().
4733 *
4734 * Also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4735 *
4736 * @returns VBox status code.
4737 * @param pVCpu The cross context virtual CPU structure.
4738 *
4739 * @remarks No-long-jump zone!!!
4740 */
4741static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu)
4742{
4743 AssertPtr(pVCpu);
4744 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4745
4746 /*
4747 * MSRs that we use the auto-load/store MSR area in the VMCS.
4748 * For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
4749 */
4750 PVM pVM = pVCpu->CTX_SUFF(pVM);
4751 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4752 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4753 {
4754 if (pVM->hm.s.fAllow64BitGuests)
4755 {
4756#if HC_ARCH_BITS == 32
4757 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS | CPUMCTX_EXTRN_KERNEL_GS_BASE);
4758
4759 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pCtx->msrLSTAR, false, NULL);
4760 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pCtx->msrSTAR, false, NULL);
4761 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pCtx->msrSFMASK, false, NULL);
4762 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, false, NULL);
4763 AssertRCReturn(rc, rc);
4764# ifdef LOG_ENABLED
4765 PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4766 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4767 Log4Func(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4768# endif
4769#endif
4770 }
4771 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4772 }
4773
4774 /*
4775 * Guest Sysenter MSRs.
4776 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4777 * VM-exits on WRMSRs for these MSRs.
4778 */
4779 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4780 {
4781 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
4782
4783 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4784 {
4785 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
4786 AssertRCReturn(rc, rc);
4787 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4788 }
4789
4790 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4791 {
4792 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
4793 AssertRCReturn(rc, rc);
4794 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4795 }
4796
4797 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4798 {
4799 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
4800 AssertRCReturn(rc, rc);
4801 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4802 }
4803 }
4804
4805 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4806 {
4807 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
4808
4809 if (hmR0VmxShouldSwapEferMsr(pVCpu))
4810 {
4811 /*
4812 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4813 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4814 */
4815 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4816 {
4817 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
4818 AssertRCReturn(rc,rc);
4819 Log4Func(("EFER=%#RX64\n", pCtx->msrEFER));
4820 }
4821 else
4822 {
4823 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pCtx->msrEFER, false /* fUpdateHostMsr */,
4824 NULL /* pfAddedAndUpdated */);
4825 AssertRCReturn(rc, rc);
4826
4827 /* We need to intercept reads too, see @bugref{7386#c16}. */
4828 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
4829 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4830 Log4Func(("MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pCtx->msrEFER,
4831 pVCpu->hm.s.vmx.cMsrs));
4832 }
4833 }
4834 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4835 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4836 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4837 }
4838
4839 return VINF_SUCCESS;
4840}
4841
4842
4843#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4844/**
4845 * Check if guest state allows safe use of 32-bit switcher again.
4846 *
4847 * Segment bases and protected mode structures must be 32-bit addressable
4848 * because the 32-bit switcher will ignore high dword when writing these VMCS
4849 * fields. See @bugref{8432} for details.
4850 *
4851 * @returns true if safe, false if must continue to use the 64-bit switcher.
4852 * @param pCtx Pointer to the guest-CPU context.
4853 *
4854 * @remarks No-long-jump zone!!!
4855 */
4856static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
4857{
4858 if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4859 if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4860 if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4861 if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4862 if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4863 if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4864 if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4865 if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4866 if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4867 if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4868
4869 /* All good, bases are 32-bit. */
4870 return true;
4871}
4872#endif
4873
4874
4875/**
4876 * Selects up the appropriate function to run guest code.
4877 *
4878 * @returns VBox status code.
4879 * @param pVCpu The cross context virtual CPU structure.
4880 *
4881 * @remarks No-long-jump zone!!!
4882 */
4883static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu)
4884{
4885 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4886 if (CPUMIsGuestInLongModeEx(pCtx))
4887 {
4888#ifndef VBOX_ENABLE_64_BITS_GUESTS
4889 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4890#endif
4891 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4892#if HC_ARCH_BITS == 32
4893 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4894 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4895 {
4896#ifdef VBOX_STRICT
4897 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4898 {
4899 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4900 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4901 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4902 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4903 | HM_CHANGED_VMX_ENTRY_CTLS
4904 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4905 }
4906#endif
4907 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4908
4909 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4910 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4911 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4912 Log4Func(("Selected 64-bit switcher\n"));
4913 }
4914#else
4915 /* 64-bit host. */
4916 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4917#endif
4918 }
4919 else
4920 {
4921 /* Guest is not in long mode, use the 32-bit handler. */
4922#if HC_ARCH_BITS == 32
4923 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4924 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4925 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4926 {
4927# ifdef VBOX_STRICT
4928 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4929 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4930 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4931 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4932 | HM_CHANGED_VMX_ENTRY_CTLS
4933 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4934# endif
4935 }
4936# ifdef VBOX_ENABLE_64_BITS_GUESTS
4937 /*
4938 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
4939 * design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
4940 * switcher flag because now we know the guest is in a sane state where it's safe
4941 * to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4942 * the much faster 32-bit switcher again.
4943 */
4944 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4945 {
4946 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4947 Log4Func(("Selected 32-bit switcher\n"));
4948 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4949 }
4950 else
4951 {
4952 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4953 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4954 || hmR0VmxIs32BitSwitcherSafe(pCtx))
4955 {
4956 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4957 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4958 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
4959 | HM_CHANGED_VMX_ENTRY_CTLS
4960 | HM_CHANGED_VMX_EXIT_CTLS
4961 | HM_CHANGED_HOST_CONTEXT);
4962 Log4Func(("Selected 32-bit switcher (safe)\n"));
4963 }
4964 }
4965# else
4966 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4967# endif
4968#else
4969 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4970#endif
4971 }
4972 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4973 return VINF_SUCCESS;
4974}
4975
4976
4977/**
4978 * Wrapper for running the guest code in VT-x.
4979 *
4980 * @returns VBox status code, no informational status codes.
4981 * @param pVCpu The cross context virtual CPU structure.
4982 *
4983 * @remarks No-long-jump zone!!!
4984 */
4985DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu)
4986{
4987 /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4988 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4989 pCtx->fExtrn |= HMVMX_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
4990
4991 /*
4992 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4993 * floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
4994 * callee-saved and thus the need for this XMM wrapper.
4995 *
4996 * See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
4997 */
4998 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4999 /** @todo Add stats for resume vs launch. */
5000 PVM pVM = pVCpu->CTX_SUFF(pVM);
5001#ifdef VBOX_WITH_KERNEL_USING_XMM
5002 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
5003#else
5004 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
5005#endif
5006 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
5007 return rc;
5008}
5009
5010
5011/**
5012 * Reports world-switch error and dumps some useful debug info.
5013 *
5014 * @param pVCpu The cross context virtual CPU structure.
5015 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5016 * @param pVmxTransient Pointer to the VMX transient structure (only
5017 * exitReason updated).
5018 */
5019static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
5020{
5021 Assert(pVCpu);
5022 Assert(pVmxTransient);
5023 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
5024
5025 Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
5026 switch (rcVMRun)
5027 {
5028 case VERR_VMX_INVALID_VMXON_PTR:
5029 AssertFailed();
5030 break;
5031 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5032 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5033 {
5034 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5035 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5036 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
5037 AssertRC(rc);
5038
5039 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5040 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5041 Cannot do it here as we may have been long preempted. */
5042
5043#ifdef VBOX_STRICT
5044 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5045 pVmxTransient->uExitReason));
5046 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
5047 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5048 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5049 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5050 else
5051 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5052 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5053 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5054
5055 /* VMX control bits. */
5056 uint32_t u32Val;
5057 uint64_t u64Val;
5058 RTHCUINTREG uHCReg;
5059 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5060 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5061 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5062 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5063 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
5064 {
5065 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5066 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5067 }
5068 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5069 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5070 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5071 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5072 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5073 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5074 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5075 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5076 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5077 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5078 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5079 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5080 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5081 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5082 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5083 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5084 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5085 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5086 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5087 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5088 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5089 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5090 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5091 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5092 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5093 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5094 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5095 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5096 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5097 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5098 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5099 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5100 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5101 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5102 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5103 {
5104 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5105 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5106 }
5107
5108 /* Guest bits. */
5109 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5110 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
5111 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5112 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
5113 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5114 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
5115 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
5116 {
5117 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5118 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5119 }
5120
5121 /* Host bits. */
5122 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5123 Log4(("Host CR0 %#RHr\n", uHCReg));
5124 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5125 Log4(("Host CR3 %#RHr\n", uHCReg));
5126 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5127 Log4(("Host CR4 %#RHr\n", uHCReg));
5128
5129 RTGDTR HostGdtr;
5130 PCX86DESCHC pDesc;
5131 ASMGetGDTR(&HostGdtr);
5132 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5133 Log4(("Host CS %#08x\n", u32Val));
5134 if (u32Val < HostGdtr.cbGdt)
5135 {
5136 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5137 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5138 }
5139
5140 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5141 Log4(("Host DS %#08x\n", u32Val));
5142 if (u32Val < HostGdtr.cbGdt)
5143 {
5144 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5145 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5146 }
5147
5148 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5149 Log4(("Host ES %#08x\n", u32Val));
5150 if (u32Val < HostGdtr.cbGdt)
5151 {
5152 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5153 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5154 }
5155
5156 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5157 Log4(("Host FS %#08x\n", u32Val));
5158 if (u32Val < HostGdtr.cbGdt)
5159 {
5160 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5161 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5162 }
5163
5164 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5165 Log4(("Host GS %#08x\n", u32Val));
5166 if (u32Val < HostGdtr.cbGdt)
5167 {
5168 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5169 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5170 }
5171
5172 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5173 Log4(("Host SS %#08x\n", u32Val));
5174 if (u32Val < HostGdtr.cbGdt)
5175 {
5176 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5177 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5178 }
5179
5180 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5181 Log4(("Host TR %#08x\n", u32Val));
5182 if (u32Val < HostGdtr.cbGdt)
5183 {
5184 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5185 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5186 }
5187
5188 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5189 Log4(("Host TR Base %#RHv\n", uHCReg));
5190 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5191 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5192 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5193 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5194 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5195 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5196 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5197 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5198 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5199 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5200 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5201 Log4(("Host RSP %#RHv\n", uHCReg));
5202 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5203 Log4(("Host RIP %#RHv\n", uHCReg));
5204# if HC_ARCH_BITS == 64
5205 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5206 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5207 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5208 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5209 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5210 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5211# endif
5212#endif /* VBOX_STRICT */
5213 break;
5214 }
5215
5216 default:
5217 /* Impossible */
5218 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5219 break;
5220 }
5221}
5222
5223
5224#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5225#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5226# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5227#endif
5228#ifdef VBOX_STRICT
5229static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5230{
5231 switch (idxField)
5232 {
5233 case VMX_VMCS_GUEST_RIP:
5234 case VMX_VMCS_GUEST_RSP:
5235 case VMX_VMCS_GUEST_SYSENTER_EIP:
5236 case VMX_VMCS_GUEST_SYSENTER_ESP:
5237 case VMX_VMCS_GUEST_GDTR_BASE:
5238 case VMX_VMCS_GUEST_IDTR_BASE:
5239 case VMX_VMCS_GUEST_CS_BASE:
5240 case VMX_VMCS_GUEST_DS_BASE:
5241 case VMX_VMCS_GUEST_ES_BASE:
5242 case VMX_VMCS_GUEST_FS_BASE:
5243 case VMX_VMCS_GUEST_GS_BASE:
5244 case VMX_VMCS_GUEST_SS_BASE:
5245 case VMX_VMCS_GUEST_LDTR_BASE:
5246 case VMX_VMCS_GUEST_TR_BASE:
5247 case VMX_VMCS_GUEST_CR3:
5248 return true;
5249 }
5250 return false;
5251}
5252
5253static bool hmR0VmxIsValidReadField(uint32_t idxField)
5254{
5255 switch (idxField)
5256 {
5257 /* Read-only fields. */
5258 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5259 return true;
5260 }
5261 /* Remaining readable fields should also be writable. */
5262 return hmR0VmxIsValidWriteField(idxField);
5263}
5264#endif /* VBOX_STRICT */
5265
5266
5267/**
5268 * Executes the specified handler in 64-bit mode.
5269 *
5270 * @returns VBox status code (no informational status codes).
5271 * @param pVCpu The cross context virtual CPU structure.
5272 * @param enmOp The operation to perform.
5273 * @param cParams Number of parameters.
5274 * @param paParam Array of 32-bit parameters.
5275 */
5276VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
5277{
5278 PVM pVM = pVCpu->CTX_SUFF(pVM);
5279 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5280 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5281 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5282 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5283
5284#ifdef VBOX_STRICT
5285 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5286 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5287
5288 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5289 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5290#endif
5291
5292 /* Disable interrupts. */
5293 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5294
5295#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5296 RTCPUID idHostCpu = RTMpCpuId();
5297 CPUMR0SetLApic(pVCpu, idHostCpu);
5298#endif
5299
5300 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5301 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5302
5303 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5304 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5305 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5306
5307 /* Leave VMX Root Mode. */
5308 VMXDisable();
5309
5310 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5311
5312 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5313 CPUMSetHyperEIP(pVCpu, enmOp);
5314 for (int i = (int)cParams - 1; i >= 0; i--)
5315 CPUMPushHyper(pVCpu, paParam[i]);
5316
5317 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5318
5319 /* Call the switcher. */
5320 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
5321 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5322
5323 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5324 /* Make sure the VMX instructions don't cause #UD faults. */
5325 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5326
5327 /* Re-enter VMX Root Mode */
5328 int rc2 = VMXEnable(HCPhysCpuPage);
5329 if (RT_FAILURE(rc2))
5330 {
5331 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5332 ASMSetFlags(fOldEFlags);
5333 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5334 return rc2;
5335 }
5336
5337 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5338 AssertRC(rc2);
5339 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5340 Assert(!(ASMGetFlags() & X86_EFL_IF));
5341 ASMSetFlags(fOldEFlags);
5342 return rc;
5343}
5344
5345
5346/**
5347 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5348 * supporting 64-bit guests.
5349 *
5350 * @returns VBox status code.
5351 * @param fResume Whether to VMLAUNCH or VMRESUME.
5352 * @param pCtx Pointer to the guest-CPU context.
5353 * @param pCache Pointer to the VMCS cache.
5354 * @param pVM The cross context VM structure.
5355 * @param pVCpu The cross context virtual CPU structure.
5356 */
5357DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5358{
5359 NOREF(fResume);
5360
5361 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5362 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5363
5364#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5365 pCache->uPos = 1;
5366 pCache->interPD = PGMGetInterPaeCR3(pVM);
5367 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5368#endif
5369
5370#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5371 pCache->TestIn.HCPhysCpuPage = 0;
5372 pCache->TestIn.HCPhysVmcs = 0;
5373 pCache->TestIn.pCache = 0;
5374 pCache->TestOut.HCPhysVmcs = 0;
5375 pCache->TestOut.pCache = 0;
5376 pCache->TestOut.pCtx = 0;
5377 pCache->TestOut.eflags = 0;
5378#else
5379 NOREF(pCache);
5380#endif
5381
5382 uint32_t aParam[10];
5383 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5384 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5385 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5386 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5387 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5388 aParam[5] = 0;
5389 aParam[6] = VM_RC_ADDR(pVM, pVM);
5390 aParam[7] = 0;
5391 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5392 aParam[9] = 0;
5393
5394#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5395 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5396 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5397#endif
5398 int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5399
5400#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5401 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5402 Assert(pCtx->dr[4] == 10);
5403 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5404#endif
5405
5406#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5407 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5408 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5409 pVCpu->hm.s.vmx.HCPhysVmcs));
5410 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5411 pCache->TestOut.HCPhysVmcs));
5412 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5413 pCache->TestOut.pCache));
5414 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5415 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5416 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5417 pCache->TestOut.pCtx));
5418 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5419#endif
5420 NOREF(pCtx);
5421 return rc;
5422}
5423
5424
5425/**
5426 * Initialize the VMCS-Read cache.
5427 *
5428 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5429 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5430 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5431 * (those that have a 32-bit FULL & HIGH part).
5432 *
5433 * @returns VBox status code.
5434 * @param pVCpu The cross context virtual CPU structure.
5435 */
5436static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
5437{
5438#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5439 do { \
5440 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5441 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5442 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5443 ++cReadFields; \
5444 } while (0)
5445
5446 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5447 uint32_t cReadFields = 0;
5448
5449 /*
5450 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5451 * and serve to indicate exceptions to the rules.
5452 */
5453
5454 /* Guest-natural selector base fields. */
5455#if 0
5456 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5457 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5458 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5459#endif
5460 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5461 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5462 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5463 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5464 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5465 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5466 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5467 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5468 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5469 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5470 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5471 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5472#if 0
5473 /* Unused natural width guest-state fields. */
5474 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5475 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5476#endif
5477 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5478 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5479
5480 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
5481 these 64-bit fields (using "FULL" and "HIGH" fields). */
5482#if 0
5483 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5484 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5485 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5486 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5487 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5488 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5489 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5490 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5491 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5492#endif
5493
5494 /* Natural width guest-state fields. */
5495 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5496#if 0
5497 /* Currently unused field. */
5498 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5499#endif
5500
5501 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5502 {
5503 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5504 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5505 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5506 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5507 }
5508 else
5509 {
5510 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5511 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5512 }
5513
5514#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5515 return VINF_SUCCESS;
5516}
5517
5518
5519/**
5520 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5521 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5522 * darwin, running 64-bit guests).
5523 *
5524 * @returns VBox status code.
5525 * @param pVCpu The cross context virtual CPU structure.
5526 * @param idxField The VMCS field encoding.
5527 * @param u64Val 16, 32 or 64-bit value.
5528 */
5529VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5530{
5531 int rc;
5532 switch (idxField)
5533 {
5534 /*
5535 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5536 */
5537 /* 64-bit Control fields. */
5538 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5539 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5540 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5541 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5542 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5543 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5544 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5545 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5546 case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
5547 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5548 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5549 case VMX_VMCS64_CTRL_EPTP_FULL:
5550 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5551 /* 64-bit Guest-state fields. */
5552 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5553 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5554 case VMX_VMCS64_GUEST_PAT_FULL:
5555 case VMX_VMCS64_GUEST_EFER_FULL:
5556 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5557 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5558 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5559 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5560 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5561 /* 64-bit Host-state fields. */
5562 case VMX_VMCS64_HOST_PAT_FULL:
5563 case VMX_VMCS64_HOST_EFER_FULL:
5564 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5565 {
5566 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5567 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5568 break;
5569 }
5570
5571 /*
5572 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5573 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5574 */
5575 /* Natural-width Guest-state fields. */
5576 case VMX_VMCS_GUEST_CR3:
5577 case VMX_VMCS_GUEST_ES_BASE:
5578 case VMX_VMCS_GUEST_CS_BASE:
5579 case VMX_VMCS_GUEST_SS_BASE:
5580 case VMX_VMCS_GUEST_DS_BASE:
5581 case VMX_VMCS_GUEST_FS_BASE:
5582 case VMX_VMCS_GUEST_GS_BASE:
5583 case VMX_VMCS_GUEST_LDTR_BASE:
5584 case VMX_VMCS_GUEST_TR_BASE:
5585 case VMX_VMCS_GUEST_GDTR_BASE:
5586 case VMX_VMCS_GUEST_IDTR_BASE:
5587 case VMX_VMCS_GUEST_RSP:
5588 case VMX_VMCS_GUEST_RIP:
5589 case VMX_VMCS_GUEST_SYSENTER_ESP:
5590 case VMX_VMCS_GUEST_SYSENTER_EIP:
5591 {
5592 if (!(RT_HI_U32(u64Val)))
5593 {
5594 /* If this field is 64-bit, VT-x will zero out the top bits. */
5595 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5596 }
5597 else
5598 {
5599 /* Assert that only the 32->64 switcher case should ever come here. */
5600 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5601 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5602 }
5603 break;
5604 }
5605
5606 default:
5607 {
5608 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5609 rc = VERR_INVALID_PARAMETER;
5610 break;
5611 }
5612 }
5613 AssertRCReturn(rc, rc);
5614 return rc;
5615}
5616
5617
5618/**
5619 * Queue up a VMWRITE by using the VMCS write cache.
5620 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5621 *
5622 * @param pVCpu The cross context virtual CPU structure.
5623 * @param idxField The VMCS field encoding.
5624 * @param u64Val 16, 32 or 64-bit value.
5625 */
5626VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5627{
5628 AssertPtr(pVCpu);
5629 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5630
5631 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5632 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5633
5634 /* Make sure there are no duplicates. */
5635 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5636 {
5637 if (pCache->Write.aField[i] == idxField)
5638 {
5639 pCache->Write.aFieldVal[i] = u64Val;
5640 return VINF_SUCCESS;
5641 }
5642 }
5643
5644 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5645 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5646 pCache->Write.cValidEntries++;
5647 return VINF_SUCCESS;
5648}
5649#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5650
5651
5652/**
5653 * Sets up the usage of TSC-offsetting and updates the VMCS.
5654 *
5655 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5656 * VMX preemption timer.
5657 *
5658 * @returns VBox status code.
5659 * @param pVCpu The cross context virtual CPU structure.
5660 *
5661 * @remarks No-long-jump zone!!!
5662 */
5663static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5664{
5665 bool fOffsettedTsc;
5666 bool fParavirtTsc;
5667 PVM pVM = pVCpu->CTX_SUFF(pVM);
5668 uint64_t uTscOffset;
5669 if (pVM->hm.s.vmx.fUsePreemptTimer)
5670 {
5671 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
5672
5673 /* Make sure the returned values have sane upper and lower boundaries. */
5674 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5675 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5676 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5677 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5678
5679 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5680 int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
5681 AssertRC(rc);
5682 }
5683 else
5684 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
5685
5686 if (fParavirtTsc)
5687 {
5688 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5689 information before every VM-entry, hence disable it for performance sake. */
5690#if 0
5691 int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5692 AssertRC(rc);
5693#endif
5694 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5695 }
5696
5697 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
5698 if ( fOffsettedTsc
5699 && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5700 {
5701 if (pVCpu->hm.s.vmx.u64TscOffset != uTscOffset)
5702 {
5703 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
5704 AssertRC(rc);
5705 pVCpu->hm.s.vmx.u64TscOffset = uTscOffset;
5706 }
5707
5708 if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
5709 {
5710 uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
5711 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5712 AssertRC(rc);
5713 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5714 }
5715 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5716 }
5717 else
5718 {
5719 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5720 if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
5721 {
5722 uProcCtls |= VMX_PROC_CTLS_RDTSC_EXIT;
5723 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5724 AssertRC(rc);
5725 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5726 }
5727 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5728 }
5729}
5730
5731
5732/**
5733 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5734 * VM-exit interruption info type.
5735 *
5736 * @returns The IEM exception flags.
5737 * @param uVector The event vector.
5738 * @param uVmxVectorType The VMX event type.
5739 *
5740 * @remarks This function currently only constructs flags required for
5741 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5742 * and CR2 aspects of an exception are not included).
5743 */
5744static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5745{
5746 uint32_t fIemXcptFlags;
5747 switch (uVmxVectorType)
5748 {
5749 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5750 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5751 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5752 break;
5753
5754 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5755 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5756 break;
5757
5758 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5759 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5760 break;
5761
5762 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5763 {
5764 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5765 if (uVector == X86_XCPT_BP)
5766 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5767 else if (uVector == X86_XCPT_OF)
5768 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5769 else
5770 {
5771 fIemXcptFlags = 0;
5772 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5773 }
5774 break;
5775 }
5776
5777 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5778 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5779 break;
5780
5781 default:
5782 fIemXcptFlags = 0;
5783 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5784 break;
5785 }
5786 return fIemXcptFlags;
5787}
5788
5789
5790/**
5791 * Sets an event as a pending event to be injected into the guest.
5792 *
5793 * @param pVCpu The cross context virtual CPU structure.
5794 * @param u32IntInfo The VM-entry interruption-information field.
5795 * @param cbInstr The VM-entry instruction length in bytes (for software
5796 * interrupts, exceptions and privileged software
5797 * exceptions).
5798 * @param u32ErrCode The VM-entry exception error code.
5799 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5800 * page-fault.
5801 *
5802 * @remarks Statistics counter assumes this is a guest event being injected or
5803 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5804 * always incremented.
5805 */
5806DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5807 RTGCUINTPTR GCPtrFaultAddress)
5808{
5809 Assert(!pVCpu->hm.s.Event.fPending);
5810 pVCpu->hm.s.Event.fPending = true;
5811 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5812 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5813 pVCpu->hm.s.Event.cbInstr = cbInstr;
5814 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5815}
5816
5817
5818/**
5819 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5820 *
5821 * @param pVCpu The cross context virtual CPU structure.
5822 */
5823DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
5824{
5825 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
5826 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5827 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
5828 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5829 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5830}
5831
5832
5833/**
5834 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
5835 *
5836 * @param pVCpu The cross context virtual CPU structure.
5837 */
5838DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
5839{
5840 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
5841 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5842 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
5843 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5844 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5845}
5846
5847
5848/**
5849 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
5850 *
5851 * @param pVCpu The cross context virtual CPU structure.
5852 */
5853DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
5854{
5855 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
5856 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5857 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
5858 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5859 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5860}
5861
5862
5863#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5864/**
5865 * Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
5866 *
5867 * @param pVCpu The cross context virtual CPU structure.
5868 * @param u32ErrCode The error code for the general-protection exception.
5869 */
5870DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
5871{
5872 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
5873 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5874 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
5875 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5876 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrCode, 0 /* GCPtrFaultAddress */);
5877}
5878
5879
5880/**
5881 * Sets a stack (\#SS) exception as pending-for-injection into the VM.
5882 *
5883 * @param pVCpu The cross context virtual CPU structure.
5884 * @param u32ErrCode The error code for the stack exception.
5885 */
5886DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
5887{
5888 uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
5889 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
5890 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
5891 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
5892 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrCode, 0 /* GCPtrFaultAddress */);
5893}
5894
5895
5896/**
5897 * Decodes the memory operand of an instruction that caused a VM-exit.
5898 *
5899 * The VM-exit qualification field provides the displacement field for memory
5900 * operand instructions, if any.
5901 *
5902 * @returns Strict VBox status code (i.e. informational status codes too).
5903 * @retval VINF_SUCCESS if the operand was successfully decoded.
5904 * @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
5905 * operand.
5906 * @param pVCpu The cross context virtual CPU structure.
5907 * @param uExitInstrInfo The VM-exit instruction information field.
5908 * @param enmMemAccess The memory operand's access type (read or write).
5909 * @param GCPtrDisp The instruction displacement field, if any. For
5910 * RIP-relative addressing pass RIP + displacement here.
5911 * @param pGCPtrMem Where to store the effective destination memory address.
5912 */
5913static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
5914 PRTGCPTR pGCPtrMem)
5915{
5916 Assert(pGCPtrMem);
5917 Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
5918 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_EFER | CPUMCTX_EXTRN_CR0);
5919
5920 static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
5921 static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
5922 AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
5923
5924 VMXEXITINSTRINFO ExitInstrInfo;
5925 ExitInstrInfo.u = uExitInstrInfo;
5926 uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
5927 uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
5928 bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
5929 uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
5930 uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
5931 bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
5932 uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
5933 bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
5934 bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
5935
5936 /*
5937 * Validate instruction information.
5938 * This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
5939 */
5940 AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
5941 ("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
5942 AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
5943 ("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
5944 AssertLogRelMsgReturn(fIsMemOperand,
5945 ("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
5946
5947 /*
5948 * Compute the complete effective address.
5949 *
5950 * See AMD instruction spec. 1.4.2 "SIB Byte Format"
5951 * See AMD spec. 4.5.2 "Segment Registers".
5952 */
5953 RTGCPTR GCPtrMem = GCPtrDisp;
5954 if (fBaseRegValid)
5955 GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
5956 if (fIdxRegValid)
5957 GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
5958
5959 RTGCPTR const GCPtrOff = GCPtrMem;
5960 if ( !fIsLongMode
5961 || iSegReg >= X86_SREG_FS)
5962 GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
5963 GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
5964
5965 /*
5966 * Validate effective address.
5967 * See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
5968 */
5969 uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
5970 Assert(cbAccess > 0);
5971 if (fIsLongMode)
5972 {
5973 if (X86_IS_CANONICAL(GCPtrMem))
5974 {
5975 *pGCPtrMem = GCPtrMem;
5976 return VINF_SUCCESS;
5977 }
5978
5979 /** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
5980 * "Data Limit Checks in 64-bit Mode". */
5981 Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
5982 hmR0VmxSetPendingXcptGP(pVCpu, 0);
5983 return VINF_HM_PENDING_XCPT;
5984 }
5985
5986 /*
5987 * This is a watered down version of iemMemApplySegment().
5988 * Parts that are not applicable for VMX instructions like real-or-v8086 mode
5989 * and segment CPL/DPL checks are skipped.
5990 */
5991 RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
5992 RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
5993 PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
5994
5995 /* Check if the segment is present and usable. */
5996 if ( pSel->Attr.n.u1Present
5997 && !pSel->Attr.n.u1Unusable)
5998 {
5999 Assert(pSel->Attr.n.u1DescType);
6000 if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
6001 {
6002 /* Check permissions for the data segment. */
6003 if ( enmMemAccess == VMXMEMACCESS_WRITE
6004 && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
6005 {
6006 Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6007 hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
6008 return VINF_HM_PENDING_XCPT;
6009 }
6010
6011 /* Check limits if it's a normal data segment. */
6012 if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
6013 {
6014 if ( GCPtrFirst32 > pSel->u32Limit
6015 || GCPtrLast32 > pSel->u32Limit)
6016 {
6017 Log4Func(("Data segment limit exceeded."
6018 "iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6019 GCPtrLast32, pSel->u32Limit));
6020 if (iSegReg == X86_SREG_SS)
6021 hmR0VmxSetPendingXcptSS(pVCpu, 0);
6022 else
6023 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6024 return VINF_HM_PENDING_XCPT;
6025 }
6026 }
6027 else
6028 {
6029 /* Check limits if it's an expand-down data segment.
6030 Note! The upper boundary is defined by the B bit, not the G bit! */
6031 if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
6032 || GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
6033 {
6034 Log4Func(("Expand-down data segment limit exceeded."
6035 "iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6036 GCPtrLast32, pSel->u32Limit));
6037 if (iSegReg == X86_SREG_SS)
6038 hmR0VmxSetPendingXcptSS(pVCpu, 0);
6039 else
6040 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6041 return VINF_HM_PENDING_XCPT;
6042 }
6043 }
6044 }
6045 else
6046 {
6047 /* Check permissions for the code segment. */
6048 if ( enmMemAccess == VMXMEMACCESS_WRITE
6049 || ( enmMemAccess == VMXMEMACCESS_READ
6050 && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
6051 {
6052 Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
6053 Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
6054 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6055 return VINF_HM_PENDING_XCPT;
6056 }
6057
6058 /* Check limits for the code segment (normal/expand-down not applicable for code segments). */
6059 if ( GCPtrFirst32 > pSel->u32Limit
6060 || GCPtrLast32 > pSel->u32Limit)
6061 {
6062 Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
6063 GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
6064 if (iSegReg == X86_SREG_SS)
6065 hmR0VmxSetPendingXcptSS(pVCpu, 0);
6066 else
6067 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6068 return VINF_HM_PENDING_XCPT;
6069 }
6070 }
6071 }
6072 else
6073 {
6074 Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6075 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6076 return VINF_HM_PENDING_XCPT;
6077 }
6078
6079 *pGCPtrMem = GCPtrMem;
6080 return VINF_SUCCESS;
6081}
6082
6083
6084/**
6085 * Perform the relevant VMX instruction checks for VM-exits that occurred due to the
6086 * guest attempting to execute a VMX instruction.
6087 *
6088 * @returns Strict VBox status code (i.e. informational status codes too).
6089 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
6090 * @retval VINF_HM_PENDING_XCPT if an exception was raised.
6091 *
6092 * @param pVCpu The cross context virtual CPU structure.
6093 * @param uExitReason The VM-exit reason.
6094 *
6095 * @todo NstVmx: Document other error codes when VM-exit is implemented.
6096 * @remarks No-long-jump zone!!!
6097 */
6098static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
6099{
6100 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS
6101 | CPUMCTX_EXTRN_HWVIRT);
6102
6103 if ( CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx)
6104 || ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6105 && !CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx)))
6106 {
6107 Log4Func(("In real/v86-mode or long-mode outside 64-bit code segment -> #UD\n"));
6108 hmR0VmxSetPendingXcptUD(pVCpu);
6109 return VINF_HM_PENDING_XCPT;
6110 }
6111
6112 if (uExitReason == VMX_EXIT_VMXON)
6113 {
6114 /*
6115 * We check CR4.VMXE because it is required to be always set while in VMX operation
6116 * by physical CPUs and our CR4 read shadow is only consulted when executing specific
6117 * instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
6118 * otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
6119 */
6120 if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
6121 {
6122 Log4Func(("CR4.VMXE is not set -> #UD\n"));
6123 hmR0VmxSetPendingXcptUD(pVCpu);
6124 return VINF_HM_PENDING_XCPT;
6125 }
6126 }
6127 else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
6128 {
6129 /*
6130 * The guest has not entered VMX operation but attempted to execute a VMX instruction
6131 * (other than VMXON), we need to raise a #UD.
6132 */
6133 Log4Func(("Not in VMX root mode -> #UD\n"));
6134 hmR0VmxSetPendingXcptUD(pVCpu);
6135 return VINF_HM_PENDING_XCPT;
6136 }
6137
6138 if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
6139 {
6140 /*
6141 * The nested-guest attempted to execute a VMX instruction, cause a VM-exit and let
6142 * the guest hypervisor deal with it.
6143 */
6144 /** @todo NSTVMX: Trigger a VM-exit */
6145 }
6146
6147 /*
6148 * VMX instructions require CPL 0 except in VMX non-root mode where the VM-exit intercept
6149 * (above) takes preceedence over the CPL check.
6150 */
6151 if (CPUMGetGuestCPL(pVCpu) > 0)
6152 {
6153 Log4Func(("CPL > 0 -> #GP(0)\n"));
6154 hmR0VmxSetPendingXcptGP(pVCpu, 0);
6155 return VINF_HM_PENDING_XCPT;
6156 }
6157
6158 return VINF_SUCCESS;
6159}
6160
6161#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
6162
6163
6164/**
6165 * Handle a condition that occurred while delivering an event through the guest
6166 * IDT.
6167 *
6168 * @returns Strict VBox status code (i.e. informational status codes too).
6169 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
6170 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
6171 * to continue execution of the guest which will delivery the \#DF.
6172 * @retval VINF_EM_RESET if we detected a triple-fault condition.
6173 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
6174 *
6175 * @param pVCpu The cross context virtual CPU structure.
6176 * @param pVmxTransient Pointer to the VMX transient structure.
6177 *
6178 * @remarks No-long-jump zone!!!
6179 */
6180static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6181{
6182 uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
6183
6184 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
6185 rc2 |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
6186 AssertRCReturn(rc2, rc2);
6187
6188 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
6189 if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6190 {
6191 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
6192 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
6193
6194 /*
6195 * If the event was a software interrupt (generated with INT n) or a software exception
6196 * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
6197 * can handle the VM-exit and continue guest execution which will re-execute the
6198 * instruction rather than re-injecting the exception, as that can cause premature
6199 * trips to ring-3 before injection and involve TRPM which currently has no way of
6200 * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
6201 * the problem).
6202 */
6203 IEMXCPTRAISE enmRaise;
6204 IEMXCPTRAISEINFO fRaiseInfo;
6205 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6206 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6207 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6208 {
6209 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
6210 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6211 }
6212 else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
6213 {
6214 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
6215 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
6216 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
6217 /** @todo Make AssertMsgReturn as just AssertMsg later. */
6218 AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
6219 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
6220 uExitVectorType), VERR_VMX_IPE_5);
6221
6222 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
6223
6224 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
6225 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
6226 {
6227 pVmxTransient->fVectoringPF = true;
6228 enmRaise = IEMXCPTRAISE_PREV_EVENT;
6229 }
6230 }
6231 else
6232 {
6233 /*
6234 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
6235 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
6236 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
6237 */
6238 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6239 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6240 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
6241 enmRaise = IEMXCPTRAISE_PREV_EVENT;
6242 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6243 }
6244
6245 /*
6246 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
6247 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
6248 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
6249 * subsequent VM-entry would fail.
6250 *
6251 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
6252 */
6253 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
6254 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6255 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
6256 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
6257 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
6258 {
6259 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6260 }
6261
6262 switch (enmRaise)
6263 {
6264 case IEMXCPTRAISE_CURRENT_XCPT:
6265 {
6266 Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
6267 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
6268 Assert(rcStrict == VINF_SUCCESS);
6269 break;
6270 }
6271
6272 case IEMXCPTRAISE_PREV_EVENT:
6273 {
6274 uint32_t u32ErrCode;
6275 if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
6276 {
6277 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6278 AssertRCReturn(rc2, rc2);
6279 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6280 }
6281 else
6282 u32ErrCode = 0;
6283
6284 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
6285 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6286 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6287 0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
6288
6289 Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
6290 pVCpu->hm.s.Event.u32ErrCode));
6291 Assert(rcStrict == VINF_SUCCESS);
6292 break;
6293 }
6294
6295 case IEMXCPTRAISE_REEXEC_INSTR:
6296 Assert(rcStrict == VINF_SUCCESS);
6297 break;
6298
6299 case IEMXCPTRAISE_DOUBLE_FAULT:
6300 {
6301 /*
6302 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
6303 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
6304 */
6305 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6306 {
6307 pVmxTransient->fVectoringDoublePF = true;
6308 Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
6309 pVCpu->cpum.GstCtx.cr2));
6310 rcStrict = VINF_SUCCESS;
6311 }
6312 else
6313 {
6314 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6315 hmR0VmxSetPendingXcptDF(pVCpu);
6316 Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
6317 uIdtVector, uExitVector));
6318 rcStrict = VINF_HM_DOUBLE_FAULT;
6319 }
6320 break;
6321 }
6322
6323 case IEMXCPTRAISE_TRIPLE_FAULT:
6324 {
6325 Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
6326 rcStrict = VINF_EM_RESET;
6327 break;
6328 }
6329
6330 case IEMXCPTRAISE_CPU_HANG:
6331 {
6332 Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
6333 rcStrict = VERR_EM_GUEST_CPU_HANG;
6334 break;
6335 }
6336
6337 default:
6338 {
6339 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6340 rcStrict = VERR_VMX_IPE_2;
6341 break;
6342 }
6343 }
6344 }
6345 else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6346 && VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6347 && uExitVector != X86_XCPT_DF
6348 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
6349 {
6350 /*
6351 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6352 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6353 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6354 */
6355 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6356 {
6357 Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
6358 VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6359 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6360 }
6361 }
6362
6363 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
6364 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
6365 return rcStrict;
6366}
6367
6368
6369/**
6370 * Imports a guest segment register from the current VMCS into
6371 * the guest-CPU context.
6372 *
6373 * @returns VBox status code.
6374 * @param pVCpu The cross context virtual CPU structure.
6375 * @param idxSel Index of the selector in the VMCS.
6376 * @param idxLimit Index of the segment limit in the VMCS.
6377 * @param idxBase Index of the segment base in the VMCS.
6378 * @param idxAccess Index of the access rights of the segment in the VMCS.
6379 * @param pSelReg Pointer to the segment selector.
6380 *
6381 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6382 * do not log!
6383 *
6384 * @remarks Never call this function directly!!! Use the
6385 * HMVMX_IMPORT_SREG() macro as that takes care
6386 * of whether to read from the VMCS cache or not.
6387 */
6388static int hmR0VmxImportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6389 PCPUMSELREG pSelReg)
6390{
6391 NOREF(pVCpu);
6392
6393 uint32_t u32Sel;
6394 uint32_t u32Limit;
6395 uint32_t u32Attr;
6396 uint64_t u64Base;
6397 int rc = VMXReadVmcs32(idxSel, &u32Sel);
6398 rc |= VMXReadVmcs32(idxLimit, &u32Limit);
6399 rc |= VMXReadVmcs32(idxAccess, &u32Attr);
6400 rc |= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
6401 AssertRCReturn(rc, rc);
6402
6403 pSelReg->Sel = (uint16_t)u32Sel;
6404 pSelReg->ValidSel = (uint16_t)u32Sel;
6405 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6406 pSelReg->u32Limit = u32Limit;
6407 pSelReg->u64Base = u64Base;
6408 pSelReg->Attr.u = u32Attr;
6409
6410 /*
6411 * If VT-x marks the segment as unusable, most other bits remain undefined:
6412 * - For CS the L, D and G bits have meaning.
6413 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6414 * - For the remaining data segments no bits are defined.
6415 *
6416 * The present bit and the unusable bit has been observed to be set at the
6417 * same time (the selector was supposed to be invalid as we started executing
6418 * a V8086 interrupt in ring-0).
6419 *
6420 * What should be important for the rest of the VBox code, is that the P bit is
6421 * cleared. Some of the other VBox code recognizes the unusable bit, but
6422 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6423 * safe side here, we'll strip off P and other bits we don't care about. If
6424 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6425 *
6426 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6427 */
6428 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6429 {
6430 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6431
6432 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6433 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6434 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6435#ifdef VBOX_STRICT
6436 VMMRZCallRing3Disable(pVCpu);
6437 Log4Func(("Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Sel, pSelReg->Attr.u));
6438# ifdef DEBUG_bird
6439 AssertMsg((u32Attr & ~X86DESCATTR_P) == pSelReg->Attr.u,
6440 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6441 idxSel, u32Sel, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6442# endif
6443 VMMRZCallRing3Enable(pVCpu);
6444#endif
6445 }
6446 return VINF_SUCCESS;
6447}
6448
6449
6450/**
6451 * Imports the guest RIP from the VMCS back into the guest-CPU context.
6452 *
6453 * @returns VBox status code.
6454 * @param pVCpu The cross context virtual CPU structure.
6455 *
6456 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6457 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6458 * instead!!!
6459 */
6460DECLINLINE(int) hmR0VmxImportGuestRip(PVMCPU pVCpu)
6461{
6462 uint64_t u64Val;
6463 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6464 if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
6465 {
6466 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6467 if (RT_SUCCESS(rc))
6468 {
6469 pCtx->rip = u64Val;
6470 EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
6471 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
6472 }
6473 return rc;
6474 }
6475 return VINF_SUCCESS;
6476}
6477
6478
6479/**
6480 * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
6481 *
6482 * @returns VBox status code.
6483 * @param pVCpu The cross context virtual CPU structure.
6484 *
6485 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6486 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6487 * instead!!!
6488 */
6489DECLINLINE(int) hmR0VmxImportGuestRFlags(PVMCPU pVCpu)
6490{
6491 uint32_t u32Val;
6492 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6493 if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
6494 {
6495 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
6496 if (RT_SUCCESS(rc))
6497 {
6498 pCtx->eflags.u32 = u32Val;
6499
6500 /* Restore eflags for real-on-v86-mode hack. */
6501 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6502 {
6503 pCtx->eflags.Bits.u1VM = 0;
6504 pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6505 }
6506 }
6507 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
6508 return rc;
6509 }
6510 return VINF_SUCCESS;
6511}
6512
6513
6514/**
6515 * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
6516 * context.
6517 *
6518 * @returns VBox status code.
6519 * @param pVCpu The cross context virtual CPU structure.
6520 *
6521 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6522 * do not log!
6523 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6524 * instead!!!
6525 */
6526DECLINLINE(int) hmR0VmxImportGuestIntrState(PVMCPU pVCpu)
6527{
6528 uint32_t u32Val;
6529 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6530 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
6531 if (RT_SUCCESS(rc))
6532 {
6533 /*
6534 * We additionally have a requirement to import RIP, RFLAGS depending on whether we
6535 * might need them in hmR0VmxEvaluatePendingEvent().
6536 */
6537 if (!u32Val)
6538 {
6539 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6540 {
6541 rc = hmR0VmxImportGuestRip(pVCpu);
6542 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6543 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6544 }
6545
6546 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6547 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6548 }
6549 else
6550 {
6551 rc = hmR0VmxImportGuestRip(pVCpu);
6552 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6553
6554 if (u32Val & ( VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
6555 | VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
6556 {
6557 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
6558 }
6559 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6560 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6561
6562 if (u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
6563 {
6564 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6565 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6566 }
6567 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6568 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6569 }
6570 }
6571 return rc;
6572}
6573
6574
6575/**
6576 * Worker for VMXR0ImportStateOnDemand.
6577 *
6578 * @returns VBox status code.
6579 * @param pVCpu The cross context virtual CPU structure.
6580 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6581 */
6582static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
6583{
6584#define VMXLOCAL_BREAK_RC(a_rc) \
6585 if (RT_FAILURE(a_rc)) \
6586 break
6587
6588 int rc = VINF_SUCCESS;
6589 PVM pVM = pVCpu->CTX_SUFF(pVM);
6590 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6591 uint64_t u64Val;
6592 uint32_t u32Val;
6593
6594 Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
6595 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
6596
6597 /*
6598 * We disable interrupts to make the updating of the state and in particular
6599 * the fExtrn modification atomic wrt to preemption hooks.
6600 */
6601 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
6602
6603 fWhat &= pCtx->fExtrn;
6604 if (fWhat)
6605 {
6606 do
6607 {
6608 if (fWhat & CPUMCTX_EXTRN_RIP)
6609 {
6610 rc = hmR0VmxImportGuestRip(pVCpu);
6611 VMXLOCAL_BREAK_RC(rc);
6612 }
6613
6614 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
6615 {
6616 rc = hmR0VmxImportGuestRFlags(pVCpu);
6617 VMXLOCAL_BREAK_RC(rc);
6618 }
6619
6620 if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
6621 {
6622 rc = hmR0VmxImportGuestIntrState(pVCpu);
6623 VMXLOCAL_BREAK_RC(rc);
6624 }
6625
6626 if (fWhat & CPUMCTX_EXTRN_RSP)
6627 {
6628 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6629 VMXLOCAL_BREAK_RC(rc);
6630 pCtx->rsp = u64Val;
6631 }
6632
6633 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
6634 {
6635 if (fWhat & CPUMCTX_EXTRN_CS)
6636 {
6637 rc = HMVMX_IMPORT_SREG(CS, &pCtx->cs);
6638 rc |= hmR0VmxImportGuestRip(pVCpu);
6639 VMXLOCAL_BREAK_RC(rc);
6640 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6641 pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6642 EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true);
6643 }
6644 if (fWhat & CPUMCTX_EXTRN_SS)
6645 {
6646 rc = HMVMX_IMPORT_SREG(SS, &pCtx->ss);
6647 VMXLOCAL_BREAK_RC(rc);
6648 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6649 pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6650 }
6651 if (fWhat & CPUMCTX_EXTRN_DS)
6652 {
6653 rc = HMVMX_IMPORT_SREG(DS, &pCtx->ds);
6654 VMXLOCAL_BREAK_RC(rc);
6655 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6656 pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6657 }
6658 if (fWhat & CPUMCTX_EXTRN_ES)
6659 {
6660 rc = HMVMX_IMPORT_SREG(ES, &pCtx->es);
6661 VMXLOCAL_BREAK_RC(rc);
6662 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6663 pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6664 }
6665 if (fWhat & CPUMCTX_EXTRN_FS)
6666 {
6667 rc = HMVMX_IMPORT_SREG(FS, &pCtx->fs);
6668 VMXLOCAL_BREAK_RC(rc);
6669 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6670 pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6671 }
6672 if (fWhat & CPUMCTX_EXTRN_GS)
6673 {
6674 rc = HMVMX_IMPORT_SREG(GS, &pCtx->gs);
6675 VMXLOCAL_BREAK_RC(rc);
6676 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6677 pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6678 }
6679 }
6680
6681 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
6682 {
6683 if (fWhat & CPUMCTX_EXTRN_LDTR)
6684 {
6685 rc = HMVMX_IMPORT_SREG(LDTR, &pCtx->ldtr);
6686 VMXLOCAL_BREAK_RC(rc);
6687 }
6688
6689 if (fWhat & CPUMCTX_EXTRN_GDTR)
6690 {
6691 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6692 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
6693 VMXLOCAL_BREAK_RC(rc);
6694 pCtx->gdtr.pGdt = u64Val;
6695 pCtx->gdtr.cbGdt = u32Val;
6696 }
6697
6698 /* Guest IDTR. */
6699 if (fWhat & CPUMCTX_EXTRN_IDTR)
6700 {
6701 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6702 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
6703 VMXLOCAL_BREAK_RC(rc);
6704 pCtx->idtr.pIdt = u64Val;
6705 pCtx->idtr.cbIdt = u32Val;
6706 }
6707
6708 /* Guest TR. */
6709 if (fWhat & CPUMCTX_EXTRN_TR)
6710 {
6711 /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
6712 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6713 {
6714 rc = HMVMX_IMPORT_SREG(TR, &pCtx->tr);
6715 VMXLOCAL_BREAK_RC(rc);
6716 }
6717 }
6718 }
6719
6720 if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
6721 {
6722 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
6723 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
6724 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
6725 pCtx->SysEnter.cs = u32Val;
6726 VMXLOCAL_BREAK_RC(rc);
6727 }
6728
6729#if HC_ARCH_BITS == 64
6730 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
6731 {
6732 if ( pVM->hm.s.fAllow64BitGuests
6733 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6734 pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
6735 }
6736
6737 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
6738 {
6739 if ( pVM->hm.s.fAllow64BitGuests
6740 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6741 {
6742 pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
6743 pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
6744 pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
6745 }
6746 }
6747#endif
6748
6749 if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
6750#if HC_ARCH_BITS == 32
6751 || (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS))
6752#endif
6753 )
6754 {
6755 PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6756 uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
6757 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6758 {
6759 switch (pMsr->u32Msr)
6760 {
6761#if HC_ARCH_BITS == 32
6762 case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
6763 case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
6764 case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
6765 case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6766#endif
6767 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6768 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6769 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit */ break;
6770 default:
6771 {
6772 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6773 ASMSetFlags(fEFlags);
6774 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
6775 cMsrs));
6776 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6777 }
6778 }
6779 }
6780 }
6781
6782 if (fWhat & CPUMCTX_EXTRN_DR7)
6783 {
6784 if (!pVCpu->hm.s.fUsingHyperDR7)
6785 {
6786 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6787 rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
6788 VMXLOCAL_BREAK_RC(rc);
6789 pCtx->dr[7] = u32Val;
6790 }
6791 }
6792
6793 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
6794 {
6795 uint32_t u32Shadow;
6796 if (fWhat & CPUMCTX_EXTRN_CR0)
6797 {
6798 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
6799 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
6800 VMXLOCAL_BREAK_RC(rc);
6801 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr0Mask)
6802 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr0Mask);
6803 VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
6804 CPUMSetGuestCR0(pVCpu, u32Val);
6805 VMMRZCallRing3Enable(pVCpu);
6806 }
6807
6808 if (fWhat & CPUMCTX_EXTRN_CR4)
6809 {
6810 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
6811 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
6812 VMXLOCAL_BREAK_RC(rc);
6813 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr4Mask)
6814 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr4Mask);
6815 CPUMSetGuestCR4(pVCpu, u32Val);
6816 }
6817
6818 if (fWhat & CPUMCTX_EXTRN_CR3)
6819 {
6820 /* CR0.PG bit changes are always intercepted, so it's up to date. */
6821 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6822 || ( pVM->hm.s.fNestedPaging
6823 && CPUMIsGuestPagingEnabledEx(pCtx)))
6824 {
6825 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6826 if (pCtx->cr3 != u64Val)
6827 {
6828 CPUMSetGuestCR3(pVCpu, u64Val);
6829 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6830 }
6831
6832 /* If the guest is in PAE mode, sync back the PDPE's into the guest state.
6833 Note: CR4.PAE, CR0.PG, EFER bit changes are always intercepted, so they're up to date. */
6834 if (CPUMIsGuestInPAEModeEx(pCtx))
6835 {
6836 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6837 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6838 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6839 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6840 VMXLOCAL_BREAK_RC(rc);
6841 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6842 }
6843 }
6844 }
6845 }
6846 } while (0);
6847
6848 if (RT_SUCCESS(rc))
6849 {
6850 /* Update fExtrn. */
6851 pCtx->fExtrn &= ~fWhat;
6852
6853 /* If everything has been imported, clear the HM keeper bit. */
6854 if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
6855 {
6856 pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
6857 Assert(!pCtx->fExtrn);
6858 }
6859 }
6860 }
6861 else
6862 AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
6863
6864 ASMSetFlags(fEFlags);
6865
6866 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
6867
6868 /*
6869 * Honor any pending CR3 updates.
6870 *
6871 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6872 * -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6873 * -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
6874 *
6875 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6876 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6877 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6878 * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
6879 *
6880 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6881 */
6882 if (VMMRZCallRing3IsEnabled(pVCpu))
6883 {
6884 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6885 {
6886 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
6887 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6888 }
6889
6890 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6891 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6892
6893 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6894 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6895 }
6896
6897 return VINF_SUCCESS;
6898#undef VMXLOCAL_BREAK_RC
6899}
6900
6901
6902/**
6903 * Saves the guest state from the VMCS into the guest-CPU context.
6904 *
6905 * @returns VBox status code.
6906 * @param pVCpu The cross context virtual CPU structure.
6907 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6908 */
6909VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
6910{
6911 return hmR0VmxImportGuestState(pVCpu, fWhat);
6912}
6913
6914
6915/**
6916 * Check per-VM and per-VCPU force flag actions that require us to go back to
6917 * ring-3 for one reason or another.
6918 *
6919 * @returns Strict VBox status code (i.e. informational status codes too)
6920 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6921 * ring-3.
6922 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6923 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6924 * interrupts)
6925 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6926 * all EMTs to be in ring-3.
6927 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6928 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6929 * to the EM loop.
6930 *
6931 * @param pVCpu The cross context virtual CPU structure.
6932 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6933 */
6934static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
6935{
6936 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6937
6938 /*
6939 * Anything pending? Should be more likely than not if we're doing a good job.
6940 */
6941 PVM pVM = pVCpu->CTX_SUFF(pVM);
6942 if ( !fStepping
6943 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6944 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6945 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6946 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6947 return VINF_SUCCESS;
6948
6949 /* Pending PGM C3 sync. */
6950 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6951 {
6952 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6953 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
6954 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
6955 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6956 if (rcStrict2 != VINF_SUCCESS)
6957 {
6958 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6959 Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6960 return rcStrict2;
6961 }
6962 }
6963
6964 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6965 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6966 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6967 {
6968 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6969 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6970 Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6971 return rc2;
6972 }
6973
6974 /* Pending VM request packets, such as hardware interrupts. */
6975 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6976 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6977 {
6978 Log4Func(("Pending VM request forcing us back to ring-3\n"));
6979 return VINF_EM_PENDING_REQUEST;
6980 }
6981
6982 /* Pending PGM pool flushes. */
6983 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6984 {
6985 Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
6986 return VINF_PGM_POOL_FLUSH_PENDING;
6987 }
6988
6989 /* Pending DMA requests. */
6990 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6991 {
6992 Log4Func(("Pending DMA request forcing us back to ring-3\n"));
6993 return VINF_EM_RAW_TO_R3;
6994 }
6995
6996 return VINF_SUCCESS;
6997}
6998
6999
7000/**
7001 * Converts any TRPM trap into a pending HM event. This is typically used when
7002 * entering from ring-3 (not longjmp returns).
7003 *
7004 * @param pVCpu The cross context virtual CPU structure.
7005 */
7006static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7007{
7008 Assert(TRPMHasTrap(pVCpu));
7009 Assert(!pVCpu->hm.s.Event.fPending);
7010
7011 uint8_t uVector;
7012 TRPMEVENT enmTrpmEvent;
7013 RTGCUINT uErrCode;
7014 RTGCUINTPTR GCPtrFaultAddress;
7015 uint8_t cbInstr;
7016
7017 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7018 AssertRC(rc);
7019
7020 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7021 uint32_t u32IntInfo = uVector | VMX_EXIT_INT_INFO_VALID;
7022 if (enmTrpmEvent == TRPM_TRAP)
7023 {
7024 switch (uVector)
7025 {
7026 case X86_XCPT_NMI:
7027 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7028 break;
7029
7030 case X86_XCPT_BP:
7031 case X86_XCPT_OF:
7032 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7033 break;
7034
7035 case X86_XCPT_PF:
7036 case X86_XCPT_DF:
7037 case X86_XCPT_TS:
7038 case X86_XCPT_NP:
7039 case X86_XCPT_SS:
7040 case X86_XCPT_GP:
7041 case X86_XCPT_AC:
7042 u32IntInfo |= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7043 RT_FALL_THRU();
7044 default:
7045 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7046 break;
7047 }
7048 }
7049 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7050 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7051 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7052 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7053 else
7054 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7055
7056 rc = TRPMResetTrap(pVCpu);
7057 AssertRC(rc);
7058 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7059 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7060
7061 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7062}
7063
7064
7065/**
7066 * Converts the pending HM event into a TRPM trap.
7067 *
7068 * @param pVCpu The cross context virtual CPU structure.
7069 */
7070static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7071{
7072 Assert(pVCpu->hm.s.Event.fPending);
7073
7074 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7075 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7076 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
7077 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7078
7079 /* If a trap was already pending, we did something wrong! */
7080 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7081
7082 TRPMEVENT enmTrapType;
7083 switch (uVectorType)
7084 {
7085 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7086 enmTrapType = TRPM_HARDWARE_INT;
7087 break;
7088
7089 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7090 enmTrapType = TRPM_SOFTWARE_INT;
7091 break;
7092
7093 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7094 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7095 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7096 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7097 enmTrapType = TRPM_TRAP;
7098 break;
7099
7100 default:
7101 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7102 enmTrapType = TRPM_32BIT_HACK;
7103 break;
7104 }
7105
7106 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7107
7108 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7109 AssertRC(rc);
7110
7111 if (fErrorCodeValid)
7112 TRPMSetErrorCode(pVCpu, uErrorCode);
7113
7114 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7115 && uVector == X86_XCPT_PF)
7116 {
7117 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7118 }
7119 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7120 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7121 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7122 {
7123 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7124 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
7125 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7126 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7127 }
7128
7129 /* Clear any pending events from the VMCS. */
7130 VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7131 VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, 0);
7132
7133 /* We're now done converting the pending event. */
7134 pVCpu->hm.s.Event.fPending = false;
7135}
7136
7137
7138/**
7139 * Does the necessary state syncing before returning to ring-3 for any reason
7140 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7141 *
7142 * @returns VBox status code.
7143 * @param pVCpu The cross context virtual CPU structure.
7144 * @param fImportState Whether to import the guest state from the VMCS back
7145 * to the guest-CPU context.
7146 *
7147 * @remarks No-long-jmp zone!!!
7148 */
7149static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
7150{
7151 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7152 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7153
7154 RTCPUID idCpu = RTMpCpuId();
7155 Log4Func(("HostCpuId=%u\n", idCpu));
7156
7157 /*
7158 * !!! IMPORTANT !!!
7159 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7160 */
7161
7162 /* Save the guest state if necessary. */
7163 if (fImportState)
7164 {
7165 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
7166 AssertRCReturn(rc, rc);
7167 }
7168
7169 /* Restore host FPU state if necessary. We will resync on next R0 reentry. */
7170 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7171 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
7172
7173 /* Restore host debug registers if necessary. We will resync on next R0 reentry. */
7174#ifdef VBOX_STRICT
7175 if (CPUMIsHyperDebugStateActive(pVCpu))
7176 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
7177#endif
7178 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7179 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7180 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7181
7182#if HC_ARCH_BITS == 64
7183 /* Restore host-state bits that VT-x only restores partially. */
7184 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7185 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7186 {
7187 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7188 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7189 }
7190 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7191#endif
7192
7193 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7194 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7195 {
7196 /* We shouldn't restore the host MSRs without saving the guest MSRs first. */
7197 if (!fImportState)
7198 {
7199 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS);
7200 AssertRCReturn(rc, rc);
7201 }
7202 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7203 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7204 }
7205 else
7206 pVCpu->hm.s.vmx.fLazyMsrs = 0;
7207
7208 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7209 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7210
7211 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7212 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
7213 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
7214 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
7215 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
7216 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7217 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7218 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7219 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7220
7221 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7222
7223 /** @todo This partially defeats the purpose of having preemption hooks.
7224 * The problem is, deregistering the hooks should be moved to a place that
7225 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7226 * context.
7227 */
7228 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7229 {
7230 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7231 AssertRCReturn(rc, rc);
7232
7233 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7234 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7235 }
7236 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7237 NOREF(idCpu);
7238
7239 return VINF_SUCCESS;
7240}
7241
7242
7243/**
7244 * Leaves the VT-x session.
7245 *
7246 * @returns VBox status code.
7247 * @param pVCpu The cross context virtual CPU structure.
7248 *
7249 * @remarks No-long-jmp zone!!!
7250 */
7251static int hmR0VmxLeaveSession(PVMCPU pVCpu)
7252{
7253 HM_DISABLE_PREEMPT(pVCpu);
7254 HMVMX_ASSERT_CPU_SAFE(pVCpu);
7255 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7256 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7257
7258 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7259 and done this from the VMXR0ThreadCtxCallback(). */
7260 if (!pVCpu->hm.s.fLeaveDone)
7261 {
7262 int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
7263 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7264 pVCpu->hm.s.fLeaveDone = true;
7265 }
7266 Assert(!pVCpu->cpum.GstCtx.fExtrn);
7267
7268 /*
7269 * !!! IMPORTANT !!!
7270 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7271 */
7272
7273 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7274 /** @todo Deregistering here means we need to VMCLEAR always
7275 * (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
7276 * for calling VMMR0ThreadCtxHookDisable here! */
7277 VMMR0ThreadCtxHookDisable(pVCpu);
7278
7279 /* Leave HM context. This takes care of local init (term). */
7280 int rc = HMR0LeaveCpu(pVCpu);
7281
7282 HM_RESTORE_PREEMPT();
7283 return rc;
7284}
7285
7286
7287/**
7288 * Does the necessary state syncing before doing a longjmp to ring-3.
7289 *
7290 * @returns VBox status code.
7291 * @param pVCpu The cross context virtual CPU structure.
7292 *
7293 * @remarks No-long-jmp zone!!!
7294 */
7295DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
7296{
7297 return hmR0VmxLeaveSession(pVCpu);
7298}
7299
7300
7301/**
7302 * Take necessary actions before going back to ring-3.
7303 *
7304 * An action requires us to go back to ring-3. This function does the necessary
7305 * steps before we can safely return to ring-3. This is not the same as longjmps
7306 * to ring-3, this is voluntary and prepares the guest so it may continue
7307 * executing outside HM (recompiler/IEM).
7308 *
7309 * @returns VBox status code.
7310 * @param pVCpu The cross context virtual CPU structure.
7311 * @param rcExit The reason for exiting to ring-3. Can be
7312 * VINF_VMM_UNKNOWN_RING3_CALL.
7313 */
7314static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
7315{
7316 Assert(pVCpu);
7317 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7318
7319 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7320 {
7321 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7322 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7323 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7324 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7325 }
7326
7327 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7328 VMMRZCallRing3Disable(pVCpu);
7329 Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
7330
7331 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7332 if (pVCpu->hm.s.Event.fPending)
7333 {
7334 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7335 Assert(!pVCpu->hm.s.Event.fPending);
7336 }
7337
7338 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7339 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7340
7341 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7342 and if we're injecting an event we should have a TRPM trap pending. */
7343 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7344#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
7345 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7346#endif
7347
7348 /* Save guest state and restore host state bits. */
7349 int rc = hmR0VmxLeaveSession(pVCpu);
7350 AssertRCReturn(rc, rc);
7351 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7352 /* Thread-context hooks are unregistered at this point!!! */
7353
7354 /* Sync recompiler state. */
7355 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7356 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7357 | CPUM_CHANGED_LDTR
7358 | CPUM_CHANGED_GDTR
7359 | CPUM_CHANGED_IDTR
7360 | CPUM_CHANGED_TR
7361 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7362 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
7363 && CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
7364 {
7365 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7366 }
7367
7368 Assert(!pVCpu->hm.s.fClearTrapFlag);
7369
7370 /* Update the exit-to-ring 3 reason. */
7371 pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
7372
7373 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7374 if (rcExit != VINF_EM_RAW_INTERRUPT)
7375 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7376
7377 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7378
7379 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7380 VMMRZCallRing3RemoveNotification(pVCpu);
7381 VMMRZCallRing3Enable(pVCpu);
7382
7383 return rc;
7384}
7385
7386
7387/**
7388 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7389 * longjump to ring-3 and possibly get preempted.
7390 *
7391 * @returns VBox status code.
7392 * @param pVCpu The cross context virtual CPU structure.
7393 * @param enmOperation The operation causing the ring-3 longjump.
7394 * @param pvUser User argument, currently unused, NULL.
7395 */
7396static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7397{
7398 RT_NOREF(pvUser);
7399 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7400 {
7401 /*
7402 * !!! IMPORTANT !!!
7403 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7404 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7405 */
7406 VMMRZCallRing3RemoveNotification(pVCpu);
7407 VMMRZCallRing3Disable(pVCpu);
7408 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7409 RTThreadPreemptDisable(&PreemptState);
7410
7411 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
7412 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7413 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7414
7415#if HC_ARCH_BITS == 64
7416 /* Restore host-state bits that VT-x only restores partially. */
7417 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7418 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7419 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7420 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7421#endif
7422
7423 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7424 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7425 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7426
7427 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7428 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7429 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7430 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7431 {
7432 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7433 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7434 }
7435
7436 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7437 VMMR0ThreadCtxHookDisable(pVCpu);
7438 HMR0LeaveCpu(pVCpu);
7439 RTThreadPreemptRestore(&PreemptState);
7440 return VINF_SUCCESS;
7441 }
7442
7443 Assert(pVCpu);
7444 Assert(pvUser);
7445 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7446 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7447
7448 VMMRZCallRing3Disable(pVCpu);
7449 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7450
7451 Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
7452
7453 int rc = hmR0VmxLongJmpToRing3(pVCpu);
7454 AssertRCReturn(rc, rc);
7455
7456 VMMRZCallRing3Enable(pVCpu);
7457 return VINF_SUCCESS;
7458}
7459
7460
7461/**
7462 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7463 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7464 *
7465 * @param pVCpu The cross context virtual CPU structure.
7466 */
7467DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7468{
7469 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7470 {
7471 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7472 {
7473 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_PROC_CTLS_INT_WINDOW_EXIT;
7474 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7475 AssertRC(rc);
7476 Log4Func(("Setup interrupt-window exiting\n"));
7477 }
7478 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7479}
7480
7481
7482/**
7483 * Clears the interrupt-window exiting control in the VMCS.
7484 *
7485 * @param pVCpu The cross context virtual CPU structure.
7486 */
7487DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7488{
7489 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT);
7490 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
7491 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7492 AssertRC(rc);
7493 Log4Func(("Cleared interrupt-window exiting\n"));
7494}
7495
7496
7497/**
7498 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7499 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7500 *
7501 * @param pVCpu The cross context virtual CPU structure.
7502 */
7503DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7504{
7505 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7506 {
7507 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7508 {
7509 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7510 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7511 AssertRC(rc);
7512 Log4Func(("Setup NMI-window exiting\n"));
7513 }
7514 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7515}
7516
7517
7518/**
7519 * Clears the NMI-window exiting control in the VMCS.
7520 *
7521 * @param pVCpu The cross context virtual CPU structure.
7522 */
7523DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7524{
7525 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT);
7526 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7527 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7528 AssertRC(rc);
7529 Log4Func(("Cleared NMI-window exiting\n"));
7530}
7531
7532
7533/**
7534 * Evaluates the event to be delivered to the guest and sets it as the pending
7535 * event.
7536 *
7537 * @returns The VT-x guest-interruptibility state.
7538 * @param pVCpu The cross context virtual CPU structure.
7539 */
7540static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu)
7541{
7542 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7543 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7544 uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu);
7545 bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7546 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7547 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7548
7549 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7550 Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7551 Assert(!fBlockSti || pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7552 Assert(!TRPMHasTrap(pVCpu));
7553
7554 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7555 APICUpdatePendingInterrupts(pVCpu);
7556
7557 /*
7558 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7559 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7560 */
7561 /** @todo SMI. SMIs take priority over NMIs. */
7562 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7563 {
7564 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7565 if ( !pVCpu->hm.s.Event.fPending
7566 && !fBlockNmi
7567 && !fBlockSti
7568 && !fBlockMovSS)
7569 {
7570 Log4Func(("Pending NMI\n"));
7571 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INT_INFO_VALID;
7572 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7573
7574 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7575 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7576 }
7577 else
7578 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7579 }
7580 /*
7581 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7582 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7583 */
7584 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7585 && !pVCpu->hm.s.fSingleInstruction)
7586 {
7587 Assert(!DBGFIsStepping(pVCpu));
7588 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7589 AssertRCReturn(rc, 0);
7590 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7591 if ( !pVCpu->hm.s.Event.fPending
7592 && !fBlockInt
7593 && !fBlockSti
7594 && !fBlockMovSS)
7595 {
7596 uint8_t u8Interrupt;
7597 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7598 if (RT_SUCCESS(rc))
7599 {
7600 Log4Func(("Pending external interrupt u8Interrupt=%#x\n", u8Interrupt));
7601 uint32_t u32IntInfo = u8Interrupt
7602 | VMX_EXIT_INT_INFO_VALID
7603 | (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7604
7605 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7606 }
7607 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7608 {
7609 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
7610 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7611 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7612
7613 /*
7614 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7615 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7616 * need to re-set this force-flag here.
7617 */
7618 }
7619 else
7620 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7621 }
7622 else
7623 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7624 }
7625
7626 return fIntrState;
7627}
7628
7629
7630/**
7631 * Sets a pending-debug exception to be delivered to the guest if the guest is
7632 * single-stepping in the VMCS.
7633 *
7634 * @param pVCpu The cross context virtual CPU structure.
7635 */
7636DECLINLINE(int) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7637{
7638 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7639 RT_NOREF(pVCpu);
7640 return VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
7641}
7642
7643
7644/**
7645 * Injects any pending events into the guest if the guest is in a state to
7646 * receive them.
7647 *
7648 * @returns Strict VBox status code (i.e. informational status codes too).
7649 * @param pVCpu The cross context virtual CPU structure.
7650 * @param fIntrState The VT-x guest-interruptibility state.
7651 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7652 * return VINF_EM_DBG_STEPPED if the event was
7653 * dispatched directly.
7654 */
7655static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, uint32_t fIntrState, bool fStepping)
7656{
7657 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7658 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7659
7660 bool fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7661 bool fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7662
7663 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7664 Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7665 Assert(!fBlockSti || pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7666 Assert(!TRPMHasTrap(pVCpu));
7667
7668 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7669 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7670 if (pVCpu->hm.s.Event.fPending)
7671 {
7672 /*
7673 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7674 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7675 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7676 *
7677 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7678 */
7679 uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7680#ifdef VBOX_STRICT
7681 if (uIntType == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
7682 {
7683 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7684 Assert(!fBlockInt);
7685 Assert(!fBlockSti);
7686 Assert(!fBlockMovSS);
7687 }
7688 else if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
7689 {
7690 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7691 Assert(!fBlockSti);
7692 Assert(!fBlockMovSS);
7693 Assert(!fBlockNmi);
7694 }
7695#endif
7696 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7697 uIntType));
7698 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7699 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping,
7700 &fIntrState);
7701 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7702
7703 /* Update the interruptibility-state as it could have been changed by
7704 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7705 fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7706 fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7707
7708 if (uIntType == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
7709 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7710 else
7711 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7712 }
7713
7714 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7715 if ( fBlockSti
7716 || fBlockMovSS)
7717 {
7718 if (!pVCpu->hm.s.fSingleInstruction)
7719 {
7720 /*
7721 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7722 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7723 * See Intel spec. 27.3.4 "Saving Non-Register State".
7724 */
7725 Assert(!DBGFIsStepping(pVCpu));
7726 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7727 AssertRCReturn(rc, rc);
7728 if (pCtx->eflags.Bits.u1TF)
7729 {
7730 int rc2 = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7731 AssertRCReturn(rc2, rc2);
7732 }
7733 }
7734 else if (pCtx->eflags.Bits.u1TF)
7735 {
7736 /*
7737 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7738 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7739 */
7740 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
7741 fIntrState = 0;
7742 }
7743 }
7744
7745 /*
7746 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7747 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7748 */
7749 int rc3 = hmR0VmxExportGuestIntrState(pVCpu, fIntrState);
7750 AssertRCReturn(rc3, rc3);
7751
7752 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7753 NOREF(fBlockMovSS); NOREF(fBlockSti);
7754 return rcStrict;
7755}
7756
7757
7758/**
7759 * Injects a double-fault (\#DF) exception into the VM.
7760 *
7761 * @returns Strict VBox status code (i.e. informational status codes too).
7762 * @param pVCpu The cross context virtual CPU structure.
7763 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7764 * and should return VINF_EM_DBG_STEPPED if the event
7765 * is injected directly (register modified by us, not
7766 * by hardware on VM-entry).
7767 * @param pfIntrState Pointer to the current guest interruptibility-state.
7768 * This interruptibility-state will be updated if
7769 * necessary. This cannot not be NULL.
7770 */
7771DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, bool fStepping, uint32_t *pfIntrState)
7772{
7773 uint32_t const u32IntInfo = X86_XCPT_DF | VMX_EXIT_INT_INFO_VALID
7774 | (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT)
7775 | VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7776 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */, fStepping,
7777 pfIntrState);
7778}
7779
7780
7781/**
7782 * Injects a general-protection (\#GP) fault into the VM.
7783 *
7784 * @returns Strict VBox status code (i.e. informational status codes too).
7785 * @param pVCpu The cross context virtual CPU structure.
7786 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7787 * mode, i.e. in real-mode it's not valid).
7788 * @param u32ErrorCode The error code associated with the \#GP.
7789 * @param fStepping Whether we're running in
7790 * hmR0VmxRunGuestCodeStep() and should return
7791 * VINF_EM_DBG_STEPPED if the event is injected
7792 * directly (register modified by us, not by
7793 * hardware on VM-entry).
7794 * @param pfIntrState Pointer to the current guest interruptibility-state.
7795 * This interruptibility-state will be updated if
7796 * necessary. This cannot not be NULL.
7797 */
7798DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, bool fErrorCodeValid, uint32_t u32ErrorCode, bool fStepping,
7799 uint32_t *pfIntrState)
7800{
7801 uint32_t const u32IntInfo = X86_XCPT_GP | VMX_EXIT_INT_INFO_VALID
7802 | (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT)
7803 | (fErrorCodeValid ? VMX_EXIT_INT_INFO_ERROR_CODE_VALID : 0);
7804 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */, fStepping,
7805 pfIntrState);
7806}
7807
7808
7809/**
7810 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7811 * stack.
7812 *
7813 * @returns Strict VBox status code (i.e. informational status codes too).
7814 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7815 * @param pVCpu The cross context virtual CPU structure.
7816 * @param uValue The value to push to the guest stack.
7817 */
7818static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
7819{
7820 /*
7821 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7822 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7823 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7824 */
7825 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7826 if (pCtx->sp == 1)
7827 return VINF_EM_RESET;
7828 pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7829 int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
7830 AssertRC(rc);
7831 return rc;
7832}
7833
7834
7835/**
7836 * Injects an event into the guest upon VM-entry by updating the relevant fields
7837 * in the VM-entry area in the VMCS.
7838 *
7839 * @returns Strict VBox status code (i.e. informational status codes too).
7840 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7841 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7842 *
7843 * @param pVCpu The cross context virtual CPU structure.
7844 * @param u64IntInfo The VM-entry interruption-information field.
7845 * @param cbInstr The VM-entry instruction length in bytes (for
7846 * software interrupts, exceptions and privileged
7847 * software exceptions).
7848 * @param u32ErrCode The VM-entry exception error code.
7849 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7850 * @param pfIntrState Pointer to the current guest interruptibility-state.
7851 * This interruptibility-state will be updated if
7852 * necessary. This cannot not be NULL.
7853 * @param fStepping Whether we're running in
7854 * hmR0VmxRunGuestCodeStep() and should return
7855 * VINF_EM_DBG_STEPPED if the event is injected
7856 * directly (register modified by us, not by
7857 * hardware on VM-entry).
7858 */
7859static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7860 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState)
7861{
7862 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7863 AssertMsg(!RT_HI_U32(u64IntInfo), ("%#RX64\n", u64IntInfo));
7864 Assert(pfIntrState);
7865
7866 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7867 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7868 uint32_t const uVector = VMX_EXIT_INT_INFO_VECTOR(u32IntInfo);
7869 uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(u32IntInfo);
7870
7871#ifdef VBOX_STRICT
7872 /*
7873 * Validate the error-code-valid bit for hardware exceptions.
7874 * No error codes for exceptions in real-mode.
7875 *
7876 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7877 */
7878 if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
7879 && !CPUMIsGuestInRealModeEx(pCtx))
7880 {
7881 switch (uVector)
7882 {
7883 case X86_XCPT_PF:
7884 case X86_XCPT_DF:
7885 case X86_XCPT_TS:
7886 case X86_XCPT_NP:
7887 case X86_XCPT_SS:
7888 case X86_XCPT_GP:
7889 case X86_XCPT_AC:
7890 AssertMsg(VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
7891 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7892 RT_FALL_THRU();
7893 default:
7894 break;
7895 }
7896 }
7897#endif
7898
7899 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7900 Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
7901 || !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
7902
7903 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7904
7905 /*
7906 * Hardware interrupts & exceptions cannot be delivered through the software interrupt
7907 * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
7908 * interrupt handler in the (real-mode) guest.
7909 *
7910 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
7911 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7912 */
7913 if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
7914 {
7915 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
7916 {
7917 /*
7918 * For unrestricted execution enabled CPUs running real-mode guests, we must not
7919 * set the deliver-error-code bit.
7920 *
7921 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7922 */
7923 u32IntInfo &= ~VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7924 }
7925 else
7926 {
7927 PVM pVM = pVCpu->CTX_SUFF(pVM);
7928 Assert(PDMVmmDevHeapIsEnabled(pVM));
7929 Assert(pVM->hm.s.vmx.pRealModeTSS);
7930
7931 /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
7932 int rc2 = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_TABLE_MASK | CPUMCTX_EXTRN_RIP
7933 | CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_RFLAGS);
7934 AssertRCReturn(rc2, rc2);
7935
7936 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7937 size_t const cbIdtEntry = sizeof(X86IDTR16);
7938 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
7939 {
7940 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7941 if (uVector == X86_XCPT_DF)
7942 return VINF_EM_RESET;
7943
7944 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7945 if (uVector == X86_XCPT_GP)
7946 return hmR0VmxInjectXcptDF(pVCpu, fStepping, pfIntrState);
7947
7948 /*
7949 * If we're injecting an event with no valid IDT entry, inject a #GP.
7950 * No error codes for exceptions in real-mode.
7951 *
7952 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7953 */
7954 return hmR0VmxInjectXcptGP(pVCpu, false /* fErrCodeValid */, 0 /* u32ErrCode */, fStepping, pfIntrState);
7955 }
7956
7957 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7958 uint16_t uGuestIp = pCtx->ip;
7959 if (uIntType == VMX_EXIT_INT_INFO_TYPE_SW_XCPT)
7960 {
7961 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7962 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7963 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7964 }
7965 else if (uIntType == VMX_EXIT_INT_INFO_TYPE_SW_INT)
7966 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7967
7968 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7969 X86IDTR16 IdtEntry;
7970 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
7971 rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7972 AssertRCReturn(rc2, rc2);
7973
7974 /* Construct the stack frame for the interrupt/exception handler. */
7975 VBOXSTRICTRC rcStrict;
7976 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
7977 if (rcStrict == VINF_SUCCESS)
7978 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
7979 if (rcStrict == VINF_SUCCESS)
7980 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
7981
7982 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7983 if (rcStrict == VINF_SUCCESS)
7984 {
7985 pCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7986 pCtx->rip = IdtEntry.offSel;
7987 pCtx->cs.Sel = IdtEntry.uSel;
7988 pCtx->cs.ValidSel = IdtEntry.uSel;
7989 pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7990 if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
7991 && uVector == X86_XCPT_PF)
7992 pCtx->cr2 = GCPtrFaultAddress;
7993
7994 /* If any other guest-state bits are changed here, make sure to update
7995 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7996 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS | HM_CHANGED_GUEST_CR2
7997 | HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
7998 | HM_CHANGED_GUEST_RSP);
7999
8000 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8001 if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
8002 {
8003 Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
8004 && uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
8005 Log4Func(("Clearing inhibition due to STI\n"));
8006 *pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
8007 }
8008 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8009 u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
8010
8011 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8012 it, if we are returning to ring-3 before executing guest code. */
8013 pVCpu->hm.s.Event.fPending = false;
8014
8015 /* Make hmR0VmxPreRunGuest() return if we're stepping since we've changed cs:rip. */
8016 if (fStepping)
8017 rcStrict = VINF_EM_DBG_STEPPED;
8018 }
8019 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8020 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8021 return rcStrict;
8022 }
8023 }
8024
8025 /* Validate. */
8026 Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8027 Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
8028
8029 /* Inject. */
8030 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8031 if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8032 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8033 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8034 AssertRCReturn(rc, rc);
8035
8036 /* Update CR2. */
8037 if ( VMX_EXIT_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8038 && uVector == X86_XCPT_PF)
8039 pCtx->cr2 = GCPtrFaultAddress;
8040
8041 Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8042
8043 return VINF_SUCCESS;
8044}
8045
8046
8047/**
8048 * Clears the interrupt-window exiting control in the VMCS and if necessary
8049 * clears the current event in the VMCS as well.
8050 *
8051 * @returns VBox status code.
8052 * @param pVCpu The cross context virtual CPU structure.
8053 *
8054 * @remarks Use this function only to clear events that have not yet been
8055 * delivered to the guest but are injected in the VMCS!
8056 * @remarks No-long-jump zone!!!
8057 */
8058static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8059{
8060 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
8061 {
8062 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8063 Log4Func(("Cleared interrupt widow\n"));
8064 }
8065
8066 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
8067 {
8068 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8069 Log4Func(("Cleared interrupt widow\n"));
8070 }
8071}
8072
8073
8074/**
8075 * Enters the VT-x session.
8076 *
8077 * @returns VBox status code.
8078 * @param pVCpu The cross context virtual CPU structure.
8079 * @param pHostCpu Pointer to the global CPU info struct.
8080 */
8081VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu, PHMGLOBALCPUINFO pHostCpu)
8082{
8083 AssertPtr(pVCpu);
8084 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8085 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8086 RT_NOREF(pHostCpu);
8087
8088 LogFlowFunc(("pVCpu=%p\n", pVCpu));
8089 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8090 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8091
8092#ifdef VBOX_STRICT
8093 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8094 RTCCUINTREG uHostCR4 = ASMGetCR4();
8095 if (!(uHostCR4 & X86_CR4_VMXE))
8096 {
8097 LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
8098 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8099 }
8100#endif
8101
8102 /*
8103 * Load the VCPU's VMCS as the current (and active) one.
8104 */
8105 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8106 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8107 if (RT_FAILURE(rc))
8108 return rc;
8109
8110 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8111 pVCpu->hm.s.fLeaveDone = false;
8112 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8113
8114 return VINF_SUCCESS;
8115}
8116
8117
8118/**
8119 * The thread-context callback (only on platforms which support it).
8120 *
8121 * @param enmEvent The thread-context event.
8122 * @param pVCpu The cross context virtual CPU structure.
8123 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8124 * @thread EMT(pVCpu)
8125 */
8126VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8127{
8128 NOREF(fGlobalInit);
8129
8130 switch (enmEvent)
8131 {
8132 case RTTHREADCTXEVENT_OUT:
8133 {
8134 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8135 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8136 VMCPU_ASSERT_EMT(pVCpu);
8137
8138 /* No longjmps (logger flushes, locks) in this fragile context. */
8139 VMMRZCallRing3Disable(pVCpu);
8140 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8141
8142 /*
8143 * Restore host-state (FPU, debug etc.)
8144 */
8145 if (!pVCpu->hm.s.fLeaveDone)
8146 {
8147 /*
8148 * Do -not- import the guest-state here as we might already be in the middle of importing
8149 * it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
8150 */
8151 hmR0VmxLeave(pVCpu, false /* fImportState */);
8152 pVCpu->hm.s.fLeaveDone = true;
8153 }
8154
8155 /* Leave HM context, takes care of local init (term). */
8156 int rc = HMR0LeaveCpu(pVCpu);
8157 AssertRC(rc); NOREF(rc);
8158
8159 /* Restore longjmp state. */
8160 VMMRZCallRing3Enable(pVCpu);
8161 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8162 break;
8163 }
8164
8165 case RTTHREADCTXEVENT_IN:
8166 {
8167 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8168 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8169 VMCPU_ASSERT_EMT(pVCpu);
8170
8171 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8172 VMMRZCallRing3Disable(pVCpu);
8173 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8174
8175 /* Initialize the bare minimum state required for HM. This takes care of
8176 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8177 int rc = hmR0EnterCpu(pVCpu);
8178 AssertRC(rc);
8179 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8180 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8181
8182 /* Load the active VMCS as the current one. */
8183 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8184 {
8185 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8186 AssertRC(rc); NOREF(rc);
8187 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8188 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8189 }
8190 pVCpu->hm.s.fLeaveDone = false;
8191
8192 /* Restore longjmp state. */
8193 VMMRZCallRing3Enable(pVCpu);
8194 break;
8195 }
8196
8197 default:
8198 break;
8199 }
8200}
8201
8202
8203/**
8204 * Exports the host state into the VMCS host-state area.
8205 * Sets up the VM-exit MSR-load area.
8206 *
8207 * The CPU state will be loaded from these fields on every successful VM-exit.
8208 *
8209 * @returns VBox status code.
8210 * @param pVCpu The cross context virtual CPU structure.
8211 *
8212 * @remarks No-long-jump zone!!!
8213 */
8214static int hmR0VmxExportHostState(PVMCPU pVCpu)
8215{
8216 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8217
8218 int rc = VINF_SUCCESS;
8219 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8220 {
8221 rc = hmR0VmxExportHostControlRegs();
8222 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8223
8224 rc = hmR0VmxExportHostSegmentRegs(pVCpu);
8225 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8226
8227 rc = hmR0VmxExportHostMsrs(pVCpu);
8228 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8229
8230 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
8231 }
8232 return rc;
8233}
8234
8235
8236/**
8237 * Saves the host state in the VMCS host-state.
8238 *
8239 * @returns VBox status code.
8240 * @param pVCpu The cross context virtual CPU structure.
8241 *
8242 * @remarks No-long-jump zone!!!
8243 */
8244VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
8245{
8246 AssertPtr(pVCpu);
8247 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8248
8249 /*
8250 * Export the host state here while entering HM context.
8251 * When thread-context hooks are used, we might get preempted and have to re-save the host
8252 * state but most of the time we won't be, so do it here before we disable interrupts.
8253 */
8254 return hmR0VmxExportHostState(pVCpu);
8255}
8256
8257
8258/**
8259 * Exports the guest state into the VMCS guest-state area.
8260 *
8261 * The will typically be done before VM-entry when the guest-CPU state and the
8262 * VMCS state may potentially be out of sync.
8263 *
8264 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8265 * VM-entry controls.
8266 * Sets up the appropriate VMX non-root function to execute guest code based on
8267 * the guest CPU mode.
8268 *
8269 * @returns VBox strict status code.
8270 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8271 * without unrestricted guest access and the VMMDev is not presently
8272 * mapped (e.g. EFI32).
8273 *
8274 * @param pVCpu The cross context virtual CPU structure.
8275 *
8276 * @remarks No-long-jump zone!!!
8277 */
8278static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu)
8279{
8280 AssertPtr(pVCpu);
8281 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8282
8283 LogFlowFunc(("pVCpu=%p\n", pVCpu));
8284
8285 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
8286
8287 /* Determine real-on-v86 mode. */
8288 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8289 if ( !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
8290 && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
8291 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8292
8293 /*
8294 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8295 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8296 */
8297 int rc = hmR0VmxSelectVMRunHandler(pVCpu);
8298 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8299
8300 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8301 rc = hmR0VmxExportGuestEntryCtls(pVCpu);
8302 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8303
8304 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8305 rc = hmR0VmxExportGuestExitCtls(pVCpu);
8306 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8307
8308 rc = hmR0VmxExportGuestCR0(pVCpu);
8309 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8310
8311 VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu);
8312 if (rcStrict == VINF_SUCCESS)
8313 { /* likely */ }
8314 else
8315 {
8316 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8317 return rcStrict;
8318 }
8319
8320 rc = hmR0VmxExportGuestSegmentRegs(pVCpu);
8321 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8322
8323 /* This needs to be done after hmR0VmxExportGuestEntryCtls() and hmR0VmxExportGuestExitCtls() as it
8324 may alter controls if we determine we don't have to swap EFER after all. */
8325 rc = hmR0VmxExportGuestMsrs(pVCpu);
8326 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8327
8328 rc = hmR0VmxExportGuestApicTpr(pVCpu);
8329 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8330
8331 rc = hmR0VmxExportGuestXcptIntercepts(pVCpu);
8332 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8333
8334 /* Exporting RFLAGS here is fine, even though RFLAGS.TF might depend on guest debug state which is
8335 not exported here. It is re-evaluated and updated if necessary in hmR0VmxExportSharedState(). */
8336 rc = hmR0VmxExportGuestRip(pVCpu);
8337 rc |= hmR0VmxExportGuestRsp(pVCpu);
8338 rc |= hmR0VmxExportGuestRflags(pVCpu);
8339 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8340
8341 /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
8342 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
8343 | HM_CHANGED_GUEST_CR2
8344 | (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
8345 | HM_CHANGED_GUEST_X87
8346 | HM_CHANGED_GUEST_SSE_AVX
8347 | HM_CHANGED_GUEST_OTHER_XSAVE
8348 | HM_CHANGED_GUEST_XCRx
8349 | HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
8350 | HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
8351 | HM_CHANGED_GUEST_TSC_AUX
8352 | HM_CHANGED_GUEST_OTHER_MSRS
8353 | HM_CHANGED_GUEST_HWVIRT
8354 | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
8355
8356 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
8357 return rc;
8358}
8359
8360
8361/**
8362 * Exports the state shared between the host and guest into the VMCS.
8363 *
8364 * @param pVCpu The cross context virtual CPU structure.
8365 *
8366 * @remarks No-long-jump zone!!!
8367 */
8368static void hmR0VmxExportSharedState(PVMCPU pVCpu)
8369{
8370 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8371 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8372
8373 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
8374 {
8375 int rc = hmR0VmxExportSharedDebugState(pVCpu);
8376 AssertRC(rc);
8377 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
8378
8379 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8380 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
8381 {
8382 rc = hmR0VmxExportGuestRflags(pVCpu);
8383 AssertRC(rc);
8384 }
8385 }
8386
8387 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
8388 {
8389 hmR0VmxLazyLoadGuestMsrs(pVCpu);
8390 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
8391 }
8392
8393 AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
8394 ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8395}
8396
8397
8398/**
8399 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8400 *
8401 * @returns Strict VBox status code (i.e. informational status codes too).
8402 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8403 * without unrestricted guest access and the VMMDev is not presently
8404 * mapped (e.g. EFI32).
8405 *
8406 * @param pVCpu The cross context virtual CPU structure.
8407 *
8408 * @remarks No-long-jump zone!!!
8409 */
8410static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu)
8411{
8412 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8413 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8414 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8415
8416#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8417 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8418#endif
8419
8420 /*
8421 * For many exits it's only RIP that changes and hence try to export it first
8422 * without going through a lot of change flag checks.
8423 */
8424 VBOXSTRICTRC rcStrict;
8425 uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8426 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8427 if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
8428 {
8429 rcStrict = hmR0VmxExportGuestRip(pVCpu);
8430 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8431 { /* likely */}
8432 else
8433 AssertMsgFailedReturn(("hmR0VmxExportGuestRip failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8434 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
8435 }
8436 else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8437 {
8438 rcStrict = hmR0VmxExportGuestState(pVCpu);
8439 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8440 { /* likely */}
8441 else
8442 {
8443 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("hmR0VmxExportGuestState failed! rc=%Rrc\n",
8444 VBOXSTRICTRC_VAL(rcStrict)));
8445 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8446 return rcStrict;
8447 }
8448 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
8449 }
8450 else
8451 rcStrict = VINF_SUCCESS;
8452
8453#ifdef VBOX_STRICT
8454 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8455 fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8456 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8457 AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
8458 ("fCtxChanged=%#RX64\n", fCtxChanged));
8459#endif
8460 return rcStrict;
8461}
8462
8463
8464/**
8465 * Does the preparations before executing guest code in VT-x.
8466 *
8467 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8468 * recompiler/IEM. We must be cautious what we do here regarding committing
8469 * guest-state information into the VMCS assuming we assuredly execute the
8470 * guest in VT-x mode.
8471 *
8472 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8473 * the common-state (TRPM/forceflags), we must undo those changes so that the
8474 * recompiler/IEM can (and should) use them when it resumes guest execution.
8475 * Otherwise such operations must be done when we can no longer exit to ring-3.
8476 *
8477 * @returns Strict VBox status code (i.e. informational status codes too).
8478 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8479 * have been disabled.
8480 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8481 * double-fault into the guest.
8482 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8483 * dispatched directly.
8484 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8485 *
8486 * @param pVCpu The cross context virtual CPU structure.
8487 * @param pVmxTransient Pointer to the VMX transient structure.
8488 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8489 * us ignore some of the reasons for returning to
8490 * ring-3, and return VINF_EM_DBG_STEPPED if event
8491 * dispatching took place.
8492 */
8493static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
8494{
8495 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8496
8497#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_ONLY_IN_IEM
8498 Log2(("hmR0SvmPreRunGuest: Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
8499 return VINF_EM_RESCHEDULE_REM;
8500#endif
8501
8502#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8503 PGMRZDynMapFlushAutoSet(pVCpu);
8504#endif
8505
8506 /* Check force flag actions that might require us to go back to ring-3. */
8507 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
8508 if (rcStrict == VINF_SUCCESS)
8509 { /* FFs doesn't get set all the time. */ }
8510 else
8511 return rcStrict;
8512
8513 /*
8514 * Setup the virtualized-APIC accesses.
8515 *
8516 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8517 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8518 *
8519 * This is the reason we do it here and not in hmR0VmxExportGuestState().
8520 */
8521 PVM pVM = pVCpu->CTX_SUFF(pVM);
8522 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8523 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
8524 && PDMHasApic(pVM))
8525 {
8526 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8527 Assert(u64MsrApicBase);
8528 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8529
8530 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8531
8532 /* Unalias any existing mapping. */
8533 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8534 AssertRCReturn(rc, rc);
8535
8536 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8537 Log4Func(("Mapped HC APIC-access page at %#RGp\n", GCPhysApicBase));
8538 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8539 AssertRCReturn(rc, rc);
8540
8541 /* Update the per-VCPU cache of the APIC base MSR. */
8542 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8543 }
8544
8545 if (TRPMHasTrap(pVCpu))
8546 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8547 uint32_t fIntrState = hmR0VmxEvaluatePendingEvent(pVCpu);
8548
8549 /*
8550 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
8551 * needs to be done with longjmps or interrupts + preemption enabled. Event injection might
8552 * also result in triple-faulting the VM.
8553 */
8554 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, fIntrState, fStepping);
8555 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8556 { /* likely */ }
8557 else
8558 {
8559 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8560 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8561 return rcStrict;
8562 }
8563
8564 /*
8565 * A longjump might result in importing CR3 even for VM-exits that don't necessarily
8566 * import CR3 themselves. We will need to update them here, as even as late as the above
8567 * hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
8568 * the below force flags to be set.
8569 */
8570 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8571 {
8572 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
8573 int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8574 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
8575 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
8576 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8577 }
8578 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8579 {
8580 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8581 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8582 }
8583
8584 /*
8585 * No longjmps to ring-3 from this point on!!!
8586 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8587 * This also disables flushing of the R0-logger instance (if any).
8588 */
8589 VMMRZCallRing3Disable(pVCpu);
8590
8591 /*
8592 * Export the guest state bits.
8593 *
8594 * We cannot perform longjmps while loading the guest state because we do not preserve the
8595 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8596 * CPU migration.
8597 *
8598 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8599 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8600 * Hence, loading of the guest state needs to be done -after- injection of events.
8601 */
8602 rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu);
8603 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8604 { /* likely */ }
8605 else
8606 {
8607 VMMRZCallRing3Enable(pVCpu);
8608 return rcStrict;
8609 }
8610
8611 /*
8612 * We disable interrupts so that we don't miss any interrupts that would flag preemption
8613 * (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
8614 * preemption disabled for a while. Since this is purly to aid the
8615 * RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
8616 * disable interrupt on NT.
8617 *
8618 * We need to check for force-flags that could've possible been altered since we last
8619 * checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
8620 * see @bugref{6398}).
8621 *
8622 * We also check a couple of other force-flags as a last opportunity to get the EMT back
8623 * to ring-3 before executing guest code.
8624 */
8625 pVmxTransient->fEFlags = ASMIntDisableFlags();
8626
8627 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8628 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8629 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8630 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8631 {
8632 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8633 {
8634 pVCpu->hm.s.Event.fPending = false;
8635
8636 /*
8637 * We've injected any pending events. This is really the point of no return (to ring-3).
8638 *
8639 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
8640 * returns from this function, so don't enable them here.
8641 */
8642 return VINF_SUCCESS;
8643 }
8644
8645 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
8646 rcStrict = VINF_EM_RAW_INTERRUPT;
8647 }
8648 else
8649 {
8650 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8651 rcStrict = VINF_EM_RAW_TO_R3;
8652 }
8653
8654 ASMSetFlags(pVmxTransient->fEFlags);
8655 VMMRZCallRing3Enable(pVCpu);
8656
8657 return rcStrict;
8658}
8659
8660
8661/**
8662 * Prepares to run guest code in VT-x and we've committed to doing so. This
8663 * means there is no backing out to ring-3 or anywhere else at this
8664 * point.
8665 *
8666 * @param pVCpu The cross context virtual CPU structure.
8667 * @param pVmxTransient Pointer to the VMX transient structure.
8668 *
8669 * @remarks Called with preemption disabled.
8670 * @remarks No-long-jump zone!!!
8671 */
8672static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
8673{
8674 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8675 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8676 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8677
8678 /*
8679 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8680 */
8681 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8682 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8683
8684 PVM pVM = pVCpu->CTX_SUFF(pVM);
8685 if (!CPUMIsGuestFPUStateActive(pVCpu))
8686 {
8687 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8688 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8689 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT;
8690 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8691 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
8692 }
8693
8694 /*
8695 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8696 */
8697 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8698 && pVCpu->hm.s.vmx.cMsrs > 0)
8699 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8700
8701 /*
8702 * Re-save the host state bits as we may've been preempted (only happens when
8703 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8704 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8705 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8706 * See @bugref{8432}.
8707 */
8708 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8709 {
8710 int rc = hmR0VmxExportHostState(pVCpu);
8711 AssertRC(rc);
8712 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
8713 }
8714 Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
8715
8716 /*
8717 * Export the state shared between host and guest (FPU, debug, lazy MSRs).
8718 */
8719 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
8720 hmR0VmxExportSharedState(pVCpu);
8721 AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8722
8723 /* Store status of the shared guest-host state at the time of VM-entry. */
8724#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8725 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
8726 {
8727 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8728 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8729 }
8730 else
8731#endif
8732 {
8733 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8734 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8735 }
8736
8737 /*
8738 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8739 */
8740 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
8741 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8742
8743 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8744 RTCPUID idCurrentCpu = pCpu->idCpu;
8745 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8746 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8747 {
8748 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8749 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8750 }
8751
8752 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8753 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8754 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8755 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8756
8757 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8758
8759 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8760 to start executing. */
8761
8762 /*
8763 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8764 */
8765 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
8766 {
8767 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
8768 {
8769 bool fMsrUpdated;
8770 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
8771 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8772 &fMsrUpdated);
8773 AssertRC(rc2);
8774 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8775 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8776 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8777 }
8778 else
8779 {
8780 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8781 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8782 }
8783 }
8784
8785 if (pVM->cpum.ro.GuestFeatures.fIbrs)
8786 {
8787 bool fMsrUpdated;
8788 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_OTHER_MSRS);
8789 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
8790 &fMsrUpdated);
8791 AssertRC(rc2);
8792 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8793 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8794 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8795 }
8796
8797#ifdef VBOX_STRICT
8798 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8799 hmR0VmxCheckHostEferMsr(pVCpu);
8800 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8801#endif
8802#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8803 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
8804 {
8805 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
8806 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8807 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8808 }
8809#endif
8810}
8811
8812
8813/**
8814 * Performs some essential restoration of state after running guest code in
8815 * VT-x.
8816 *
8817 * @param pVCpu The cross context virtual CPU structure.
8818 * @param pVmxTransient Pointer to the VMX transient structure.
8819 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8820 *
8821 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8822 *
8823 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8824 * unconditionally when it is safe to do so.
8825 */
8826static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8827{
8828 uint64_t const uHostTsc = ASMReadTSC();
8829 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8830
8831 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8832 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8833 pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8834 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8835 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8836 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8837
8838 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
8839 TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TscOffset);
8840
8841 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
8842 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8843 Assert(!ASMIntAreEnabled());
8844 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8845
8846#if HC_ARCH_BITS == 64
8847 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8848#endif
8849#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8850 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8851 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8852 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8853#else
8854 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8855#endif
8856#ifdef VBOX_STRICT
8857 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8858#endif
8859 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8860
8861 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8862 uint32_t uExitReason;
8863 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8864 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8865 AssertRC(rc);
8866 pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
8867 pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
8868
8869 if (rcVMRun == VINF_SUCCESS)
8870 {
8871 /*
8872 * Update the VM-exit history array here even if the VM-entry failed due to:
8873 * - Invalid guest state.
8874 * - MSR loading.
8875 * - Machine-check event.
8876 *
8877 * In any of the above cases we will still have a "valid" VM-exit reason
8878 * despite @a fVMEntryFailed being false.
8879 *
8880 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8881 *
8882 * Note! We don't have CS or RIP at this point. Will probably address that later
8883 * by amending the history entry added here.
8884 */
8885 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
8886 UINT64_MAX, uHostTsc);
8887
8888 if (!pVmxTransient->fVMEntryFailed)
8889 {
8890 VMMRZCallRing3Enable(pVCpu);
8891
8892 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8893 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8894
8895#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8896 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
8897 AssertRC(rc);
8898#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8899 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_RFLAGS);
8900 AssertRC(rc);
8901#else
8902 /*
8903 * Import the guest-interruptibility state always as we need it while evaluating
8904 * injecting events on re-entry.
8905 *
8906 * We don't import CR0 (when Unrestricted guest execution is unavailable) despite
8907 * checking for real-mode while exporting the state because all bits that cause
8908 * mode changes wrt CR0 are intercepted.
8909 */
8910 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
8911 AssertRC(rc);
8912#endif
8913
8914 /*
8915 * Sync the TPR shadow with our APIC state.
8916 */
8917 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
8918 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8919 {
8920 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8921 AssertRC(rc);
8922 ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8923 }
8924
8925 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8926 return;
8927 }
8928 }
8929 else
8930 Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
8931
8932 VMMRZCallRing3Enable(pVCpu);
8933}
8934
8935
8936/**
8937 * Runs the guest code using VT-x the normal way.
8938 *
8939 * @returns VBox status code.
8940 * @param pVCpu The cross context virtual CPU structure.
8941 *
8942 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8943 */
8944static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu)
8945{
8946 VMXTRANSIENT VmxTransient;
8947 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8948 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8949 uint32_t cLoops = 0;
8950
8951 for (;; cLoops++)
8952 {
8953 Assert(!HMR0SuspendPending());
8954 HMVMX_ASSERT_CPU_SAFE(pVCpu);
8955
8956 /* Preparatory work for running guest code, this may force us to return
8957 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8958 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8959 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
8960 if (rcStrict != VINF_SUCCESS)
8961 break;
8962
8963 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
8964 int rcRun = hmR0VmxRunGuest(pVCpu);
8965
8966 /* Restore any residual host-state and save any bits shared between host
8967 and guest into the guest-CPU state. Re-enables interrupts! */
8968 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
8969
8970 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8971 if (RT_SUCCESS(rcRun))
8972 { /* very likely */ }
8973 else
8974 {
8975 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
8976 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
8977 return rcRun;
8978 }
8979
8980 /* Profile the VM-exit. */
8981 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8982 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8983 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8984 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
8985 HMVMX_START_EXIT_DISPATCH_PROF();
8986
8987 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
8988
8989 /* Handle the VM-exit. */
8990#ifdef HMVMX_USE_FUNCTION_TABLE
8991 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
8992#else
8993 rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient, VmxTransient.uExitReason);
8994#endif
8995 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
8996 if (rcStrict == VINF_SUCCESS)
8997 {
8998 if (cLoops <= pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
8999 continue; /* likely */
9000 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9001 rcStrict = VINF_EM_RAW_INTERRUPT;
9002 }
9003 break;
9004 }
9005
9006 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9007 return rcStrict;
9008}
9009
9010
9011
9012/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
9013 * probes.
9014 *
9015 * The following few functions and associated structure contains the bloat
9016 * necessary for providing detailed debug events and dtrace probes as well as
9017 * reliable host side single stepping. This works on the principle of
9018 * "subclassing" the normal execution loop and workers. We replace the loop
9019 * method completely and override selected helpers to add necessary adjustments
9020 * to their core operation.
9021 *
9022 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
9023 * any performance for debug and analysis features.
9024 *
9025 * @{
9026 */
9027
9028/**
9029 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
9030 * the debug run loop.
9031 */
9032typedef struct VMXRUNDBGSTATE
9033{
9034 /** The RIP we started executing at. This is for detecting that we stepped. */
9035 uint64_t uRipStart;
9036 /** The CS we started executing with. */
9037 uint16_t uCsStart;
9038
9039 /** Whether we've actually modified the 1st execution control field. */
9040 bool fModifiedProcCtls : 1;
9041 /** Whether we've actually modified the 2nd execution control field. */
9042 bool fModifiedProcCtls2 : 1;
9043 /** Whether we've actually modified the exception bitmap. */
9044 bool fModifiedXcptBitmap : 1;
9045
9046 /** We desire the modified the CR0 mask to be cleared. */
9047 bool fClearCr0Mask : 1;
9048 /** We desire the modified the CR4 mask to be cleared. */
9049 bool fClearCr4Mask : 1;
9050 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9051 uint32_t fCpe1Extra;
9052 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9053 uint32_t fCpe1Unwanted;
9054 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9055 uint32_t fCpe2Extra;
9056 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9057 uint32_t bmXcptExtra;
9058 /** The sequence number of the Dtrace provider settings the state was
9059 * configured against. */
9060 uint32_t uDtraceSettingsSeqNo;
9061 /** VM-exits to check (one bit per VM-exit). */
9062 uint32_t bmExitsToCheck[3];
9063
9064 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9065 uint32_t fProcCtlsInitial;
9066 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9067 uint32_t fProcCtls2Initial;
9068 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9069 uint32_t bmXcptInitial;
9070} VMXRUNDBGSTATE;
9071AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9072typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9073
9074
9075/**
9076 * Initializes the VMXRUNDBGSTATE structure.
9077 *
9078 * @param pVCpu The cross context virtual CPU structure of the
9079 * calling EMT.
9080 * @param pDbgState The structure to initialize.
9081 */
9082static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9083{
9084 pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
9085 pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
9086
9087 pDbgState->fModifiedProcCtls = false;
9088 pDbgState->fModifiedProcCtls2 = false;
9089 pDbgState->fModifiedXcptBitmap = false;
9090 pDbgState->fClearCr0Mask = false;
9091 pDbgState->fClearCr4Mask = false;
9092 pDbgState->fCpe1Extra = 0;
9093 pDbgState->fCpe1Unwanted = 0;
9094 pDbgState->fCpe2Extra = 0;
9095 pDbgState->bmXcptExtra = 0;
9096 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9097 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9098 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9099}
9100
9101
9102/**
9103 * Updates the VMSC fields with changes requested by @a pDbgState.
9104 *
9105 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9106 * immediately before executing guest code, i.e. when interrupts are disabled.
9107 * We don't check status codes here as we cannot easily assert or return in the
9108 * latter case.
9109 *
9110 * @param pVCpu The cross context virtual CPU structure.
9111 * @param pDbgState The debug state.
9112 */
9113static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9114{
9115 /*
9116 * Ensure desired flags in VMCS control fields are set.
9117 * (Ignoring write failure here, as we're committed and it's just debug extras.)
9118 *
9119 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9120 * there should be no stale data in pCtx at this point.
9121 */
9122 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9123 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9124 {
9125 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
9126 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9127 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9128 Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9129 pDbgState->fModifiedProcCtls = true;
9130 }
9131
9132 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9133 {
9134 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
9135 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9136 Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9137 pDbgState->fModifiedProcCtls2 = true;
9138 }
9139
9140 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9141 {
9142 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
9143 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9144 Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9145 pDbgState->fModifiedXcptBitmap = true;
9146 }
9147
9148 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32Cr0Mask != 0)
9149 {
9150 pVCpu->hm.s.vmx.u32Cr0Mask = 0;
9151 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9152 Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9153 }
9154
9155 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32Cr4Mask != 0)
9156 {
9157 pVCpu->hm.s.vmx.u32Cr4Mask = 0;
9158 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9159 Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9160 }
9161}
9162
9163
9164/**
9165 * Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
9166 * re-entry next time around.
9167 *
9168 * @returns Strict VBox status code (i.e. informational status codes too).
9169 * @param pVCpu The cross context virtual CPU structure.
9170 * @param pDbgState The debug state.
9171 * @param rcStrict The return code from executing the guest using single
9172 * stepping.
9173 */
9174static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9175{
9176 /*
9177 * Restore VM-exit control settings as we may not reenter this function the
9178 * next time around.
9179 */
9180 /* We reload the initial value, trigger what we can of recalculations the
9181 next time around. From the looks of things, that's all that's required atm. */
9182 if (pDbgState->fModifiedProcCtls)
9183 {
9184 if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9185 pDbgState->fProcCtlsInitial |= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9186 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9187 AssertRCReturn(rc2, rc2);
9188 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9189 }
9190
9191 /* We're currently the only ones messing with this one, so just restore the
9192 cached value and reload the field. */
9193 if ( pDbgState->fModifiedProcCtls2
9194 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9195 {
9196 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9197 AssertRCReturn(rc2, rc2);
9198 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9199 }
9200
9201 /* If we've modified the exception bitmap, we restore it and trigger
9202 reloading and partial recalculation the next time around. */
9203 if (pDbgState->fModifiedXcptBitmap)
9204 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9205
9206 return rcStrict;
9207}
9208
9209
9210/**
9211 * Configures VM-exit controls for current DBGF and DTrace settings.
9212 *
9213 * This updates @a pDbgState and the VMCS execution control fields to reflect
9214 * the necessary VM-exits demanded by DBGF and DTrace.
9215 *
9216 * @param pVCpu The cross context virtual CPU structure.
9217 * @param pDbgState The debug state.
9218 * @param pVmxTransient Pointer to the VMX transient structure. May update
9219 * fUpdateTscOffsettingAndPreemptTimer.
9220 */
9221static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9222{
9223 /*
9224 * Take down the dtrace serial number so we can spot changes.
9225 */
9226 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9227 ASMCompilerBarrier();
9228
9229 /*
9230 * We'll rebuild most of the middle block of data members (holding the
9231 * current settings) as we go along here, so start by clearing it all.
9232 */
9233 pDbgState->bmXcptExtra = 0;
9234 pDbgState->fCpe1Extra = 0;
9235 pDbgState->fCpe1Unwanted = 0;
9236 pDbgState->fCpe2Extra = 0;
9237 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9238 pDbgState->bmExitsToCheck[i] = 0;
9239
9240 /*
9241 * Software interrupts (INT XXh) - no idea how to trigger these...
9242 */
9243 PVM pVM = pVCpu->CTX_SUFF(pVM);
9244 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9245 || VBOXVMM_INT_SOFTWARE_ENABLED())
9246 {
9247 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9248 }
9249
9250 /*
9251 * INT3 breakpoints - triggered by #BP exceptions.
9252 */
9253 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9254 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9255
9256 /*
9257 * Exception bitmap and XCPT events+probes.
9258 */
9259 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9260 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9261 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9262
9263 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9264 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9265 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9266 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9267 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9268 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9269 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9270 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9271 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9272 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9273 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9274 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9275 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9276 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9277 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9278 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9279 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9280 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9281
9282 if (pDbgState->bmXcptExtra)
9283 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9284
9285 /*
9286 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9287 *
9288 * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
9289 * So, when adding/changing/removing please don't forget to update it.
9290 *
9291 * Some of the macros are picking up local variables to save horizontal space,
9292 * (being able to see it in a table is the lesser evil here).
9293 */
9294#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9295 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9296 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9297#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9298 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9299 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9300 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9301 } else do { } while (0)
9302#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9303 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9304 { \
9305 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9306 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9307 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9308 } else do { } while (0)
9309#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9310 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9311 { \
9312 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9313 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9314 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9315 } else do { } while (0)
9316#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9317 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9318 { \
9319 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9320 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9321 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9322 } else do { } while (0)
9323
9324 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9325 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9326 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9327 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9328 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9329
9330 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9331 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9332 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9333 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9334 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
9335 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9336 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9337 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9338 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
9339 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9340 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
9341 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9342 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
9343 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9344 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9345 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9346 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9347 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9348 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9349 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9350 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9351 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9352 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9353 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9354 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9355 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9356 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9357 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9358 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9359 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9360 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9361 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9362 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9363 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9364 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9365 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9366
9367 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9368 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9369 {
9370 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_APIC_TPR);
9371 AssertRC(rc);
9372
9373#if 0 /** @todo fix me */
9374 pDbgState->fClearCr0Mask = true;
9375 pDbgState->fClearCr4Mask = true;
9376#endif
9377 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9378 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_STORE_EXIT | VMX_PROC_CTLS_CR8_STORE_EXIT;
9379 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9380 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_LOAD_EXIT | VMX_PROC_CTLS_CR8_LOAD_EXIT;
9381 pDbgState->fCpe1Unwanted |= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
9382 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9383 require clearing here and in the loop if we start using it. */
9384 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9385 }
9386 else
9387 {
9388 if (pDbgState->fClearCr0Mask)
9389 {
9390 pDbgState->fClearCr0Mask = false;
9391 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
9392 }
9393 if (pDbgState->fClearCr4Mask)
9394 {
9395 pDbgState->fClearCr4Mask = false;
9396 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
9397 }
9398 }
9399 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9400 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9401
9402 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9403 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9404 {
9405 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9406 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9407 }
9408 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9409 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9410
9411 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
9412 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9413 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
9414 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9415 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
9416 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9417 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
9418 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9419#if 0 /** @todo too slow, fix handler. */
9420 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
9421#endif
9422 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9423
9424 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9425 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9426 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9427 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9428 {
9429 pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
9430 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
9431 }
9432 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9433 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9434 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9435 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
9436
9437 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9438 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9439 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9440 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9441 {
9442 pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
9443 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
9444 }
9445 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
9446 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
9447 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
9448 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
9449
9450 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9451 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9452 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
9453 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9454 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9455 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9456 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
9457 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9458 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9459 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9460 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
9461 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9462 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
9463 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9464 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9465 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9466 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
9467 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9468 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9469 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9470 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9471 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9472
9473#undef IS_EITHER_ENABLED
9474#undef SET_ONLY_XBM_IF_EITHER_EN
9475#undef SET_CPE1_XBM_IF_EITHER_EN
9476#undef SET_CPEU_XBM_IF_EITHER_EN
9477#undef SET_CPE2_XBM_IF_EITHER_EN
9478
9479 /*
9480 * Sanitize the control stuff.
9481 */
9482 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
9483 if (pDbgState->fCpe2Extra)
9484 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
9485 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
9486 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0;
9487 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
9488 {
9489 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9490 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9491 }
9492
9493 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9494 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9495 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9496 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9497}
9498
9499
9500/**
9501 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9502 * appropriate.
9503 *
9504 * The caller has checked the VM-exit against the
9505 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9506 * already, so we don't have to do that either.
9507 *
9508 * @returns Strict VBox status code (i.e. informational status codes too).
9509 * @param pVCpu The cross context virtual CPU structure.
9510 * @param pVmxTransient Pointer to the VMX-transient structure.
9511 * @param uExitReason The VM-exit reason.
9512 *
9513 * @remarks The name of this function is displayed by dtrace, so keep it short
9514 * and to the point. No longer than 33 chars long, please.
9515 */
9516static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9517{
9518 /*
9519 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9520 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9521 *
9522 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9523 * does. Must add/change/remove both places. Same ordering, please.
9524 *
9525 * Added/removed events must also be reflected in the next section
9526 * where we dispatch dtrace events.
9527 */
9528 bool fDtrace1 = false;
9529 bool fDtrace2 = false;
9530 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9531 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9532 uint32_t uEventArg = 0;
9533#define SET_EXIT(a_EventSubName) \
9534 do { \
9535 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9536 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9537 } while (0)
9538#define SET_BOTH(a_EventSubName) \
9539 do { \
9540 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9541 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9542 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9543 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9544 } while (0)
9545 switch (uExitReason)
9546 {
9547 case VMX_EXIT_MTF:
9548 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9549
9550 case VMX_EXIT_XCPT_OR_NMI:
9551 {
9552 uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9553 switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
9554 {
9555 case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
9556 case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
9557 case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
9558 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9559 {
9560 if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
9561 {
9562 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9563 uEventArg = pVmxTransient->uExitIntErrorCode;
9564 }
9565 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9566 switch (enmEvent1)
9567 {
9568 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9569 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9570 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9571 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9572 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9573 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9574 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9575 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9576 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9577 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9578 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9579 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9580 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9581 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9582 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9583 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9584 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9585 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9586 default: break;
9587 }
9588 }
9589 else
9590 AssertFailed();
9591 break;
9592
9593 case VMX_EXIT_INT_INFO_TYPE_SW_INT:
9594 uEventArg = idxVector;
9595 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9596 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9597 break;
9598 }
9599 break;
9600 }
9601
9602 case VMX_EXIT_TRIPLE_FAULT:
9603 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9604 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9605 break;
9606 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9607 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9608 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9609 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9610 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9611
9612 /* Instruction specific VM-exits: */
9613 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9614 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9615 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9616 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9617 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9618 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9619 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9620 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9621 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9622 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9623 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9624 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9625 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9626 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9627 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9628 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9629 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9630 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9631 case VMX_EXIT_MOV_CRX:
9632 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9633 if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
9634 SET_BOTH(CRX_READ);
9635 else
9636 SET_BOTH(CRX_WRITE);
9637 uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
9638 break;
9639 case VMX_EXIT_MOV_DRX:
9640 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9641 if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
9642 == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
9643 SET_BOTH(DRX_READ);
9644 else
9645 SET_BOTH(DRX_WRITE);
9646 uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
9647 break;
9648 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9649 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9650 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9651 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9652 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9653 case VMX_EXIT_GDTR_IDTR_ACCESS:
9654 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9655 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
9656 {
9657 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9658 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9659 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9660 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9661 }
9662 break;
9663
9664 case VMX_EXIT_LDTR_TR_ACCESS:
9665 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9666 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
9667 {
9668 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9669 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9670 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9671 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9672 }
9673 break;
9674
9675 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9676 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9677 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9678 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9679 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9680 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9681 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9682 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9683 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9684 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9685 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9686
9687 /* Events that aren't relevant at this point. */
9688 case VMX_EXIT_EXT_INT:
9689 case VMX_EXIT_INT_WINDOW:
9690 case VMX_EXIT_NMI_WINDOW:
9691 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9692 case VMX_EXIT_PREEMPT_TIMER:
9693 case VMX_EXIT_IO_INSTR:
9694 break;
9695
9696 /* Errors and unexpected events. */
9697 case VMX_EXIT_INIT_SIGNAL:
9698 case VMX_EXIT_SIPI:
9699 case VMX_EXIT_IO_SMI:
9700 case VMX_EXIT_SMI:
9701 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9702 case VMX_EXIT_ERR_MSR_LOAD:
9703 case VMX_EXIT_ERR_MACHINE_CHECK:
9704 break;
9705
9706 default:
9707 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9708 break;
9709 }
9710#undef SET_BOTH
9711#undef SET_EXIT
9712
9713 /*
9714 * Dtrace tracepoints go first. We do them here at once so we don't
9715 * have to copy the guest state saving and stuff a few dozen times.
9716 * Down side is that we've got to repeat the switch, though this time
9717 * we use enmEvent since the probes are a subset of what DBGF does.
9718 */
9719 if (fDtrace1 || fDtrace2)
9720 {
9721 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9722 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9723 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9724 switch (enmEvent1)
9725 {
9726 /** @todo consider which extra parameters would be helpful for each probe. */
9727 case DBGFEVENT_END: break;
9728 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
9729 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
9730 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
9731 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
9732 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
9733 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
9734 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
9735 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
9736 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
9737 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
9738 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
9739 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
9740 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
9741 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
9742 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
9743 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
9744 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
9745 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
9746 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9747 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9748 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
9749 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
9750 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
9751 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
9752 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
9753 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
9754 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
9755 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9756 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9757 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9758 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9759 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9760 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
9761 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9762 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
9763 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
9764 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
9765 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
9766 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
9767 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
9768 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
9769 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
9770 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
9771 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
9772 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
9773 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
9774 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
9775 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
9776 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
9777 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
9778 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
9779 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
9780 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
9781 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
9782 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
9783 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
9784 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
9785 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
9786 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
9787 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
9788 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
9789 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
9790 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
9791 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
9792 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
9793 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
9794 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9795 }
9796 switch (enmEvent2)
9797 {
9798 /** @todo consider which extra parameters would be helpful for each probe. */
9799 case DBGFEVENT_END: break;
9800 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
9801 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9802 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
9803 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
9804 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
9805 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
9806 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
9807 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
9808 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
9809 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9810 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9811 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9812 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9813 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9814 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
9815 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9816 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
9817 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
9818 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
9819 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
9820 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
9821 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
9822 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
9823 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
9824 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
9825 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
9826 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
9827 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
9828 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
9829 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
9830 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
9831 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
9832 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
9833 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
9834 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
9835 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
9836 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
9837 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
9838 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
9839 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
9840 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
9841 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
9842 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
9843 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
9844 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
9845 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
9846 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
9847 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
9848 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
9849 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
9850 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
9851 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
9852 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9853 }
9854 }
9855
9856 /*
9857 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9858 * the DBGF call will do a full check).
9859 *
9860 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9861 * Note! If we have to events, we prioritize the first, i.e. the instruction
9862 * one, in order to avoid event nesting.
9863 */
9864 PVM pVM = pVCpu->CTX_SUFF(pVM);
9865 if ( enmEvent1 != DBGFEVENT_END
9866 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9867 {
9868 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
9869 if (rcStrict != VINF_SUCCESS)
9870 return rcStrict;
9871 }
9872 else if ( enmEvent2 != DBGFEVENT_END
9873 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9874 {
9875 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
9876 if (rcStrict != VINF_SUCCESS)
9877 return rcStrict;
9878 }
9879
9880 return VINF_SUCCESS;
9881}
9882
9883
9884/**
9885 * Single-stepping VM-exit filtering.
9886 *
9887 * This is preprocessing the VM-exits and deciding whether we've gotten far
9888 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9889 * handling is performed.
9890 *
9891 * @returns Strict VBox status code (i.e. informational status codes too).
9892 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9893 * @param pVmxTransient Pointer to the VMX-transient structure.
9894 * @param pDbgState The debug state.
9895 */
9896DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
9897{
9898 /*
9899 * Expensive (saves context) generic dtrace VM-exit probe.
9900 */
9901 uint32_t const uExitReason = pVmxTransient->uExitReason;
9902 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9903 { /* more likely */ }
9904 else
9905 {
9906 hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
9907 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9908 AssertRC(rc);
9909 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
9910 }
9911
9912 /*
9913 * Check for host NMI, just to get that out of the way.
9914 */
9915 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9916 { /* normally likely */ }
9917 else
9918 {
9919 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9920 AssertRCReturn(rc2, rc2);
9921 uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
9922 if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
9923 return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
9924 }
9925
9926 /*
9927 * Check for single stepping event if we're stepping.
9928 */
9929 if (pVCpu->hm.s.fSingleInstruction)
9930 {
9931 switch (uExitReason)
9932 {
9933 case VMX_EXIT_MTF:
9934 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9935
9936 /* Various events: */
9937 case VMX_EXIT_XCPT_OR_NMI:
9938 case VMX_EXIT_EXT_INT:
9939 case VMX_EXIT_TRIPLE_FAULT:
9940 case VMX_EXIT_INT_WINDOW:
9941 case VMX_EXIT_NMI_WINDOW:
9942 case VMX_EXIT_TASK_SWITCH:
9943 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9944 case VMX_EXIT_APIC_ACCESS:
9945 case VMX_EXIT_EPT_VIOLATION:
9946 case VMX_EXIT_EPT_MISCONFIG:
9947 case VMX_EXIT_PREEMPT_TIMER:
9948
9949 /* Instruction specific VM-exits: */
9950 case VMX_EXIT_CPUID:
9951 case VMX_EXIT_GETSEC:
9952 case VMX_EXIT_HLT:
9953 case VMX_EXIT_INVD:
9954 case VMX_EXIT_INVLPG:
9955 case VMX_EXIT_RDPMC:
9956 case VMX_EXIT_RDTSC:
9957 case VMX_EXIT_RSM:
9958 case VMX_EXIT_VMCALL:
9959 case VMX_EXIT_VMCLEAR:
9960 case VMX_EXIT_VMLAUNCH:
9961 case VMX_EXIT_VMPTRLD:
9962 case VMX_EXIT_VMPTRST:
9963 case VMX_EXIT_VMREAD:
9964 case VMX_EXIT_VMRESUME:
9965 case VMX_EXIT_VMWRITE:
9966 case VMX_EXIT_VMXOFF:
9967 case VMX_EXIT_VMXON:
9968 case VMX_EXIT_MOV_CRX:
9969 case VMX_EXIT_MOV_DRX:
9970 case VMX_EXIT_IO_INSTR:
9971 case VMX_EXIT_RDMSR:
9972 case VMX_EXIT_WRMSR:
9973 case VMX_EXIT_MWAIT:
9974 case VMX_EXIT_MONITOR:
9975 case VMX_EXIT_PAUSE:
9976 case VMX_EXIT_GDTR_IDTR_ACCESS:
9977 case VMX_EXIT_LDTR_TR_ACCESS:
9978 case VMX_EXIT_INVEPT:
9979 case VMX_EXIT_RDTSCP:
9980 case VMX_EXIT_INVVPID:
9981 case VMX_EXIT_WBINVD:
9982 case VMX_EXIT_XSETBV:
9983 case VMX_EXIT_RDRAND:
9984 case VMX_EXIT_INVPCID:
9985 case VMX_EXIT_VMFUNC:
9986 case VMX_EXIT_RDSEED:
9987 case VMX_EXIT_XSAVES:
9988 case VMX_EXIT_XRSTORS:
9989 {
9990 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
9991 AssertRCReturn(rc, rc);
9992 if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
9993 || pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
9994 return VINF_EM_DBG_STEPPED;
9995 break;
9996 }
9997
9998 /* Errors and unexpected events: */
9999 case VMX_EXIT_INIT_SIGNAL:
10000 case VMX_EXIT_SIPI:
10001 case VMX_EXIT_IO_SMI:
10002 case VMX_EXIT_SMI:
10003 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10004 case VMX_EXIT_ERR_MSR_LOAD:
10005 case VMX_EXIT_ERR_MACHINE_CHECK:
10006 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
10007 break;
10008
10009 default:
10010 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10011 break;
10012 }
10013 }
10014
10015 /*
10016 * Check for debugger event breakpoints and dtrace probes.
10017 */
10018 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
10019 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
10020 {
10021 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
10022 if (rcStrict != VINF_SUCCESS)
10023 return rcStrict;
10024 }
10025
10026 /*
10027 * Normal processing.
10028 */
10029#ifdef HMVMX_USE_FUNCTION_TABLE
10030 return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
10031#else
10032 return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
10033#endif
10034}
10035
10036
10037/**
10038 * Single steps guest code using VT-x.
10039 *
10040 * @returns Strict VBox status code (i.e. informational status codes too).
10041 * @param pVCpu The cross context virtual CPU structure.
10042 *
10043 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10044 */
10045static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu)
10046{
10047 VMXTRANSIENT VmxTransient;
10048 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10049
10050 /* Set HMCPU indicators. */
10051 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10052 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
10053 pVCpu->hm.s.fDebugWantRdTscExit = false;
10054 pVCpu->hm.s.fUsingDebugLoop = true;
10055
10056 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10057 VMXRUNDBGSTATE DbgState;
10058 hmR0VmxRunDebugStateInit(pVCpu, &DbgState);
10059 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
10060
10061 /*
10062 * The loop.
10063 */
10064 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10065 for (uint32_t cLoops = 0; ; cLoops++)
10066 {
10067 Assert(!HMR0SuspendPending());
10068 HMVMX_ASSERT_CPU_SAFE(pVCpu);
10069 bool fStepping = pVCpu->hm.s.fSingleInstruction;
10070
10071 /*
10072 * Preparatory work for running guest code, this may force us to return
10073 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10074 */
10075 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10076 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10077 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
10078 if (rcStrict != VINF_SUCCESS)
10079 break;
10080
10081 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10082 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10083
10084 /*
10085 * Now we can run the guest code.
10086 */
10087 int rcRun = hmR0VmxRunGuest(pVCpu);
10088
10089 /*
10090 * Restore any residual host-state and save any bits shared between host
10091 * and guest into the guest-CPU state. Re-enables interrupts!
10092 */
10093 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10094
10095 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10096 if (RT_SUCCESS(rcRun))
10097 { /* very likely */ }
10098 else
10099 {
10100 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10101 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10102 return rcRun;
10103 }
10104
10105 /* Profile the VM-exit. */
10106 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10107 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10108 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10109 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10110 HMVMX_START_EXIT_DISPATCH_PROF();
10111
10112 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10113
10114 /*
10115 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10116 */
10117 rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
10118 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10119 if (rcStrict != VINF_SUCCESS)
10120 break;
10121 if (cLoops > pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
10122 {
10123 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10124 rcStrict = VINF_EM_RAW_INTERRUPT;
10125 break;
10126 }
10127
10128 /*
10129 * Stepping: Did the RIP change, if so, consider it a single step.
10130 * Otherwise, make sure one of the TFs gets set.
10131 */
10132 if (fStepping)
10133 {
10134 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
10135 AssertRC(rc);
10136 if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
10137 || pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
10138 {
10139 rcStrict = VINF_EM_DBG_STEPPED;
10140 break;
10141 }
10142 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
10143 }
10144
10145 /*
10146 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
10147 */
10148 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10149 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
10150 }
10151
10152 /*
10153 * Clear the X86_EFL_TF if necessary.
10154 */
10155 if (pVCpu->hm.s.fClearTrapFlag)
10156 {
10157 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
10158 AssertRC(rc);
10159 pVCpu->hm.s.fClearTrapFlag = false;
10160 pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
10161 }
10162 /** @todo there seems to be issues with the resume flag when the monitor trap
10163 * flag is pending without being used. Seen early in bios init when
10164 * accessing APIC page in protected mode. */
10165
10166 /*
10167 * Restore VM-exit control settings as we may not reenter this function the
10168 * next time around.
10169 */
10170 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10171
10172 /* Restore HMCPU indicators. */
10173 pVCpu->hm.s.fUsingDebugLoop = false;
10174 pVCpu->hm.s.fDebugWantRdTscExit = false;
10175 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10176
10177 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10178 return rcStrict;
10179}
10180
10181
10182/** @} */
10183
10184
10185/**
10186 * Checks if any expensive dtrace probes are enabled and we should go to the
10187 * debug loop.
10188 *
10189 * @returns true if we should use debug loop, false if not.
10190 */
10191static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10192{
10193 /* It's probably faster to OR the raw 32-bit counter variables together.
10194 Since the variables are in an array and the probes are next to one
10195 another (more or less), we have good locality. So, better read
10196 eight-nine cache lines ever time and only have one conditional, than
10197 128+ conditionals, right? */
10198 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10199 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10200 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10201 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10202 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10203 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10204 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10205 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10206 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10207 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10208 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10209 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10210 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10211 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10212 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10213 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10214 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10215 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10216 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10217 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10218 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10219 ) != 0
10220 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10221 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10222 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10223 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10224 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10225 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10226 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10227 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10228 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10229 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10230 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10231 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10232 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10233 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10234 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10235 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10236 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10237 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10238 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10239 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10240 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10241 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10242 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10243 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10244 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10245 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10246 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10247 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10248 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10249 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10250 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10251 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10252 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10253 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10254 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10255 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10256 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10257 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10258 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10259 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10260 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10261 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10262 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10263 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10264 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10265 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10266 ) != 0
10267 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10268 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10269 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10270 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10271 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10272 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10273 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10274 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10275 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10276 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10277 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10278 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10279 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10280 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10281 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10282 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10283 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10284 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10285 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10286 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10287 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10288 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10289 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10290 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10291 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10292 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10293 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10294 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10295 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10296 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10297 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10298 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10299 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10300 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10301 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10302 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10303 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10304 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10305 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10306 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10307 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10308 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10309 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10310 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10311 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10312 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10313 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10314 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10315 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10316 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10317 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10318 ) != 0;
10319}
10320
10321
10322/**
10323 * Runs the guest code using VT-x.
10324 *
10325 * @returns Strict VBox status code (i.e. informational status codes too).
10326 * @param pVCpu The cross context virtual CPU structure.
10327 */
10328VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
10329{
10330 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10331 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10332 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10333 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
10334
10335 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10336
10337 VBOXSTRICTRC rcStrict;
10338 if ( !pVCpu->hm.s.fUseDebugLoop
10339 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10340 && !DBGFIsStepping(pVCpu)
10341 && !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
10342 rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu);
10343 else
10344 rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu);
10345
10346 if (rcStrict == VERR_EM_INTERPRETER)
10347 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10348 else if (rcStrict == VINF_EM_RESET)
10349 rcStrict = VINF_EM_TRIPLE_FAULT;
10350
10351 int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
10352 if (RT_FAILURE(rc2))
10353 {
10354 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10355 rcStrict = rc2;
10356 }
10357 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10358 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10359 return rcStrict;
10360}
10361
10362
10363#ifndef HMVMX_USE_FUNCTION_TABLE
10364DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10365{
10366#ifdef DEBUG_ramshankar
10367#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
10368 do { \
10369 if (a_fSave != 0) \
10370 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
10371 VBOXSTRICTRC rcStrict = a_CallExpr; \
10372 if (a_fSave != 0) \
10373 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
10374 return rcStrict; \
10375 } while (0)
10376#else
10377# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
10378#endif
10379 switch (rcReason)
10380 {
10381 case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
10382 case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
10383 case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
10384 case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
10385 case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
10386 case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
10387 case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
10388 case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
10389 case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
10390 case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
10391 case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
10392 case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
10393 case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
10394 case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
10395 case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
10396 case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
10397 case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
10398 case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
10399 case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
10400 case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
10401 case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
10402 case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
10403 case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
10404 case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pVmxTransient));
10405 case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
10406 case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
10407 case VMX_EXIT_GDTR_IDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10408 case VMX_EXIT_LDTR_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10409 case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
10410 case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
10411 case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
10412 case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
10413 case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
10414 case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
10415
10416 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
10417 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
10418 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pVmxTransient);
10419 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pVmxTransient);
10420 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pVmxTransient);
10421 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pVmxTransient);
10422 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pVmxTransient);
10423 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
10424 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pVmxTransient);
10425
10426 case VMX_EXIT_VMCLEAR:
10427 case VMX_EXIT_VMLAUNCH:
10428 case VMX_EXIT_VMPTRLD:
10429 case VMX_EXIT_VMPTRST:
10430 case VMX_EXIT_VMREAD:
10431 case VMX_EXIT_VMRESUME:
10432 case VMX_EXIT_VMWRITE:
10433 case VMX_EXIT_VMXOFF:
10434 case VMX_EXIT_VMXON:
10435 case VMX_EXIT_INVEPT:
10436 case VMX_EXIT_INVVPID:
10437 case VMX_EXIT_VMFUNC:
10438 case VMX_EXIT_XSAVES:
10439 case VMX_EXIT_XRSTORS:
10440 return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
10441
10442 case VMX_EXIT_ENCLS:
10443 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10444 case VMX_EXIT_PML_FULL:
10445 default:
10446 return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
10447 }
10448#undef VMEXIT_CALL_RET
10449}
10450#endif /* !HMVMX_USE_FUNCTION_TABLE */
10451
10452
10453#ifdef VBOX_STRICT
10454/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10455# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10456 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10457
10458# define HMVMX_ASSERT_PREEMPT_CPUID() \
10459 do { \
10460 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10461 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10462 } while (0)
10463
10464# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10465 do { \
10466 AssertPtr((a_pVCpu)); \
10467 AssertPtr((a_pVmxTransient)); \
10468 Assert((a_pVmxTransient)->fVMEntryFailed == false); \
10469 Assert(ASMIntAreEnabled()); \
10470 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10471 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10472 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
10473 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10474 if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
10475 HMVMX_ASSERT_PREEMPT_CPUID(); \
10476 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10477 } while (0)
10478
10479# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10480 do { \
10481 Log4Func(("\n")); \
10482 } while (0)
10483#else
10484# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10485 do { \
10486 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10487 NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
10488 } while (0)
10489# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
10490#endif
10491
10492
10493/**
10494 * Advances the guest RIP by the specified number of bytes.
10495 *
10496 * @param pVCpu The cross context virtual CPU structure.
10497 * @param cbInstr Number of bytes to advance the RIP by.
10498 *
10499 * @remarks No-long-jump zone!!!
10500 */
10501DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
10502{
10503 /* Advance the RIP. */
10504 pVCpu->cpum.GstCtx.rip += cbInstr;
10505 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
10506
10507 /* Update interrupt inhibition. */
10508 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10509 && pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
10510 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10511}
10512
10513
10514/**
10515 * Advances the guest RIP after reading it from the VMCS.
10516 *
10517 * @returns VBox status code, no informational status codes.
10518 * @param pVCpu The cross context virtual CPU structure.
10519 * @param pVmxTransient Pointer to the VMX transient structure.
10520 *
10521 * @remarks No-long-jump zone!!!
10522 */
10523static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10524{
10525 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10526 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS);
10527 AssertRCReturn(rc, rc);
10528
10529 hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
10530
10531 /*
10532 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10533 * pending debug exception field as it takes care of priority of events.
10534 *
10535 * See Intel spec. 32.2.1 "Debug Exceptions".
10536 */
10537 if ( !pVCpu->hm.s.fSingleInstruction
10538 && pVCpu->cpum.GstCtx.eflags.Bits.u1TF)
10539 {
10540 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10541 AssertRCReturn(rc, rc);
10542 }
10543
10544 return VINF_SUCCESS;
10545}
10546
10547
10548/**
10549 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10550 * and update error record fields accordingly.
10551 *
10552 * @return VMX_IGS_* return codes.
10553 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10554 * wrong with the guest state.
10555 *
10556 * @param pVCpu The cross context virtual CPU structure.
10557 *
10558 * @remarks This function assumes our cache of the VMCS controls
10559 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10560 */
10561static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu)
10562{
10563#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10564#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10565 uError = (err); \
10566 break; \
10567 } else do { } while (0)
10568
10569 int rc;
10570 PVM pVM = pVCpu->CTX_SUFF(pVM);
10571 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10572 uint32_t uError = VMX_IGS_ERROR;
10573 uint32_t u32Val;
10574 bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10575
10576 do
10577 {
10578 /*
10579 * CR0.
10580 */
10581 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10582 uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10583 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10584 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10585 if (fUnrestrictedGuest)
10586 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
10587
10588 uint32_t u32GuestCr0;
10589 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10590 AssertRCBreak(rc);
10591 HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10592 HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10593 if ( !fUnrestrictedGuest
10594 && (u32GuestCr0 & X86_CR0_PG)
10595 && !(u32GuestCr0 & X86_CR0_PE))
10596 {
10597 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10598 }
10599
10600 /*
10601 * CR4.
10602 */
10603 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10604 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10605
10606 uint32_t u32GuestCr4;
10607 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10608 AssertRCBreak(rc);
10609 HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10610 HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10611
10612 /*
10613 * IA32_DEBUGCTL MSR.
10614 */
10615 uint64_t u64Val;
10616 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10617 AssertRCBreak(rc);
10618 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10619 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10620 {
10621 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10622 }
10623 uint64_t u64DebugCtlMsr = u64Val;
10624
10625#ifdef VBOX_STRICT
10626 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10627 AssertRCBreak(rc);
10628 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10629#endif
10630 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
10631
10632 /*
10633 * RIP and RFLAGS.
10634 */
10635 uint32_t u32Eflags;
10636#if HC_ARCH_BITS == 64
10637 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10638 AssertRCBreak(rc);
10639 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10640 if ( !fLongModeGuest
10641 || !pCtx->cs.Attr.n.u1Long)
10642 {
10643 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10644 }
10645 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10646 * must be identical if the "IA-32e mode guest" VM-entry
10647 * control is 1 and CS.L is 1. No check applies if the
10648 * CPU supports 64 linear-address bits. */
10649
10650 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10651 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10652 AssertRCBreak(rc);
10653 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10654 VMX_IGS_RFLAGS_RESERVED);
10655 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10656 u32Eflags = u64Val;
10657#else
10658 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10659 AssertRCBreak(rc);
10660 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10661 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10662#endif
10663
10664 if ( fLongModeGuest
10665 || ( fUnrestrictedGuest
10666 && !(u32GuestCr0 & X86_CR0_PE)))
10667 {
10668 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10669 }
10670
10671 uint32_t u32EntryInfo;
10672 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10673 AssertRCBreak(rc);
10674 if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
10675 && VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
10676 {
10677 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10678 }
10679
10680 /*
10681 * 64-bit checks.
10682 */
10683#if HC_ARCH_BITS == 64
10684 if (fLongModeGuest)
10685 {
10686 HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10687 HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10688 }
10689
10690 if ( !fLongModeGuest
10691 && (u32GuestCr4 & X86_CR4_PCIDE))
10692 {
10693 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10694 }
10695
10696 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10697 * 51:32 beyond the processor's physical-address width are 0. */
10698
10699 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10700 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10701 {
10702 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10703 }
10704
10705 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10706 AssertRCBreak(rc);
10707 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10708
10709 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10710 AssertRCBreak(rc);
10711 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10712#endif
10713
10714 /*
10715 * PERF_GLOBAL MSR.
10716 */
10717 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
10718 {
10719 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10720 AssertRCBreak(rc);
10721 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10722 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10723 }
10724
10725 /*
10726 * PAT MSR.
10727 */
10728 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
10729 {
10730 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10731 AssertRCBreak(rc);
10732 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10733 for (unsigned i = 0; i < 8; i++)
10734 {
10735 uint8_t u8Val = (u64Val & 0xff);
10736 if ( u8Val != 0 /* UC */
10737 && u8Val != 1 /* WC */
10738 && u8Val != 4 /* WT */
10739 && u8Val != 5 /* WP */
10740 && u8Val != 6 /* WB */
10741 && u8Val != 7 /* UC- */)
10742 {
10743 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10744 }
10745 u64Val >>= 8;
10746 }
10747 }
10748
10749 /*
10750 * EFER MSR.
10751 */
10752 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
10753 {
10754 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10755 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10756 AssertRCBreak(rc);
10757 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10758 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10759 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10760 & VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
10761 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10762 /** @todo r=ramshankar: Unrestricted check here is probably wrong, see
10763 * iemVmxVmentryCheckGuestState(). */
10764 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10765 || !(u32GuestCr0 & X86_CR0_PG)
10766 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10767 VMX_IGS_EFER_LMA_LME_MISMATCH);
10768 }
10769
10770 /*
10771 * Segment registers.
10772 */
10773 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10774 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10775 if (!(u32Eflags & X86_EFL_VM))
10776 {
10777 /* CS */
10778 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10779 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10780 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10781 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10782 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10783 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10784 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10785 /* CS cannot be loaded with NULL in protected mode. */
10786 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10787 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10788 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10789 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10790 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10791 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10792 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10793 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10794 else
10795 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10796
10797 /* SS */
10798 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10799 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10800 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10801 if ( !(pCtx->cr0 & X86_CR0_PE)
10802 || pCtx->cs.Attr.n.u4Type == 3)
10803 {
10804 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10805 }
10806 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10807 {
10808 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10809 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10810 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10811 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10812 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10813 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10814 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10815 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10816 }
10817
10818 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmenReg(). */
10819 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10820 {
10821 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10822 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10823 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10824 || pCtx->ds.Attr.n.u4Type > 11
10825 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10826 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10827 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10828 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10829 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10830 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10831 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10832 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10833 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10834 }
10835 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10836 {
10837 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10838 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10839 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10840 || pCtx->es.Attr.n.u4Type > 11
10841 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10842 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10843 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10844 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10845 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10846 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10847 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10848 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10849 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10850 }
10851 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10852 {
10853 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10854 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10855 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10856 || pCtx->fs.Attr.n.u4Type > 11
10857 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10858 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10859 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10860 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10861 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10862 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10863 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10864 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10865 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10866 }
10867 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10868 {
10869 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10870 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10871 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10872 || pCtx->gs.Attr.n.u4Type > 11
10873 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10874 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10875 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10876 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10877 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10878 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10879 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10880 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10881 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10882 }
10883 /* 64-bit capable CPUs. */
10884#if HC_ARCH_BITS == 64
10885 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10886 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10887 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10888 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10889 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10890 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10891 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10892 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10893 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10894 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10895 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10896#endif
10897 }
10898 else
10899 {
10900 /* V86 mode checks. */
10901 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10902 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10903 {
10904 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10905 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10906 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10907 }
10908 else
10909 {
10910 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10911 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10912 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10913 }
10914
10915 /* CS */
10916 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10917 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10918 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10919 /* SS */
10920 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10921 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10922 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10923 /* DS */
10924 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10925 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10926 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10927 /* ES */
10928 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10929 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10930 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10931 /* FS */
10932 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10933 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10934 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10935 /* GS */
10936 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10937 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10938 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10939 /* 64-bit capable CPUs. */
10940#if HC_ARCH_BITS == 64
10941 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10942 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10943 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10944 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10945 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10946 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10947 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10948 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10949 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10950 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10951 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10952#endif
10953 }
10954
10955 /*
10956 * TR.
10957 */
10958 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10959 /* 64-bit capable CPUs. */
10960#if HC_ARCH_BITS == 64
10961 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10962#endif
10963 if (fLongModeGuest)
10964 {
10965 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10966 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10967 }
10968 else
10969 {
10970 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10971 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10972 VMX_IGS_TR_ATTR_TYPE_INVALID);
10973 }
10974 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10975 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10976 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10977 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10978 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10979 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10980 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10981 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10982
10983 /*
10984 * GDTR and IDTR.
10985 */
10986#if HC_ARCH_BITS == 64
10987 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10988 AssertRCBreak(rc);
10989 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10990
10991 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10992 AssertRCBreak(rc);
10993 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10994#endif
10995
10996 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10997 AssertRCBreak(rc);
10998 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10999
11000 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
11001 AssertRCBreak(rc);
11002 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11003
11004 /*
11005 * Guest Non-Register State.
11006 */
11007 /* Activity State. */
11008 uint32_t u32ActivityState;
11009 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
11010 AssertRCBreak(rc);
11011 HMVMX_CHECK_BREAK( !u32ActivityState
11012 || (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
11013 VMX_IGS_ACTIVITY_STATE_INVALID);
11014 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
11015 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
11016 uint32_t u32IntrState;
11017 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
11018 AssertRCBreak(rc);
11019 if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
11020 || u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
11021 {
11022 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11023 }
11024
11025 /** @todo Activity state and injecting interrupts. Left as a todo since we
11026 * currently don't use activity states but ACTIVE. */
11027
11028 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
11029 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11030
11031 /* Guest interruptibility-state. */
11032 HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11033 HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
11034 != (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
11035 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11036 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11037 || !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
11038 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11039 if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
11040 {
11041 if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
11042 {
11043 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
11044 && !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
11045 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11046 }
11047 else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
11048 {
11049 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
11050 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11051 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
11052 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11053 }
11054 }
11055 /** @todo Assumes the processor is not in SMM. */
11056 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
11057 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11058 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
11059 || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
11060 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11061 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
11062 && VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
11063 && VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
11064 {
11065 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
11066 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11067 }
11068
11069 /* Pending debug exceptions. */
11070#if HC_ARCH_BITS == 64
11071 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
11072 AssertRCBreak(rc);
11073 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11074 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11075 u32Val = u64Val; /* For pending debug exceptions checks below. */
11076#else
11077 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
11078 AssertRCBreak(rc);
11079 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11080 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11081#endif
11082
11083 if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
11084 || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
11085 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11086 {
11087 if ( (u32Eflags & X86_EFL_TF)
11088 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11089 {
11090 /* Bit 14 is PendingDebug.BS. */
11091 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11092 }
11093 if ( !(u32Eflags & X86_EFL_TF)
11094 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11095 {
11096 /* Bit 14 is PendingDebug.BS. */
11097 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11098 }
11099 }
11100
11101 /* VMCS link pointer. */
11102 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11103 AssertRCBreak(rc);
11104 if (u64Val != UINT64_C(0xffffffffffffffff))
11105 {
11106 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11107 /** @todo Bits beyond the processor's physical-address width MBZ. */
11108 /** @todo 32-bit located in memory referenced by value of this field (as a
11109 * physical address) must contain the processor's VMCS revision ID. */
11110 /** @todo SMM checks. */
11111 }
11112
11113 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11114 * not using Nested Paging? */
11115 if ( pVM->hm.s.fNestedPaging
11116 && !fLongModeGuest
11117 && CPUMIsGuestInPAEModeEx(pCtx))
11118 {
11119 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11120 AssertRCBreak(rc);
11121 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11122
11123 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11124 AssertRCBreak(rc);
11125 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11126
11127 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11128 AssertRCBreak(rc);
11129 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11130
11131 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11132 AssertRCBreak(rc);
11133 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11134 }
11135
11136 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11137 if (uError == VMX_IGS_ERROR)
11138 uError = VMX_IGS_REASON_NOT_FOUND;
11139 } while (0);
11140
11141 pVCpu->hm.s.u32HMError = uError;
11142 return uError;
11143
11144#undef HMVMX_ERROR_BREAK
11145#undef HMVMX_CHECK_BREAK
11146}
11147
11148
11149/** @name VM-exit handlers.
11150 * @{
11151 */
11152/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11153/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11154/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11155
11156/**
11157 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11158 */
11159HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11160{
11161 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11162 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11163 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11164 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11165 return VINF_SUCCESS;
11166 return VINF_EM_RAW_INTERRUPT;
11167}
11168
11169
11170/**
11171 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11172 */
11173HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11174{
11175 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11176 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11177
11178 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11179 AssertRCReturn(rc, rc);
11180
11181 uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
11182 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
11183 && uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
11184 Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11185
11186 if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
11187 {
11188 /*
11189 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
11190 * injected it ourselves and anything we inject is not going to cause a VM-exit directly
11191 * for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
11192 *
11193 * [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11194 * [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
11195 */
11196 VMXDispatchHostNmi();
11197 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11198 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11199 return VINF_SUCCESS;
11200 }
11201
11202 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11203 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
11204 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11205 { /* likely */ }
11206 else
11207 {
11208 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11209 rcStrictRc1 = VINF_SUCCESS;
11210 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11211 return rcStrictRc1;
11212 }
11213
11214 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11215 uint32_t uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
11216 switch (uIntType)
11217 {
11218 case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11219 Assert(uVector == X86_XCPT_DB);
11220 RT_FALL_THRU();
11221 case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11222 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
11223 RT_FALL_THRU();
11224 case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11225 {
11226 /*
11227 * If there's any exception caused as a result of event injection, the resulting
11228 * secondary/final execption will be pending, we shall continue guest execution
11229 * after injecting the event. The page-fault case is complicated and we manually
11230 * handle any currently pending event in hmR0VmxExitXcptPF.
11231 */
11232 if (!pVCpu->hm.s.Event.fPending)
11233 { /* likely */ }
11234 else if (uVector != X86_XCPT_PF)
11235 {
11236 rc = VINF_SUCCESS;
11237 break;
11238 }
11239
11240 switch (uVector)
11241 {
11242 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
11243 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
11244 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
11245 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
11246 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
11247 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
11248
11249 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11250 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11251 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11252 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11253 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11254 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11255 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11256 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11257 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11258 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11259 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11260 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11261 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11262 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
11263 default:
11264 {
11265 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11266 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11267 {
11268 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11269 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11270 Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
11271
11272 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
11273 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11274 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11275 AssertRCReturn(rc, rc);
11276 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11277 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11278 0 /* GCPtrFaultAddress */);
11279 }
11280 else
11281 {
11282 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11283 pVCpu->hm.s.u32HMError = uVector;
11284 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11285 }
11286 break;
11287 }
11288 }
11289 break;
11290 }
11291
11292 default:
11293 {
11294 pVCpu->hm.s.u32HMError = uExitIntInfo;
11295 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11296 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
11297 break;
11298 }
11299 }
11300 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11301 return rc;
11302}
11303
11304
11305/**
11306 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11307 */
11308HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11309{
11310 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11311
11312 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11313 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11314
11315 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11316 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11317 return VINF_SUCCESS;
11318}
11319
11320
11321/**
11322 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11323 */
11324HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11325{
11326 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11327 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)))
11328 {
11329 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11330 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11331 }
11332
11333 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11334
11335 /*
11336 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11337 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11338 */
11339 uint32_t fIntrState = 0;
11340 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
11341 AssertRCReturn(rc, rc);
11342
11343 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
11344 if ( fBlockSti
11345 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11346 {
11347 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11348 }
11349
11350 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11351 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11352
11353 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11354 return VINF_SUCCESS;
11355}
11356
11357
11358/**
11359 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11360 */
11361HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11362{
11363 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11364 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11365}
11366
11367
11368/**
11369 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11370 */
11371HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11372{
11373 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11374 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11375}
11376
11377
11378/**
11379 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11380 */
11381HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11382{
11383 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11384
11385 /*
11386 * Get the state we need and update the exit history entry.
11387 */
11388 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11389 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
11390 AssertRCReturn(rc, rc);
11391
11392 VBOXSTRICTRC rcStrict;
11393 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11394 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
11395 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11396 if (!pExitRec)
11397 {
11398 /*
11399 * Regular CPUID instruction execution.
11400 */
11401 rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
11402 if (rcStrict == VINF_SUCCESS)
11403 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11404 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11405 {
11406 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11407 rcStrict = VINF_SUCCESS;
11408 }
11409 }
11410 else
11411 {
11412 /*
11413 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
11414 */
11415 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11416 AssertRCReturn(rc2, rc2);
11417
11418 Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11419 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11420
11421 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11422 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
11423
11424 Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11425 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11426 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11427 }
11428 return rcStrict;
11429}
11430
11431
11432/**
11433 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11434 */
11435HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11436{
11437 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11438 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4);
11439 AssertRCReturn(rc, rc);
11440
11441 if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
11442 return VINF_EM_RAW_EMULATE_INSTR;
11443
11444 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11445 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11446}
11447
11448
11449/**
11450 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11451 */
11452HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11453{
11454 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11455 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11456 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11457 AssertRCReturn(rc, rc);
11458
11459 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11460 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11461 {
11462 /* If we get a spurious VM-exit when offsetting is enabled,
11463 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11464 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
11465 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11466 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11467 }
11468 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11469 {
11470 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11471 rcStrict = VINF_SUCCESS;
11472 }
11473 return rcStrict;
11474}
11475
11476
11477/**
11478 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11479 */
11480HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11481{
11482 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11483 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_TSC_AUX);
11484 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11485 AssertRCReturn(rc, rc);
11486
11487 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11488 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11489 {
11490 /* If we get a spurious VM-exit when offsetting is enabled,
11491 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11492 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
11493 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11494 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11495 }
11496 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11497 {
11498 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11499 rcStrict = VINF_SUCCESS;
11500 }
11501 return rcStrict;
11502}
11503
11504
11505/**
11506 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11507 */
11508HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11509{
11510 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11511 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11512 AssertRCReturn(rc, rc);
11513
11514 PVM pVM = pVCpu->CTX_SUFF(pVM);
11515 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11516 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11517 if (RT_LIKELY(rc == VINF_SUCCESS))
11518 {
11519 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11520 Assert(pVmxTransient->cbInstr == 2);
11521 }
11522 else
11523 {
11524 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11525 rc = VERR_EM_INTERPRETER;
11526 }
11527 return rc;
11528}
11529
11530
11531/**
11532 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11533 */
11534HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11535{
11536 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11537
11538 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11539 if (EMAreHypercallInstructionsEnabled(pVCpu))
11540 {
11541 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_SS
11542 | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_EFER);
11543 AssertRCReturn(rc, rc);
11544
11545 /* Perform the hypercall. */
11546 rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
11547 if (rcStrict == VINF_SUCCESS)
11548 {
11549 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11550 AssertRCReturn(rc, rc);
11551 }
11552 else
11553 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11554 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11555 || RT_FAILURE(rcStrict));
11556
11557 /* If the hypercall changes anything other than guest's general-purpose registers,
11558 we would need to reload the guest changed bits here before VM-entry. */
11559 }
11560 else
11561 Log4Func(("Hypercalls not enabled\n"));
11562
11563 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11564 if (RT_FAILURE(rcStrict))
11565 {
11566 hmR0VmxSetPendingXcptUD(pVCpu);
11567 rcStrict = VINF_SUCCESS;
11568 }
11569
11570 return rcStrict;
11571}
11572
11573
11574/**
11575 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11576 */
11577HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11578{
11579 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11580 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11581
11582 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11583 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11584 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
11585 AssertRCReturn(rc, rc);
11586
11587 VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
11588
11589 if (rcStrict == VINF_SUCCESS || rcStrict == VINF_PGM_SYNC_CR3)
11590 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11591 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11592 {
11593 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11594 rcStrict = VINF_SUCCESS;
11595 }
11596 else
11597 AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) sttus: %Rrc\n", pVmxTransient->uExitQual,
11598 VBOXSTRICTRC_VAL(rcStrict)));
11599 return rcStrict;
11600}
11601
11602
11603/**
11604 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11605 */
11606HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11607{
11608 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11609 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11610 AssertRCReturn(rc, rc);
11611
11612 PVM pVM = pVCpu->CTX_SUFF(pVM);
11613 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11614 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11615 if (RT_LIKELY(rc == VINF_SUCCESS))
11616 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11617 else
11618 {
11619 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11620 rc = VERR_EM_INTERPRETER;
11621 }
11622 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11623 return rc;
11624}
11625
11626
11627/**
11628 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11629 */
11630HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11631{
11632 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11633 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11634 AssertRCReturn(rc, rc);
11635
11636 PVM pVM = pVCpu->CTX_SUFF(pVM);
11637 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11638 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11639 rc = VBOXSTRICTRC_VAL(rc2);
11640 if (RT_LIKELY( rc == VINF_SUCCESS
11641 || rc == VINF_EM_HALT))
11642 {
11643 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11644 AssertRCReturn(rc3, rc3);
11645
11646 if ( rc == VINF_EM_HALT
11647 && EMMonitorWaitShouldContinue(pVCpu, pCtx))
11648 rc = VINF_SUCCESS;
11649 }
11650 else
11651 {
11652 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11653 rc = VERR_EM_INTERPRETER;
11654 }
11655 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11656 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11657 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11658 return rc;
11659}
11660
11661
11662/**
11663 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11664 */
11665HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11666{
11667 /*
11668 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
11669 * mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
11670 * treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
11671 * VMX root operation. If we get here, something funny is going on.
11672 *
11673 * See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
11674 */
11675 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11676 AssertMsgFailed(("Unexpected RSM VM-exit\n"));
11677 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11678}
11679
11680
11681/**
11682 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11683 */
11684HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11685{
11686 /*
11687 * This can only happen if we support dual-monitor treatment of SMI, which can be activated
11688 * by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
11689 * this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
11690 * an SMI. If we get here, something funny is going on.
11691 *
11692 * See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11693 * See Intel spec. 25.3 "Other Causes of VM-Exits"
11694 */
11695 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11696 AssertMsgFailed(("Unexpected SMI VM-exit\n"));
11697 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11698}
11699
11700
11701/**
11702 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11703 */
11704HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11705{
11706 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11707 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11708 AssertMsgFailed(("Unexpected IO SMI VM-exit\n"));
11709 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11710}
11711
11712
11713/**
11714 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11715 */
11716HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11717{
11718 /*
11719 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
11720 * We don't make use of it as our guests don't have direct access to the host LAPIC.
11721 * See Intel spec. 25.3 "Other Causes of VM-exits".
11722 */
11723 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11724 AssertMsgFailed(("Unexpected SIPI VM-exit\n"));
11725 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11726}
11727
11728
11729/**
11730 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11731 * VM-exit.
11732 */
11733HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11734{
11735 /*
11736 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11737 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11738 *
11739 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11740 * See Intel spec. "23.8 Restrictions on VMX operation".
11741 */
11742 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11743 return VINF_SUCCESS;
11744}
11745
11746
11747/**
11748 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11749 * VM-exit.
11750 */
11751HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11752{
11753 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11754 return VINF_EM_RESET;
11755}
11756
11757
11758/**
11759 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11760 */
11761HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11762{
11763 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11764 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_HLT_EXIT);
11765
11766 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11767 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
11768 AssertRCReturn(rc, rc);
11769
11770 if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
11771 rc = VINF_SUCCESS;
11772 else
11773 rc = VINF_EM_HALT;
11774
11775 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11776 if (rc != VINF_SUCCESS)
11777 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11778 return rc;
11779}
11780
11781
11782/**
11783 * VM-exit handler for instructions that result in a \#UD exception delivered to
11784 * the guest.
11785 */
11786HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11787{
11788 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11789 hmR0VmxSetPendingXcptUD(pVCpu);
11790 return VINF_SUCCESS;
11791}
11792
11793
11794/**
11795 * VM-exit handler for expiry of the VMX preemption timer.
11796 */
11797HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11798{
11799 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11800
11801 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11802 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11803
11804 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11805 PVM pVM = pVCpu->CTX_SUFF(pVM);
11806 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11807 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11808 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11809}
11810
11811
11812/**
11813 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11814 */
11815HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11816{
11817 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11818
11819 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11820 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_CR4);
11821 AssertRCReturn(rc, rc);
11822
11823 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11824 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11825 : HM_CHANGED_RAISED_XCPT_MASK);
11826
11827 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11828 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
11829
11830 return rcStrict;
11831}
11832
11833
11834/**
11835 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11836 */
11837HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11838{
11839 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11840 /** @todo Use VM-exit instruction information. */
11841 return VERR_EM_INTERPRETER;
11842}
11843
11844
11845/**
11846 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11847 * Error VM-exit.
11848 */
11849HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11850{
11851 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11852 AssertRCReturn(rc, rc);
11853 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11854 if (RT_FAILURE(rc))
11855 return rc;
11856
11857 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
11858 NOREF(uInvalidReason);
11859
11860#ifdef VBOX_STRICT
11861 uint32_t fIntrState;
11862 RTHCUINTREG uHCReg;
11863 uint64_t u64Val;
11864 uint32_t u32Val;
11865
11866 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11867 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11868 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11869 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
11870 AssertRCReturn(rc, rc);
11871
11872 Log4(("uInvalidReason %u\n", uInvalidReason));
11873 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11874 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11875 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11876 Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
11877
11878 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11879 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11880 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11881 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11882 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11883 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11884 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11885 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11886 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11887 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11888 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11889 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11890
11891 hmR0DumpRegs(pVCpu);
11892#else
11893 NOREF(pVmxTransient);
11894#endif
11895
11896 return VERR_VMX_INVALID_GUEST_STATE;
11897}
11898
11899
11900/**
11901 * VM-exit handler for VM-entry failure due to an MSR-load
11902 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11903 */
11904HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11905{
11906 AssertMsgFailed(("Unexpected MSR-load exit\n"));
11907 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11908}
11909
11910
11911/**
11912 * VM-exit handler for VM-entry failure due to a machine-check event
11913 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11914 */
11915HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11916{
11917 AssertMsgFailed(("Unexpected machine-check event exit\n"));
11918 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11919}
11920
11921
11922/**
11923 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11924 * theory.
11925 */
11926HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11927{
11928 RT_NOREF2(pVCpu, pVmxTransient);
11929 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d\n", pVmxTransient->uExitReason));
11930 return VERR_VMX_UNDEFINED_EXIT_CODE;
11931}
11932
11933
11934/**
11935 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11936 * (VMX_EXIT_GDTR_IDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11937 * Conditional VM-exit.
11938 */
11939HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11940{
11941 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11942
11943 /* By default, we don't enable VMX_PROC_CTLS2_DESCRIPTOR_TABLE_EXIT. */
11944 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11945 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_DESC_TABLE_EXIT)
11946 return VERR_EM_INTERPRETER;
11947 AssertMsgFailed(("Unexpected XDTR access\n"));
11948 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11949}
11950
11951
11952/**
11953 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11954 */
11955HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11956{
11957 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11958
11959 /* By default, we don't enable VMX_PROC_CTLS2_RDRAND_EXIT. */
11960 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_RDRAND_EXIT)
11961 return VERR_EM_INTERPRETER;
11962 AssertMsgFailed(("Unexpected RDRAND exit\n"));
11963 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11964}
11965
11966
11967/**
11968 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11969 */
11970HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11971{
11972 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11973
11974 /** @todo Optimize this: We currently drag in in the whole MSR state
11975 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11976 * MSRs required. That would require changes to IEM and possibly CPUM too.
11977 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11978 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; NOREF(idMsr); /* Save it. */
11979 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11980 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
11981 AssertRCReturn(rc, rc);
11982
11983 Log4Func(("ecx=%#RX32\n", idMsr));
11984
11985#ifdef VBOX_STRICT
11986 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
11987 {
11988 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr)
11989 && idMsr != MSR_K6_EFER)
11990 {
11991 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
11992 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11993 }
11994 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11995 {
11996 VMXMSREXITREAD enmRead;
11997 VMXMSREXITWRITE enmWrite;
11998 int rc2 = hmR0VmxGetMsrPermission(pVCpu, idMsr, &enmRead, &enmWrite);
11999 AssertRCReturn(rc2, rc2);
12000 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
12001 {
12002 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
12003 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12004 }
12005 }
12006 }
12007#endif
12008
12009 VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
12010 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
12011 if (rcStrict == VINF_SUCCESS)
12012 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
12013 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX);
12014 else if (rcStrict == VINF_IEM_RAISED_XCPT)
12015 {
12016 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
12017 rcStrict = VINF_SUCCESS;
12018 }
12019 else
12020 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12021
12022 return rcStrict;
12023}
12024
12025
12026/**
12027 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12028 */
12029HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12030{
12031 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12032
12033 /** @todo Optimize this: We currently drag in in the whole MSR state
12034 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
12035 * MSRs required. That would require changes to IEM and possibly CPUM too.
12036 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
12037 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; /* Save it. */
12038 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12039 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
12040 AssertRCReturn(rc, rc);
12041
12042 Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
12043
12044 VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
12045 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12046
12047 if (rcStrict == VINF_SUCCESS)
12048 {
12049 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12050
12051 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
12052 if ( idMsr == MSR_IA32_APICBASE
12053 || ( idMsr >= MSR_IA32_X2APIC_START
12054 && idMsr <= MSR_IA32_X2APIC_END))
12055 {
12056 /*
12057 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12058 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before IEM changes it.
12059 */
12060 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
12061 }
12062 else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12063 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12064 else if (idMsr == MSR_K6_EFER)
12065 {
12066 /*
12067 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12068 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12069 * the other bits as well, SCE and NXE. See @bugref{7368}.
12070 */
12071 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS
12072 | HM_CHANGED_VMX_EXIT_CTLS);
12073 }
12074
12075 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12076 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
12077 {
12078 switch (idMsr)
12079 {
12080 case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
12081 case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
12082 case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
12083 case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
12084 case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
12085 case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
12086 default:
12087 {
12088 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
12089 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12090 else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
12091 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
12092 break;
12093 }
12094 }
12095 }
12096#ifdef VBOX_STRICT
12097 else
12098 {
12099 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12100 switch (idMsr)
12101 {
12102 case MSR_IA32_SYSENTER_CS:
12103 case MSR_IA32_SYSENTER_EIP:
12104 case MSR_IA32_SYSENTER_ESP:
12105 case MSR_K8_FS_BASE:
12106 case MSR_K8_GS_BASE:
12107 {
12108 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
12109 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12110 }
12111
12112 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12113 default:
12114 {
12115 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
12116 {
12117 /* EFER writes are always intercepted, see hmR0VmxExportGuestMsrs(). */
12118 if (idMsr != MSR_K6_EFER)
12119 {
12120 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12121 idMsr));
12122 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12123 }
12124 }
12125
12126 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
12127 {
12128 VMXMSREXITREAD enmRead;
12129 VMXMSREXITWRITE enmWrite;
12130 int rc2 = hmR0VmxGetMsrPermission(pVCpu, idMsr, &enmRead, &enmWrite);
12131 AssertRCReturn(rc2, rc2);
12132 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12133 {
12134 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
12135 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12136 }
12137 }
12138 break;
12139 }
12140 }
12141 }
12142#endif /* VBOX_STRICT */
12143 }
12144 else if (rcStrict == VINF_IEM_RAISED_XCPT)
12145 {
12146 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
12147 rcStrict = VINF_SUCCESS;
12148 }
12149 else
12150 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12151
12152 return rcStrict;
12153}
12154
12155
12156/**
12157 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12158 */
12159HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12160{
12161 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12162 /** @todo The guest has likely hit a contended spinlock. We might want to
12163 * poke a schedule different guest VCPU. */
12164 return VINF_EM_RAW_INTERRUPT;
12165}
12166
12167
12168/**
12169 * VM-exit handler for when the TPR value is lowered below the specified
12170 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12171 */
12172HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12173{
12174 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12175 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
12176
12177 /*
12178 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12179 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12180 */
12181 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12182 return VINF_SUCCESS;
12183}
12184
12185
12186/**
12187 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12188 * VM-exit.
12189 *
12190 * @retval VINF_SUCCESS when guest execution can continue.
12191 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12192 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12193 * interpreter.
12194 */
12195HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12196{
12197 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12198 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12199
12200 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12201 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12202 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12203 AssertRCReturn(rc, rc);
12204
12205 VBOXSTRICTRC rcStrict;
12206 PVM pVM = pVCpu->CTX_SUFF(pVM);
12207 RTGCUINTPTR const uExitQual = pVmxTransient->uExitQual;
12208 uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
12209 switch (uAccessType)
12210 {
12211 case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
12212 {
12213 uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
12214 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
12215 VMX_EXIT_QUAL_CRX_GENREG(uExitQual));
12216 AssertMsg( rcStrict == VINF_SUCCESS
12217 || rcStrict == VINF_IEM_RAISED_XCPT
12218 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12219
12220 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
12221 {
12222 case 0:
12223 {
12224 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12225 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12226 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
12227 Log4Func(("CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
12228
12229 /*
12230 * This is a kludge for handling switches back to real mode when we try to use
12231 * V86 mode to run real mode code directly. Problem is that V86 mode cannot
12232 * deal with special selector values, so we have to return to ring-3 and run
12233 * there till the selector values are V86 mode compatible.
12234 *
12235 * Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
12236 * latter is an alias for VINF_IEM_RAISED_XCPT which is converted to VINF_SUCCESs
12237 * at the end of this function.
12238 */
12239 if ( rc == VINF_SUCCESS
12240 && !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
12241 && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
12242 && (uOldCr0 & X86_CR0_PE)
12243 && !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
12244 {
12245 /** @todo check selectors rather than returning all the time. */
12246 Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
12247 rcStrict = VINF_EM_RESCHEDULE_REM;
12248 }
12249 break;
12250 }
12251
12252 case 2:
12253 {
12254 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
12255 /* Nothing to do here, CR2 it's not part of the VMCS. */
12256 break;
12257 }
12258
12259 case 3:
12260 {
12261 Assert( !pVM->hm.s.fNestedPaging
12262 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
12263 || pVCpu->hm.s.fUsingDebugLoop);
12264 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
12265 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12266 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR3);
12267 Log4Func(("CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
12268 break;
12269 }
12270
12271 case 4:
12272 {
12273 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
12274 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12275 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR4);
12276 Log4Func(("CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
12277 pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12278 break;
12279 }
12280
12281 case 8:
12282 {
12283 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
12284 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
12285 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
12286 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_APIC_TPR);
12287 break;
12288 }
12289 default:
12290 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual)));
12291 break;
12292 }
12293 break;
12294 }
12295
12296 case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
12297 {
12298 Assert( !pVM->hm.s.fNestedPaging
12299 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
12300 || pVCpu->hm.s.fUsingDebugLoop
12301 || VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 3);
12302 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12303 Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 8
12304 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
12305
12306 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_GENREG(uExitQual),
12307 VMX_EXIT_QUAL_CRX_REGISTER(uExitQual));
12308 AssertMsg( rcStrict == VINF_SUCCESS
12309 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12310#ifdef VBOX_WITH_STATISTICS
12311 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
12312 {
12313 case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
12314 case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
12315 case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
12316 case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
12317 case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
12318 }
12319#endif
12320 Log4Func(("CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
12321 VBOXSTRICTRC_VAL(rcStrict)));
12322 if (VMX_EXIT_QUAL_CRX_GENREG(uExitQual) == X86_GREG_xSP)
12323 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_RSP);
12324 else
12325 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12326 break;
12327 }
12328
12329 case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12330 {
12331 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12332 AssertMsg( rcStrict == VINF_SUCCESS
12333 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12334
12335 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12336 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12337 Log4Func(("CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12338 break;
12339 }
12340
12341 case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12342 {
12343 /* Note! LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here. */
12344 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual));
12345 AssertMsg( rcStrict == VINF_SUCCESS
12346 || rcStrict == VINF_IEM_RAISED_XCPT
12347 , ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12348
12349 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12350 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12351 Log4Func(("LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12352 break;
12353 }
12354
12355 default:
12356 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12357 VERR_VMX_UNEXPECTED_EXCEPTION);
12358 }
12359
12360 Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS))
12361 == (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS));
12362 if (rcStrict == VINF_IEM_RAISED_XCPT)
12363 {
12364 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
12365 rcStrict = VINF_SUCCESS;
12366 }
12367
12368 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12369 NOREF(pVM);
12370 return rcStrict;
12371}
12372
12373
12374/**
12375 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12376 * VM-exit.
12377 */
12378HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12379{
12380 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12381 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12382
12383 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12384 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12385 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12386 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_EFER);
12387 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12388 AssertRCReturn(rc, rc);
12389
12390 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12391 uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
12392 uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQual);
12393 bool fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12394 bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
12395 bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
12396 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12397 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12398
12399 /*
12400 * Update exit history to see if this exit can be optimized.
12401 */
12402 VBOXSTRICTRC rcStrict;
12403 PCEMEXITREC pExitRec = NULL;
12404 if ( !fGstStepping
12405 && !fDbgStepping)
12406 pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12407 !fIOString
12408 ? !fIOWrite
12409 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12410 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12411 : !fIOWrite
12412 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12413 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12414 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12415 if (!pExitRec)
12416 {
12417 /* I/O operation lookup arrays. */
12418 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12419 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12420 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12421 uint32_t const cbInstr = pVmxTransient->cbInstr;
12422 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12423 PVM pVM = pVCpu->CTX_SUFF(pVM);
12424 if (fIOString)
12425 {
12426 /*
12427 * INS/OUTS - I/O String instruction.
12428 *
12429 * Use instruction-information if available, otherwise fall back on
12430 * interpreting the instruction.
12431 */
12432 Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12433 AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
12434 bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
12435 if (fInsOutsInfo)
12436 {
12437 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12438 AssertRCReturn(rc2, rc2);
12439 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12440 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12441 IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12442 bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
12443 if (fIOWrite)
12444 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12445 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12446 else
12447 {
12448 /*
12449 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12450 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12451 * See Intel Instruction spec. for "INS".
12452 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12453 */
12454 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12455 }
12456 }
12457 else
12458 rcStrict = IEMExecOne(pVCpu);
12459
12460 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12461 fUpdateRipAlready = true;
12462 }
12463 else
12464 {
12465 /*
12466 * IN/OUT - I/O instruction.
12467 */
12468 Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12469 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12470 Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
12471 if (fIOWrite)
12472 {
12473 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
12474 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12475 if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
12476 && !pCtx->eflags.Bits.u1TF)
12477 rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
12478 }
12479 else
12480 {
12481 uint32_t u32Result = 0;
12482 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12483 if (IOM_SUCCESS(rcStrict))
12484 {
12485 /* Save result of I/O IN instr. in AL/AX/EAX. */
12486 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12487 }
12488 if ( rcStrict == VINF_IOM_R3_IOPORT_READ
12489 && !pCtx->eflags.Bits.u1TF)
12490 rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
12491 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12492 }
12493 }
12494
12495 if (IOM_SUCCESS(rcStrict))
12496 {
12497 if (!fUpdateRipAlready)
12498 {
12499 hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
12500 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12501 }
12502
12503 /*
12504 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12505 * while booting Fedora 17 64-bit guest.
12506 *
12507 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12508 */
12509 if (fIOString)
12510 {
12511 /** @todo Single-step for INS/OUTS with REP prefix? */
12512 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12513 }
12514 else if ( !fDbgStepping
12515 && fGstStepping)
12516 {
12517 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12518 AssertRCReturn(rc, rc);
12519 }
12520
12521 /*
12522 * If any I/O breakpoints are armed, we need to check if one triggered
12523 * and take appropriate action.
12524 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12525 */
12526 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12527 AssertRCReturn(rc, rc);
12528
12529 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12530 * execution engines about whether hyper BPs and such are pending. */
12531 uint32_t const uDr7 = pCtx->dr[7];
12532 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12533 && X86_DR7_ANY_RW_IO(uDr7)
12534 && (pCtx->cr4 & X86_CR4_DE))
12535 || DBGFBpIsHwIoArmed(pVM)))
12536 {
12537 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12538
12539 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12540 VMMRZCallRing3Disable(pVCpu);
12541 HM_DISABLE_PREEMPT(pVCpu);
12542
12543 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12544
12545 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
12546 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12547 {
12548 /* Raise #DB. */
12549 if (fIsGuestDbgActive)
12550 ASMSetDR6(pCtx->dr[6]);
12551 if (pCtx->dr[7] != uDr7)
12552 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_DR7;
12553
12554 hmR0VmxSetPendingXcptDB(pVCpu);
12555 }
12556 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12557 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12558 else if ( rcStrict2 != VINF_SUCCESS
12559 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12560 rcStrict = rcStrict2;
12561 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12562
12563 HM_RESTORE_PREEMPT();
12564 VMMRZCallRing3Enable(pVCpu);
12565 }
12566 }
12567
12568#ifdef VBOX_STRICT
12569 if ( rcStrict == VINF_IOM_R3_IOPORT_READ
12570 || rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
12571 Assert(!fIOWrite);
12572 else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
12573 || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
12574 || rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
12575 Assert(fIOWrite);
12576 else
12577 {
12578# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12579 * statuses, that the VMM device and some others may return. See
12580 * IOM_SUCCESS() for guidance. */
12581 AssertMsg( RT_FAILURE(rcStrict)
12582 || rcStrict == VINF_SUCCESS
12583 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12584 || rcStrict == VINF_EM_DBG_BREAKPOINT
12585 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12586 || rcStrict == VINF_EM_RAW_TO_R3
12587 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12588# endif
12589 }
12590#endif
12591 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12592 }
12593 else
12594 {
12595 /*
12596 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12597 */
12598 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12599 AssertRCReturn(rc2, rc2);
12600 STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
12601 : fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
12602 Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12603 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12604 VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
12605 fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
12606
12607 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12608 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12609
12610 Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12611 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12612 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12613 }
12614 return rcStrict;
12615}
12616
12617
12618/**
12619 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12620 * VM-exit.
12621 */
12622HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12623{
12624 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12625
12626 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12627 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12628 AssertRCReturn(rc, rc);
12629 if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12630 {
12631 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12632 AssertRCReturn(rc, rc);
12633 if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
12634 {
12635 uint32_t uErrCode;
12636 RTGCUINTPTR GCPtrFaultAddress;
12637 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12638 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12639 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
12640 if (fErrorCodeValid)
12641 {
12642 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12643 AssertRCReturn(rc, rc);
12644 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12645 }
12646 else
12647 uErrCode = 0;
12648
12649 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12650 && uVector == X86_XCPT_PF)
12651 GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
12652 else
12653 GCPtrFaultAddress = 0;
12654
12655 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12656 AssertRCReturn(rc, rc);
12657
12658 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12659 pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
12660
12661 Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
12662 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12663 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12664 }
12665 }
12666
12667 /* Fall back to the interpreter to emulate the task-switch. */
12668 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12669 return VERR_EM_INTERPRETER;
12670}
12671
12672
12673/**
12674 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12675 */
12676HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12677{
12678 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12679 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_MONITOR_TRAP_FLAG);
12680 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
12681 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12682 AssertRCReturn(rc, rc);
12683 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12684 return VINF_EM_DBG_STEPPED;
12685}
12686
12687
12688/**
12689 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12690 */
12691HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12692{
12693 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12694
12695 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12696
12697 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12698 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12699 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12700 {
12701 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12702 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12703 {
12704 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12705 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12706 }
12707 }
12708 else
12709 {
12710 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12711 rcStrict1 = VINF_SUCCESS;
12712 return rcStrict1;
12713 }
12714
12715 /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12716 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12717 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12718 AssertRCReturn(rc, rc);
12719
12720 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12721 uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
12722 VBOXSTRICTRC rcStrict2;
12723 switch (uAccessType)
12724 {
12725 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12726 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12727 {
12728 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
12729 || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
12730 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12731
12732 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12733 GCPhys &= PAGE_BASE_GC_MASK;
12734 GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
12735 PVM pVM = pVCpu->CTX_SUFF(pVM);
12736 Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12737 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
12738
12739 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12740 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12741 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12742 CPUMCTX2CORE(pCtx), GCPhys);
12743 Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12744 if ( rcStrict2 == VINF_SUCCESS
12745 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12746 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12747 {
12748 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12749 | HM_CHANGED_GUEST_APIC_TPR);
12750 rcStrict2 = VINF_SUCCESS;
12751 }
12752 break;
12753 }
12754
12755 default:
12756 Log4Func(("uAccessType=%#x\n", uAccessType));
12757 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12758 break;
12759 }
12760
12761 if (rcStrict2 != VINF_SUCCESS)
12762 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12763 return rcStrict2;
12764}
12765
12766
12767/**
12768 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12769 * VM-exit.
12770 */
12771HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12772{
12773 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12774
12775 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12776 if (pVmxTransient->fWasGuestDebugStateActive)
12777 {
12778 AssertMsgFailed(("Unexpected MOV DRx exit\n"));
12779 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12780 }
12781
12782 if ( !pVCpu->hm.s.fSingleInstruction
12783 && !pVmxTransient->fWasHyperDebugStateActive)
12784 {
12785 Assert(!DBGFIsStepping(pVCpu));
12786 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12787
12788 /* Don't intercept MOV DRx any more. */
12789 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
12790 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12791 AssertRCReturn(rc, rc);
12792
12793 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12794 VMMRZCallRing3Disable(pVCpu);
12795 HM_DISABLE_PREEMPT(pVCpu);
12796
12797 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12798 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12799 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12800
12801 HM_RESTORE_PREEMPT();
12802 VMMRZCallRing3Enable(pVCpu);
12803
12804#ifdef VBOX_WITH_STATISTICS
12805 rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12806 AssertRCReturn(rc, rc);
12807 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12808 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12809 else
12810 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12811#endif
12812 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12813 return VINF_SUCCESS;
12814 }
12815
12816 /*
12817 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12818 * Update the segment registers and DR7 from the CPU.
12819 */
12820 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12821 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12822 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_DR7);
12823 AssertRCReturn(rc, rc);
12824 Log4Func(("CS:RIP=%04x:%08RX64\n", pCtx->cs.Sel, pCtx->rip));
12825
12826 PVM pVM = pVCpu->CTX_SUFF(pVM);
12827 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12828 {
12829 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12830 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
12831 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
12832 if (RT_SUCCESS(rc))
12833 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12834 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12835 }
12836 else
12837 {
12838 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12839 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
12840 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
12841 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12842 }
12843
12844 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12845 if (RT_SUCCESS(rc))
12846 {
12847 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12848 AssertRCReturn(rc2, rc2);
12849 return VINF_SUCCESS;
12850 }
12851 return rc;
12852}
12853
12854
12855/**
12856 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12857 * Conditional VM-exit.
12858 */
12859HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12860{
12861 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12862 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12863
12864 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12865 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12866 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12867 {
12868 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12869 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12870 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12871 {
12872 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12873 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12874 }
12875 }
12876 else
12877 {
12878 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12879 rcStrict1 = VINF_SUCCESS;
12880 return rcStrict1;
12881 }
12882
12883 /*
12884 * Get sufficent state and update the exit history entry.
12885 */
12886 RTGCPHYS GCPhys;
12887 int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
12888 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12889 AssertRCReturn(rc, rc);
12890
12891 VBOXSTRICTRC rcStrict;
12892 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12893 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
12894 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12895 if (!pExitRec)
12896 {
12897 /*
12898 * If we succeed, resume guest execution.
12899 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12900 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12901 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12902 * weird case. See @bugref{6043}.
12903 */
12904 PVM pVM = pVCpu->CTX_SUFF(pVM);
12905 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12906 rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
12907 Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12908 if ( rcStrict == VINF_SUCCESS
12909 || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12910 || rcStrict == VERR_PAGE_NOT_PRESENT)
12911 {
12912 /* Successfully handled MMIO operation. */
12913 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12914 | HM_CHANGED_GUEST_APIC_TPR);
12915 rcStrict = VINF_SUCCESS;
12916 }
12917 }
12918 else
12919 {
12920 /*
12921 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12922 */
12923 int rc2 = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12924 AssertRCReturn(rc2, rc2);
12925
12926 Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12927 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12928
12929 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12930 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12931
12932 Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12933 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12934 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12935 }
12936 return VBOXSTRICTRC_TODO(rcStrict);
12937}
12938
12939
12940/**
12941 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12942 * VM-exit.
12943 */
12944HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12945{
12946 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12947 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12948
12949 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12950 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12951 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12952 {
12953 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12954 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12955 Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12956 }
12957 else
12958 {
12959 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12960 rcStrict1 = VINF_SUCCESS;
12961 return rcStrict1;
12962 }
12963
12964 RTGCPHYS GCPhys;
12965 int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
12966 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
12967 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12968 AssertRCReturn(rc, rc);
12969
12970 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12971 AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
12972
12973 RTGCUINT uErrorCode = 0;
12974 if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
12975 uErrorCode |= X86_TRAP_PF_ID;
12976 if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
12977 uErrorCode |= X86_TRAP_PF_RW;
12978 if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
12979 uErrorCode |= X86_TRAP_PF_P;
12980
12981 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12982
12983
12984 /* Handle the pagefault trap for the nested shadow table. */
12985 PVM pVM = pVCpu->CTX_SUFF(pVM);
12986 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12987
12988 Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
12989 pCtx->cs.Sel, pCtx->rip));
12990
12991 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
12992 TRPMResetTrap(pVCpu);
12993
12994 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12995 if ( rcStrict2 == VINF_SUCCESS
12996 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12997 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12998 {
12999 /* Successfully synced our nested page tables. */
13000 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
13001 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
13002 return VINF_SUCCESS;
13003 }
13004
13005 Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
13006 return rcStrict2;
13007}
13008
13009/** @} */
13010
13011/** @name VM-exit exception handlers.
13012 * @{
13013 */
13014/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13015/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
13016/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13017
13018/**
13019 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
13020 */
13021static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13022{
13023 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13024 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13025
13026 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
13027 AssertRCReturn(rc, rc);
13028
13029 if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
13030 {
13031 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13032 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13033
13034 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13035 * provides VM-exit instruction length. If this causes problem later,
13036 * disassemble the instruction like it's done on AMD-V. */
13037 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13038 AssertRCReturn(rc2, rc2);
13039 return rc;
13040 }
13041
13042 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13043 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13044 return rc;
13045}
13046
13047
13048/**
13049 * VM-exit exception handler for \#BP (Breakpoint exception).
13050 */
13051static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13052{
13053 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13054 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13055
13056 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13057 AssertRCReturn(rc, rc);
13058
13059 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13060 rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13061 if (rc == VINF_EM_RAW_GUEST_TRAP)
13062 {
13063 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13064 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13065 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13066 AssertRCReturn(rc, rc);
13067
13068 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13069 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13070 }
13071
13072 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
13073 return rc;
13074}
13075
13076
13077/**
13078 * VM-exit exception handler for \#AC (alignment check exception).
13079 */
13080static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13081{
13082 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13083
13084 /*
13085 * Re-inject it. We'll detect any nesting before getting here.
13086 */
13087 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13088 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13089 AssertRCReturn(rc, rc);
13090 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13091
13092 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13093 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13094 return VINF_SUCCESS;
13095}
13096
13097
13098/**
13099 * VM-exit exception handler for \#DB (Debug exception).
13100 */
13101static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13102{
13103 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13104 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13105
13106 /*
13107 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13108 * for processing.
13109 */
13110 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13111
13112 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13113 uint64_t uDR6 = X86_DR6_INIT_VAL;
13114 uDR6 |= (pVmxTransient->uExitQual & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
13115
13116 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13117 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13118 Log6Func(("rc=%Rrc\n", rc));
13119 if (rc == VINF_EM_RAW_GUEST_TRAP)
13120 {
13121 /*
13122 * The exception was for the guest. Update DR6, DR7.GD and
13123 * IA32_DEBUGCTL.LBR before forwarding it.
13124 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13125 */
13126 VMMRZCallRing3Disable(pVCpu);
13127 HM_DISABLE_PREEMPT(pVCpu);
13128
13129 pCtx->dr[6] &= ~X86_DR6_B_MASK;
13130 pCtx->dr[6] |= uDR6;
13131 if (CPUMIsGuestDebugStateActive(pVCpu))
13132 ASMSetDR6(pCtx->dr[6]);
13133
13134 HM_RESTORE_PREEMPT();
13135 VMMRZCallRing3Enable(pVCpu);
13136
13137 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
13138 AssertRCReturn(rc, rc);
13139
13140 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13141 pCtx->dr[7] &= ~X86_DR7_GD;
13142
13143 /* Paranoia. */
13144 pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13145 pCtx->dr[7] |= X86_DR7_RA1_MASK;
13146
13147 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
13148 AssertRCReturn(rc, rc);
13149
13150 /*
13151 * Raise #DB in the guest.
13152 *
13153 * It is important to reflect exactly what the VM-exit gave us (preserving the
13154 * interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
13155 * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
13156 * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
13157 *
13158 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
13159 * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
13160 */
13161 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13162 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13163 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13164 AssertRCReturn(rc, rc);
13165 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13166 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13167 return VINF_SUCCESS;
13168 }
13169
13170 /*
13171 * Not a guest trap, must be a hypervisor related debug event then.
13172 * Update DR6 in case someone is interested in it.
13173 */
13174 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13175 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13176 CPUMSetHyperDR6(pVCpu, uDR6);
13177
13178 return rc;
13179}
13180
13181
13182/**
13183 * VM-exit exception handler for \#GP (General-protection exception).
13184 *
13185 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13186 */
13187static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13188{
13189 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13190 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13191
13192 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13193 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13194 { /* likely */ }
13195 else
13196 {
13197#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13198 Assert(pVCpu->hm.s.fUsingDebugLoop);
13199#endif
13200 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13201 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13202 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13203 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13204 rc |= hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13205 AssertRCReturn(rc, rc);
13206 Log4Func(("Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pCtx->cs.Sel, pCtx->rip,
13207 pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
13208 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13209 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13210 return rc;
13211 }
13212
13213 Assert(CPUMIsGuestInRealModeEx(pCtx));
13214 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13215
13216 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13217 AssertRCReturn(rc, rc);
13218
13219 VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
13220 if (rcStrict == VINF_SUCCESS)
13221 {
13222 if (!CPUMIsGuestInRealModeEx(pCtx))
13223 {
13224 /*
13225 * The guest is no longer in real-mode, check if we can continue executing the
13226 * guest using hardware-assisted VMX. Otherwise, fall back to emulation.
13227 */
13228 if (HMVmxCanExecuteGuest(pVCpu, pCtx))
13229 {
13230 Log4Func(("Mode changed but guest still suitable for executing using VT-x\n"));
13231 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
13232 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13233 }
13234 else
13235 {
13236 Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
13237 rcStrict = VINF_EM_RESCHEDULE;
13238 }
13239 }
13240 else
13241 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13242 }
13243 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13244 {
13245 rcStrict = VINF_SUCCESS;
13246 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13247 }
13248 return VBOXSTRICTRC_VAL(rcStrict);
13249}
13250
13251
13252/**
13253 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13254 * the exception reported in the VMX transient structure back into the VM.
13255 *
13256 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13257 * up-to-date.
13258 */
13259static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13260{
13261 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13262#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13263 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13264 ("uVector=%#x u32XcptBitmap=%#X32\n",
13265 VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13266#endif
13267
13268 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13269 hmR0VmxCheckExitDueToEventDelivery(). */
13270 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13271 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13272 AssertRCReturn(rc, rc);
13273 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13274
13275#ifdef DEBUG_ramshankar
13276 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
13277 uint8_t uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13278 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13279#endif
13280
13281 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
13282 pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13283 return VINF_SUCCESS;
13284}
13285
13286
13287/**
13288 * VM-exit exception handler for \#PF (Page-fault exception).
13289 */
13290static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13291{
13292 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13293 PVM pVM = pVCpu->CTX_SUFF(pVM);
13294 int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13295 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13296 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13297 AssertRCReturn(rc, rc);
13298
13299 if (!pVM->hm.s.fNestedPaging)
13300 { /* likely */ }
13301 else
13302 {
13303#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13304 Assert(pVCpu->hm.s.fUsingDebugLoop);
13305#endif
13306 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13307 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13308 {
13309 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13310 pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
13311 }
13312 else
13313 {
13314 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13315 hmR0VmxSetPendingXcptDF(pVCpu);
13316 Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13317 }
13318 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13319 return rc;
13320 }
13321
13322 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13323 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13324 if (pVmxTransient->fVectoringPF)
13325 {
13326 Assert(pVCpu->hm.s.Event.fPending);
13327 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13328 }
13329
13330 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13331 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13332 AssertRCReturn(rc, rc);
13333
13334 Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
13335 pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
13336
13337 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13338 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
13339
13340 Log4Func(("#PF: rc=%Rrc\n", rc));
13341 if (rc == VINF_SUCCESS)
13342 {
13343 /*
13344 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13345 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13346 */
13347 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13348 TRPMResetTrap(pVCpu);
13349 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13350 return rc;
13351 }
13352
13353 if (rc == VINF_EM_RAW_GUEST_TRAP)
13354 {
13355 if (!pVmxTransient->fVectoringDoublePF)
13356 {
13357 /* It's a guest page fault and needs to be reflected to the guest. */
13358 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13359 TRPMResetTrap(pVCpu);
13360 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13361 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
13362 uGstErrorCode, pVmxTransient->uExitQual);
13363 }
13364 else
13365 {
13366 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13367 TRPMResetTrap(pVCpu);
13368 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13369 hmR0VmxSetPendingXcptDF(pVCpu);
13370 Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13371 }
13372
13373 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13374 return VINF_SUCCESS;
13375 }
13376
13377 TRPMResetTrap(pVCpu);
13378 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13379 return rc;
13380}
13381
13382/** @} */
13383
13384#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13385/** @name Nested-guest VM-exit handlers.
13386 * @{
13387 */
13388/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13389/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= Nested-guest VM-exit handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13390/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13391
13392/**
13393 * VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
13394 */
13395HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13396{
13397 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13398
13399 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13400 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13401 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13402 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13403 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13404 AssertRCReturn(rc, rc);
13405
13406 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13407
13408 VMXVEXITINFO ExitInfo;
13409 RT_ZERO(ExitInfo);
13410 ExitInfo.uReason = pVmxTransient->uExitReason;
13411 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13412 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13413 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13414 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13415
13416 VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
13417 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13418 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13419 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13420 {
13421 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13422 rcStrict = VINF_SUCCESS;
13423 }
13424 return rcStrict;
13425}
13426
13427
13428/**
13429 * VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
13430 */
13431HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13432{
13433 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13434
13435 /** @todo NSTVMX: Vmlaunch. */
13436 hmR0VmxSetPendingXcptUD(pVCpu);
13437 return VINF_SUCCESS;
13438}
13439
13440
13441/**
13442 * VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
13443 */
13444HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13445{
13446 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13447
13448 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13449 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13450 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13451 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13452 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13453 AssertRCReturn(rc, rc);
13454
13455 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13456
13457 VMXVEXITINFO ExitInfo;
13458 RT_ZERO(ExitInfo);
13459 ExitInfo.uReason = pVmxTransient->uExitReason;
13460 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13461 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13462 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13463 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13464
13465 VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
13466 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13467 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13468 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13469 {
13470 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13471 rcStrict = VINF_SUCCESS;
13472 }
13473 return rcStrict;
13474}
13475
13476
13477/**
13478 * VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
13479 */
13480HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13481{
13482 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13483
13484 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13485 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13486 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13487 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13488 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13489 AssertRCReturn(rc, rc);
13490
13491 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13492
13493 VMXVEXITINFO ExitInfo;
13494 RT_ZERO(ExitInfo);
13495 ExitInfo.uReason = pVmxTransient->uExitReason;
13496 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13497 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13498 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13499 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
13500
13501 VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
13502 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13503 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13504 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13505 {
13506 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13507 rcStrict = VINF_SUCCESS;
13508 }
13509 return rcStrict;
13510}
13511
13512
13513/**
13514 * VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Unconditional VM-exit.
13515 */
13516HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13517{
13518 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13519
13520 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13521 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13522 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13523 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13524 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13525 AssertRCReturn(rc, rc);
13526
13527 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13528
13529 VMXVEXITINFO ExitInfo;
13530 RT_ZERO(ExitInfo);
13531 ExitInfo.uReason = pVmxTransient->uExitReason;
13532 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13533 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13534 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13535 if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
13536 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
13537
13538 VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
13539 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13540 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13541 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13542 {
13543 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13544 rcStrict = VINF_SUCCESS;
13545 }
13546 return rcStrict;
13547}
13548
13549
13550/**
13551 * VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
13552 */
13553HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13554{
13555 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13556
13557 /** @todo NSTVMX: Vmresume. */
13558 hmR0VmxSetPendingXcptUD(pVCpu);
13559 return VINF_SUCCESS;
13560}
13561
13562
13563/**
13564 * VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Unconditional VM-exit.
13565 */
13566HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13567{
13568 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13569
13570 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13571 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13572 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13573 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13574 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13575 AssertRCReturn(rc, rc);
13576
13577 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13578
13579 VMXVEXITINFO ExitInfo;
13580 RT_ZERO(ExitInfo);
13581 ExitInfo.uReason = pVmxTransient->uExitReason;
13582 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13583 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13584 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13585 if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
13586 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13587
13588 VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
13589 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13590 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13591 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13592 {
13593 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13594 rcStrict = VINF_SUCCESS;
13595 }
13596 return rcStrict;
13597}
13598
13599
13600/**
13601 * VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
13602 */
13603HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13604{
13605 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13606
13607 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13608 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13609 AssertRCReturn(rc, rc);
13610
13611 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13612
13613 VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
13614 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13615 {
13616 /* VMXOFF on success changes the internal hwvirt state but not anything that's visible to the guest. */
13617 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
13618 }
13619 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13620 {
13621 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13622 rcStrict = VINF_SUCCESS;
13623 }
13624 return rcStrict;
13625}
13626
13627
13628/**
13629 * VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
13630 */
13631HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13632{
13633 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13634
13635 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13636 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
13637 | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13638 rc |= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
13639 rc |= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13640 AssertRCReturn(rc, rc);
13641
13642 HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
13643
13644 VMXVEXITINFO ExitInfo;
13645 RT_ZERO(ExitInfo);
13646 ExitInfo.uReason = pVmxTransient->uExitReason;
13647 ExitInfo.u64Qual = pVmxTransient->uExitQual;
13648 ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
13649 ExitInfo.cbInstr = pVmxTransient->cbInstr;
13650 HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
13651
13652 VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
13653 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13654 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
13655 else if (rcStrict == VINF_IEM_RAISED_XCPT)
13656 {
13657 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13658 rcStrict = VINF_SUCCESS;
13659 }
13660 return rcStrict;
13661}
13662
13663/** @} */
13664#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13665
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette