VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 48227

Last change on this file since 48227 was 48227, checked in by vboxsync, 12 years ago

VMM: Ensure longjmps after hmR0[Vmx|Svm]LeaveSession() don't happen.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 443.5 KB
Line 
1/* $Id: HMVMXR0.cpp 48227 2013-09-02 11:54:23Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2013 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/asm-amd64-x86.h>
23#include <iprt/thread.h>
24#include <iprt/string.h>
25
26#include "HMInternal.h"
27#include <VBox/vmm/vm.h>
28#include "HMVMXR0.h"
29#include <VBox/vmm/pdmapi.h>
30#include <VBox/vmm/dbgf.h>
31#include <VBox/vmm/iem.h>
32#include <VBox/vmm/iom.h>
33#include <VBox/vmm/selm.h>
34#include <VBox/vmm/tm.h>
35#ifdef VBOX_WITH_REM
36# include <VBox/vmm/rem.h>
37#endif
38#ifdef DEBUG_ramshankar
39#define HMVMX_SAVE_FULL_GUEST_STATE
40#define HMVMX_SYNC_FULL_GUEST_STATE
41#define HMVMX_ALWAYS_CHECK_GUEST_STATE
42#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
43#define HMVMX_ALWAYS_TRAP_PF
44#endif
45
46
47/*******************************************************************************
48* Defined Constants And Macros *
49*******************************************************************************/
50#if defined(RT_ARCH_AMD64)
51# define HMVMX_IS_64BIT_HOST_MODE() (true)
52typedef RTHCUINTREG HMVMXHCUINTREG;
53#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
54extern "C" uint32_t g_fVMXIs64bitHost;
55# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
56typedef uint64_t HMVMXHCUINTREG;
57#else
58# define HMVMX_IS_64BIT_HOST_MODE() (false)
59typedef RTHCUINTREG HMVMXHCUINTREG;
60#endif
61
62/** Use the function table. */
63#define HMVMX_USE_FUNCTION_TABLE
64
65/** Determine which tagged-TLB flush handler to use. */
66#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
67#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
68#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
69#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
70
71/** @name Updated-guest-state flags.
72 * @{ */
73#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
74#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
75#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
76#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
77#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
78#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
79#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
80#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
81#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
82#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
83#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
84#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
85#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
86#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
87#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
88#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
89#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
90#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
91#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
92#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
93#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
94 | HMVMX_UPDATED_GUEST_RSP \
95 | HMVMX_UPDATED_GUEST_RFLAGS \
96 | HMVMX_UPDATED_GUEST_CR0 \
97 | HMVMX_UPDATED_GUEST_CR3 \
98 | HMVMX_UPDATED_GUEST_CR4 \
99 | HMVMX_UPDATED_GUEST_GDTR \
100 | HMVMX_UPDATED_GUEST_IDTR \
101 | HMVMX_UPDATED_GUEST_LDTR \
102 | HMVMX_UPDATED_GUEST_TR \
103 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
104 | HMVMX_UPDATED_GUEST_DEBUG \
105 | HMVMX_UPDATED_GUEST_FS_BASE_MSR \
106 | HMVMX_UPDATED_GUEST_GS_BASE_MSR \
107 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
112 | HMVMX_UPDATED_GUEST_APIC_STATE)
113/** @} */
114
115/** @name
116 * Flags to skip redundant reads of some common VMCS fields that are not part of
117 * the guest-CPU state but are in the transient structure.
118 */
119#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
120#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
123#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
124#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
125/** @} */
126
127/** @name
128 * States of the VMCS.
129 *
130 * This does not reflect all possible VMCS states but currently only those
131 * needed for maintaining the VMCS consistently even when thread-context hooks
132 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
133 */
134#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
135#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
136#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
137/** @} */
138
139/**
140 * Exception bitmap mask for real-mode guests (real-on-v86).
141 *
142 * We need to intercept all exceptions manually (except #PF). #NM is also
143 * handled separately, see hmR0VmxLoadGuestControlRegs(). #PF need not be
144 * intercepted even in real-mode if we have Nested Paging support.
145 */
146#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
147 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
148 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
149 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
150 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
151 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
152 | RT_BIT(X86_XCPT_XF))
153
154/**
155 * Exception bitmap mask for all contributory exceptions.
156 *
157 * Page fault is deliberately excluded here as it's conditional as to whether
158 * it's contributory or benign. Page faults are handled separately.
159 */
160#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
161 | RT_BIT(X86_XCPT_DE))
162
163/** Maximum VM-instruction error number. */
164#define HMVMX_INSTR_ERROR_MAX 28
165
166/** Profiling macro. */
167#ifdef HM_PROFILE_EXIT_DISPATCH
168# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
169# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
170#else
171# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
172# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
173#endif
174
175/** Assert that preemption is disabled or covered by thread-context hooks. */
176#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
177 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
178
179/** Assert that we haven't migrated CPUs when thread-context hooks are not
180 * used. */
181#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
182 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
183 ("Illegal migration! Entered on CPU %u Current %u\n", \
184 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
185
186/*******************************************************************************
187* Structures and Typedefs *
188*******************************************************************************/
189/**
190 * VMX transient state.
191 *
192 * A state structure for holding miscellaneous information across
193 * VMX non-root operation and restored after the transition.
194 */
195typedef struct VMXTRANSIENT
196{
197 /** The host's rflags/eflags. */
198 RTCCUINTREG uEflags;
199#if HC_ARCH_BITS == 32
200 uint32_t u32Alignment0;
201#endif
202 /** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
203 uint64_t u64LStarMsr;
204 /** The guest's TPR value used for TPR shadowing. */
205 uint8_t u8GuestTpr;
206 /** Alignment. */
207 uint8_t abAlignment0[7];
208
209 /** The basic VM-exit reason. */
210 uint16_t uExitReason;
211 /** Alignment. */
212 uint16_t u16Alignment0;
213 /** The VM-exit interruption error code. */
214 uint32_t uExitIntrErrorCode;
215 /** The VM-exit exit qualification. */
216 uint64_t uExitQualification;
217
218 /** The VM-exit interruption-information field. */
219 uint32_t uExitIntrInfo;
220 /** The VM-exit instruction-length field. */
221 uint32_t cbInstr;
222 /** The VM-exit instruction-information field. */
223 union
224 {
225 /** Plain unsigned int representation. */
226 uint32_t u;
227 /** INS and OUTS information. */
228 struct
229 {
230 uint32_t u6Reserved0 : 6;
231 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
232 uint32_t u3AddrSize : 3;
233 uint32_t u5Reserved1 : 5;
234 /** The segment register (X86_SREG_XXX). */
235 uint32_t iSegReg : 3;
236 uint32_t uReserved2 : 14;
237 } StrIo;
238 } ExitInstrInfo;
239 /** Whether the VM-entry failed or not. */
240 bool fVMEntryFailed;
241 /** Alignment. */
242 uint8_t abAlignment1[3];
243
244 /** The VM-entry interruption-information field. */
245 uint32_t uEntryIntrInfo;
246 /** The VM-entry exception error code field. */
247 uint32_t uEntryXcptErrorCode;
248 /** The VM-entry instruction length field. */
249 uint32_t cbEntryInstr;
250
251 /** IDT-vectoring information field. */
252 uint32_t uIdtVectoringInfo;
253 /** IDT-vectoring error code. */
254 uint32_t uIdtVectoringErrorCode;
255
256 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
257 uint32_t fVmcsFieldsRead;
258 /** Whether TSC-offsetting should be setup before VM-entry. */
259 bool fUpdateTscOffsettingAndPreemptTimer;
260 /** Whether the VM-exit was caused by a page-fault during delivery of a
261 * contributory exception or a page-fault. */
262 bool fVectoringPF;
263} VMXTRANSIENT;
264AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
265AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
266AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
267AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
268/** Pointer to VMX transient state. */
269typedef VMXTRANSIENT *PVMXTRANSIENT;
270
271
272/**
273 * MSR-bitmap read permissions.
274 */
275typedef enum VMXMSREXITREAD
276{
277 /** Reading this MSR causes a VM-exit. */
278 VMXMSREXIT_INTERCEPT_READ = 0xb,
279 /** Reading this MSR does not cause a VM-exit. */
280 VMXMSREXIT_PASSTHRU_READ
281} VMXMSREXITREAD;
282
283/**
284 * MSR-bitmap write permissions.
285 */
286typedef enum VMXMSREXITWRITE
287{
288 /** Writing to this MSR causes a VM-exit. */
289 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
290 /** Writing to this MSR does not cause a VM-exit. */
291 VMXMSREXIT_PASSTHRU_WRITE
292} VMXMSREXITWRITE;
293
294/**
295 * VM-exit handler.
296 *
297 * @returns VBox status code.
298 * @param pVCpu Pointer to the VMCPU.
299 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
300 * out-of-sync. Make sure to update the required
301 * fields before using them.
302 * @param pVmxTransient Pointer to the VMX-transient structure.
303 */
304#ifndef HMVMX_USE_FUNCTION_TABLE
305typedef int FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
306#else
307typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
308/** Pointer to VM-exit handler. */
309typedef FNVMEXITHANDLER *PFNVMEXITHANDLER;
310#endif
311
312
313/*******************************************************************************
314* Internal Functions *
315*******************************************************************************/
316static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
317static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
318static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
319 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
320#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
321static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
322#endif
323#ifndef HMVMX_USE_FUNCTION_TABLE
324DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
325# define HMVMX_EXIT_DECL static int
326#else
327# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
328#endif
329
330/** @name VM-exit handlers.
331 * @{
332 */
333static FNVMEXITHANDLER hmR0VmxExitXcptOrNmi;
334static FNVMEXITHANDLER hmR0VmxExitExtInt;
335static FNVMEXITHANDLER hmR0VmxExitTripleFault;
336static FNVMEXITHANDLER hmR0VmxExitInitSignal;
337static FNVMEXITHANDLER hmR0VmxExitSipi;
338static FNVMEXITHANDLER hmR0VmxExitIoSmi;
339static FNVMEXITHANDLER hmR0VmxExitSmi;
340static FNVMEXITHANDLER hmR0VmxExitIntWindow;
341static FNVMEXITHANDLER hmR0VmxExitNmiWindow;
342static FNVMEXITHANDLER hmR0VmxExitTaskSwitch;
343static FNVMEXITHANDLER hmR0VmxExitCpuid;
344static FNVMEXITHANDLER hmR0VmxExitGetsec;
345static FNVMEXITHANDLER hmR0VmxExitHlt;
346static FNVMEXITHANDLER hmR0VmxExitInvd;
347static FNVMEXITHANDLER hmR0VmxExitInvlpg;
348static FNVMEXITHANDLER hmR0VmxExitRdpmc;
349static FNVMEXITHANDLER hmR0VmxExitRdtsc;
350static FNVMEXITHANDLER hmR0VmxExitRsm;
351static FNVMEXITHANDLER hmR0VmxExitSetPendingXcptUD;
352static FNVMEXITHANDLER hmR0VmxExitMovCRx;
353static FNVMEXITHANDLER hmR0VmxExitMovDRx;
354static FNVMEXITHANDLER hmR0VmxExitIoInstr;
355static FNVMEXITHANDLER hmR0VmxExitRdmsr;
356static FNVMEXITHANDLER hmR0VmxExitWrmsr;
357static FNVMEXITHANDLER hmR0VmxExitErrInvalidGuestState;
358static FNVMEXITHANDLER hmR0VmxExitErrMsrLoad;
359static FNVMEXITHANDLER hmR0VmxExitErrUndefined;
360static FNVMEXITHANDLER hmR0VmxExitMwait;
361static FNVMEXITHANDLER hmR0VmxExitMtf;
362static FNVMEXITHANDLER hmR0VmxExitMonitor;
363static FNVMEXITHANDLER hmR0VmxExitPause;
364static FNVMEXITHANDLER hmR0VmxExitErrMachineCheck;
365static FNVMEXITHANDLER hmR0VmxExitTprBelowThreshold;
366static FNVMEXITHANDLER hmR0VmxExitApicAccess;
367static FNVMEXITHANDLER hmR0VmxExitXdtrAccess;
368static FNVMEXITHANDLER hmR0VmxExitXdtrAccess;
369static FNVMEXITHANDLER hmR0VmxExitEptViolation;
370static FNVMEXITHANDLER hmR0VmxExitEptMisconfig;
371static FNVMEXITHANDLER hmR0VmxExitRdtscp;
372static FNVMEXITHANDLER hmR0VmxExitPreemptTimer;
373static FNVMEXITHANDLER hmR0VmxExitWbinvd;
374static FNVMEXITHANDLER hmR0VmxExitXsetbv;
375static FNVMEXITHANDLER hmR0VmxExitRdrand;
376static FNVMEXITHANDLER hmR0VmxExitInvpcid;
377/** @} */
378
379static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
380static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
381static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
382static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
383static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
384static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
385static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
386static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
387
388/*******************************************************************************
389* Global Variables *
390*******************************************************************************/
391#ifdef HMVMX_USE_FUNCTION_TABLE
392
393/**
394 * VMX_EXIT dispatch table.
395 */
396static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
397{
398 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
399 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
400 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
401 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
402 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
403 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
404 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
405 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
406 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
407 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
408 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
409 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
410 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
411 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
412 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
413 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
414 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
415 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
416 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
417 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
418 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
419 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
420 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
421 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
422 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
423 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
424 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
425 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
426 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
427 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
428 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
429 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
430 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
431 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
432 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
433 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
434 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
435 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
436 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
437 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
438 /* 40 UNDEFINED */ hmR0VmxExitPause,
439 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
440 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
441 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
442 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
443 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
444 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
445 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
446 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
447 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
448 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
449 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
450 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
451 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
452 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
453 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
454 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
455 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
456 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
457 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
458};
459#endif /* HMVMX_USE_FUNCTION_TABLE */
460
461#ifdef VBOX_STRICT
462static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
463{
464 /* 0 */ "(Not Used)",
465 /* 1 */ "VMCALL executed in VMX root operation.",
466 /* 2 */ "VMCLEAR with invalid physical address.",
467 /* 3 */ "VMCLEAR with VMXON pointer.",
468 /* 4 */ "VMLAUNCH with non-clear VMCS.",
469 /* 5 */ "VMRESUME with non-launched VMCS.",
470 /* 6 */ "VMRESUME after VMXOFF",
471 /* 7 */ "VM entry with invalid control fields.",
472 /* 8 */ "VM entry with invalid host state fields.",
473 /* 9 */ "VMPTRLD with invalid physical address.",
474 /* 10 */ "VMPTRLD with VMXON pointer.",
475 /* 11 */ "VMPTRLD with incorrect revision identifier.",
476 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
477 /* 13 */ "VMWRITE to read-only VMCS component.",
478 /* 14 */ "(Not Used)",
479 /* 15 */ "VMXON executed in VMX root operation.",
480 /* 16 */ "VM entry with invalid executive-VMCS pointer.",
481 /* 17 */ "VM entry with non-launched executing VMCS.",
482 /* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
483 /* 19 */ "VMCALL with non-clear VMCS.",
484 /* 20 */ "VMCALL with invalid VM-exit control fields.",
485 /* 21 */ "(Not Used)",
486 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
487 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
488 /* 24 */ "VMCALL with invalid SMM-monitor features.",
489 /* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
490 /* 26 */ "VM entry with events blocked by MOV SS.",
491 /* 27 */ "(Not Used)",
492 /* 28 */ "Invalid operand to INVEPT/INVVPID."
493};
494#endif /* VBOX_STRICT */
495
496
497
498/**
499 * Updates the VM's last error record. If there was a VMX instruction error,
500 * reads the error data from the VMCS and updates VCPU's last error record as
501 * well.
502 *
503 * @param pVM Pointer to the VM.
504 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
505 * VERR_VMX_UNABLE_TO_START_VM or
506 * VERR_VMX_INVALID_VMCS_FIELD).
507 * @param rc The error code.
508 */
509static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
510{
511 AssertPtr(pVM);
512 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
513 || rc == VERR_VMX_UNABLE_TO_START_VM)
514 {
515 AssertPtrReturnVoid(pVCpu);
516 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
517 }
518 pVM->hm.s.lLastError = rc;
519}
520
521
522/**
523 * Reads the VM-entry interruption-information field from the VMCS into the VMX
524 * transient structure.
525 *
526 * @returns VBox status code.
527 * @param pVmxTransient Pointer to the VMX transient structure.
528 *
529 * @remarks No-long-jump zone!!!
530 */
531DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
532{
533 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
534 AssertRCReturn(rc, rc);
535 return VINF_SUCCESS;
536}
537
538
539/**
540 * Reads the VM-entry exception error code field from the VMCS into
541 * the VMX transient structure.
542 *
543 * @returns VBox status code.
544 * @param pVmxTransient Pointer to the VMX transient structure.
545 *
546 * @remarks No-long-jump zone!!!
547 */
548DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
549{
550 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
551 AssertRCReturn(rc, rc);
552 return VINF_SUCCESS;
553}
554
555
556/**
557 * Reads the VM-entry exception error code field from the VMCS into
558 * the VMX transient structure.
559 *
560 * @returns VBox status code.
561 * @param pVCpu Pointer to the VMCPU.
562 * @param pVmxTransient Pointer to the VMX transient structure.
563 *
564 * @remarks No-long-jump zone!!!
565 */
566DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
567{
568 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
569 AssertRCReturn(rc, rc);
570 return VINF_SUCCESS;
571}
572
573
574/**
575 * Reads the VM-exit interruption-information field from the VMCS into the VMX
576 * transient structure.
577 *
578 * @returns VBox status code.
579 * @param pVCpu Pointer to the VMCPU.
580 * @param pVmxTransient Pointer to the VMX transient structure.
581 */
582DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
583{
584 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
585 {
586 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
587 AssertRCReturn(rc, rc);
588 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
589 }
590 return VINF_SUCCESS;
591}
592
593
594/**
595 * Reads the VM-exit interruption error code from the VMCS into the VMX
596 * transient structure.
597 *
598 * @returns VBox status code.
599 * @param pVCpu Pointer to the VMCPU.
600 * @param pVmxTransient Pointer to the VMX transient structure.
601 */
602DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
603{
604 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
605 {
606 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
607 AssertRCReturn(rc, rc);
608 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
609 }
610 return VINF_SUCCESS;
611}
612
613
614/**
615 * Reads the VM-exit instruction length field from the VMCS into the VMX
616 * transient structure.
617 *
618 * @returns VBox status code.
619 * @param pVCpu Pointer to the VMCPU.
620 * @param pVmxTransient Pointer to the VMX transient structure.
621 */
622DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
623{
624 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
625 {
626 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
627 AssertRCReturn(rc, rc);
628 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
629 }
630 return VINF_SUCCESS;
631}
632
633
634/**
635 * Reads the VM-exit instruction-information field from the VMCS into
636 * the VMX transient structure.
637 *
638 * @returns VBox status code.
639 * @param pVCpu The cross context per CPU structure.
640 * @param pVmxTransient Pointer to the VMX transient structure.
641 */
642DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
643{
644 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
645 {
646 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->cbInstr);
647 AssertRCReturn(rc, rc);
648 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
649 }
650 return VINF_SUCCESS;
651}
652
653
654/**
655 * Reads the exit qualification from the VMCS into the VMX transient structure.
656 *
657 * @returns VBox status code.
658 * @param pVCpu Pointer to the VMCPU.
659 * @param pVmxTransient Pointer to the VMX transient structure.
660 */
661DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
662{
663 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
664 {
665 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
666 AssertRCReturn(rc, rc);
667 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
668 }
669 return VINF_SUCCESS;
670}
671
672
673/**
674 * Reads the IDT-vectoring information field from the VMCS into the VMX
675 * transient structure.
676 *
677 * @returns VBox status code.
678 * @param pVmxTransient Pointer to the VMX transient structure.
679 *
680 * @remarks No-long-jump zone!!!
681 */
682DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
683{
684 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
685 {
686 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
687 AssertRCReturn(rc, rc);
688 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
689 }
690 return VINF_SUCCESS;
691}
692
693
694/**
695 * Reads the IDT-vectoring error code from the VMCS into the VMX
696 * transient structure.
697 *
698 * @returns VBox status code.
699 * @param pVmxTransient Pointer to the VMX transient structure.
700 */
701DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
702{
703 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
704 {
705 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
706 AssertRCReturn(rc, rc);
707 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
708 }
709 return VINF_SUCCESS;
710}
711
712
713/**
714 * Enters VMX root mode operation on the current CPU.
715 *
716 * @returns VBox status code.
717 * @param pVM Pointer to the VM (optional, can be NULL, after
718 * a resume).
719 * @param HCPhysCpuPage Physical address of the VMXON region.
720 * @param pvCpuPage Pointer to the VMXON region.
721 */
722static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
723{
724 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
725 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
726 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
727
728 if (pVM)
729 {
730 /* Write the VMCS revision dword to the VMXON region. */
731 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
732 }
733
734 /* Enable the VMX bit in CR4 if necessary. */
735 RTCCUINTREG uCr4 = ASMGetCR4();
736 if (!(uCr4 & X86_CR4_VMXE))
737 ASMSetCR4(uCr4 | X86_CR4_VMXE);
738
739 /* Enter VMX root mode. */
740 int rc = VMXEnable(HCPhysCpuPage);
741 if (RT_FAILURE(rc))
742 ASMSetCR4(uCr4);
743
744 return rc;
745}
746
747
748/**
749 * Exits VMX root mode operation on the current CPU.
750 *
751 * @returns VBox status code.
752 */
753static int hmR0VmxLeaveRootMode(void)
754{
755 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
756
757 /* If we're for some reason not in VMX root mode, then don't leave it. */
758 RTCCUINTREG uHostCR4 = ASMGetCR4();
759 if (uHostCR4 & X86_CR4_VMXE)
760 {
761 /* Exit VMX root mode and clear the VMX bit in CR4. */
762 VMXDisable();
763 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
764 return VINF_SUCCESS;
765 }
766
767 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
768}
769
770
771/**
772 * Allocates and maps one physically contiguous page. The allocated page is
773 * zero'd out. (Used by various VT-x structures).
774 *
775 * @returns IPRT status code.
776 * @param pMemObj Pointer to the ring-0 memory object.
777 * @param ppVirt Where to store the virtual address of the
778 * allocation.
779 * @param pPhys Where to store the physical address of the
780 * allocation.
781 */
782DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
783{
784 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
785 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
786 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
787
788 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
789 if (RT_FAILURE(rc))
790 return rc;
791 *ppVirt = RTR0MemObjAddress(*pMemObj);
792 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
793 ASMMemZero32(*ppVirt, PAGE_SIZE);
794 return VINF_SUCCESS;
795}
796
797
798/**
799 * Frees and unmaps an allocated physical page.
800 *
801 * @param pMemObj Pointer to the ring-0 memory object.
802 * @param ppVirt Where to re-initialize the virtual address of
803 * allocation as 0.
804 * @param pHCPhys Where to re-initialize the physical address of the
805 * allocation as 0.
806 */
807DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
808{
809 AssertPtr(pMemObj);
810 AssertPtr(ppVirt);
811 AssertPtr(pHCPhys);
812 if (*pMemObj != NIL_RTR0MEMOBJ)
813 {
814 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
815 AssertRC(rc);
816 *pMemObj = NIL_RTR0MEMOBJ;
817 *ppVirt = 0;
818 *pHCPhys = 0;
819 }
820}
821
822
823/**
824 * Worker function to free VT-x related structures.
825 *
826 * @returns IPRT status code.
827 * @param pVM Pointer to the VM.
828 */
829static void hmR0VmxStructsFree(PVM pVM)
830{
831 for (VMCPUID i = 0; i < pVM->cCpus; i++)
832 {
833 PVMCPU pVCpu = &pVM->aCpus[i];
834 AssertPtr(pVCpu);
835
836#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
837 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
838 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
839#endif
840
841 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
842 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
843
844 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
845 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
846 }
847
848 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
849#ifdef VBOX_WITH_CRASHDUMP_MAGIC
850 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
851#endif
852}
853
854
855/**
856 * Worker function to allocate VT-x related VM structures.
857 *
858 * @returns IPRT status code.
859 * @param pVM Pointer to the VM.
860 */
861static int hmR0VmxStructsAlloc(PVM pVM)
862{
863 /*
864 * Initialize members up-front so we can cleanup properly on allocation failure.
865 */
866#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
867 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
868 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
869 pVM->hm.s.vmx.HCPhys##a_Name = 0;
870
871#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
872 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
873 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
874 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
875
876#ifdef VBOX_WITH_CRASHDUMP_MAGIC
877 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
878#endif
879 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
880
881 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
882 for (VMCPUID i = 0; i < pVM->cCpus; i++)
883 {
884 PVMCPU pVCpu = &pVM->aCpus[i];
885 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
886 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
887 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
888#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
889 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
890 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
891#endif
892 }
893#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
894#undef VMXLOCAL_INIT_VM_MEMOBJ
895
896 /*
897 * Allocate all the VT-x structures.
898 */
899 int rc = VINF_SUCCESS;
900#ifdef VBOX_WITH_CRASHDUMP_MAGIC
901 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
902 if (RT_FAILURE(rc))
903 goto cleanup;
904 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
905 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
906#endif
907
908 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
909 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
910 {
911 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
912 &pVM->hm.s.vmx.HCPhysApicAccess);
913 if (RT_FAILURE(rc))
914 goto cleanup;
915 }
916
917 /*
918 * Initialize per-VCPU VT-x structures.
919 */
920 for (VMCPUID i = 0; i < pVM->cCpus; i++)
921 {
922 PVMCPU pVCpu = &pVM->aCpus[i];
923 AssertPtr(pVCpu);
924
925 /* Allocate the VM control structure (VMCS). */
926 AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
927 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
928 if (RT_FAILURE(rc))
929 goto cleanup;
930
931 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
932 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
933 {
934 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
935 &pVCpu->hm.s.vmx.HCPhysVirtApic);
936 if (RT_FAILURE(rc))
937 goto cleanup;
938 }
939
940 /* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
941 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
942 {
943 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
944 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
945 if (RT_FAILURE(rc))
946 goto cleanup;
947 memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
948 }
949
950#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
951 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
952 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
953 if (RT_FAILURE(rc))
954 goto cleanup;
955
956 /* Allocate the VM-exit MSR-load page for the host MSRs. */
957 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
958 if (RT_FAILURE(rc))
959 goto cleanup;
960#endif
961 }
962
963 return VINF_SUCCESS;
964
965cleanup:
966 hmR0VmxStructsFree(pVM);
967 return rc;
968}
969
970
971/**
972 * Does global VT-x initialization (called during module initialization).
973 *
974 * @returns VBox status code.
975 */
976VMMR0DECL(int) VMXR0GlobalInit(void)
977{
978#ifdef HMVMX_USE_FUNCTION_TABLE
979 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
980# ifdef VBOX_STRICT
981 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
982 Assert(g_apfnVMExitHandlers[i]);
983# endif
984#endif
985 return VINF_SUCCESS;
986}
987
988
989/**
990 * Does global VT-x termination (called during module termination).
991 */
992VMMR0DECL(void) VMXR0GlobalTerm()
993{
994 /* Nothing to do currently. */
995}
996
997
998/**
999 * Sets up and activates VT-x on the current CPU.
1000 *
1001 * @returns VBox status code.
1002 * @param pCpu Pointer to the global CPU info struct.
1003 * @param pVM Pointer to the VM (can be NULL after a host resume
1004 * operation).
1005 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1006 * fEnabledByHost is true).
1007 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1008 * @a fEnabledByHost is true).
1009 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1010 * enable VT-x on the host.
1011 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1012 */
1013VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1014 void *pvMsrs)
1015{
1016 AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1017 AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1018 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1019
1020 /* Enable VT-x if it's not already enabled by the host. */
1021 if (!fEnabledByHost)
1022 {
1023 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1024 if (RT_FAILURE(rc))
1025 return rc;
1026 }
1027
1028 /*
1029 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1030 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1031 */
1032 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1033 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1034 {
1035 hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1036 pCpu->fFlushAsidBeforeUse = false;
1037 }
1038 else
1039 pCpu->fFlushAsidBeforeUse = true;
1040
1041 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1042 ++pCpu->cTlbFlushes;
1043
1044 return VINF_SUCCESS;
1045}
1046
1047
1048/**
1049 * Deactivates VT-x on the current CPU.
1050 *
1051 * @returns VBox status code.
1052 * @param pCpu Pointer to the global CPU info struct.
1053 * @param pvCpuPage Pointer to the VMXON region.
1054 * @param HCPhysCpuPage Physical address of the VMXON region.
1055 *
1056 * @remarks This function should never be called when SUPR0EnableVTx() or
1057 * similar was used to enable VT-x on the host.
1058 */
1059VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1060{
1061 NOREF(pCpu);
1062 NOREF(pvCpuPage);
1063 NOREF(HCPhysCpuPage);
1064
1065 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1066 return hmR0VmxLeaveRootMode();
1067}
1068
1069
1070/**
1071 * Sets the permission bits for the specified MSR in the MSR bitmap.
1072 *
1073 * @param pVCpu Pointer to the VMCPU.
1074 * @param uMSR The MSR value.
1075 * @param enmRead Whether reading this MSR causes a VM-exit.
1076 * @param enmWrite Whether writing this MSR causes a VM-exit.
1077 */
1078static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1079{
1080 int32_t iBit;
1081 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1082
1083 /*
1084 * Layout:
1085 * 0x000 - 0x3ff - Low MSR read bits
1086 * 0x400 - 0x7ff - High MSR read bits
1087 * 0x800 - 0xbff - Low MSR write bits
1088 * 0xc00 - 0xfff - High MSR write bits
1089 */
1090 if (uMsr <= 0x00001FFF)
1091 iBit = uMsr;
1092 else if ( uMsr >= 0xC0000000
1093 && uMsr <= 0xC0001FFF)
1094 {
1095 iBit = (uMsr - 0xC0000000);
1096 pbMsrBitmap += 0x400;
1097 }
1098 else
1099 {
1100 AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1101 return;
1102 }
1103
1104 Assert(iBit <= 0x1fff);
1105 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1106 ASMBitSet(pbMsrBitmap, iBit);
1107 else
1108 ASMBitClear(pbMsrBitmap, iBit);
1109
1110 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1111 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1112 else
1113 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1114}
1115
1116
1117/**
1118 * Flushes the TLB using EPT.
1119 *
1120 * @returns VBox status code.
1121 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1122 * enmFlush).
1123 * @param enmFlush Type of flush.
1124 *
1125 * @remarks Caller is responsible for making sure this function is called only
1126 * when NestedPaging is supported and providing @a enmFlush that is
1127 * supported by the CPU.
1128 */
1129static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1130{
1131 uint64_t au64Descriptor[2];
1132 if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1133 au64Descriptor[0] = 0;
1134 else
1135 {
1136 Assert(pVCpu);
1137 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1138 }
1139 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1140
1141 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1142 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1143 rc));
1144 if ( RT_SUCCESS(rc)
1145 && pVCpu)
1146 {
1147 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1148 }
1149}
1150
1151
1152/**
1153 * Flushes the TLB using VPID.
1154 *
1155 * @returns VBox status code.
1156 * @param pVM Pointer to the VM.
1157 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1158 * enmFlush).
1159 * @param enmFlush Type of flush.
1160 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1161 * on @a enmFlush).
1162 */
1163static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1164{
1165 AssertPtr(pVM);
1166 Assert(pVM->hm.s.vmx.fVpid);
1167
1168 uint64_t au64Descriptor[2];
1169 if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1170 {
1171 au64Descriptor[0] = 0;
1172 au64Descriptor[1] = 0;
1173 }
1174 else
1175 {
1176 AssertPtr(pVCpu);
1177 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1178 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1179 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1180 au64Descriptor[1] = GCPtr;
1181 }
1182
1183 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1184 AssertMsg(rc == VINF_SUCCESS,
1185 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1186 if ( RT_SUCCESS(rc)
1187 && pVCpu)
1188 {
1189 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1190 }
1191}
1192
1193
1194/**
1195 * Invalidates a guest page by guest virtual address. Only relevant for
1196 * EPT/VPID, otherwise there is nothing really to invalidate.
1197 *
1198 * @returns VBox status code.
1199 * @param pVM Pointer to the VM.
1200 * @param pVCpu Pointer to the VMCPU.
1201 * @param GCVirt Guest virtual address of the page to invalidate.
1202 */
1203VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1204{
1205 AssertPtr(pVM);
1206 AssertPtr(pVCpu);
1207 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1208
1209 bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1210 if (!fFlushPending)
1211 {
1212 /*
1213 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1214 * See @bugref{6043} and @bugref{6177}.
1215 *
1216 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1217 * function maybe called in a loop with individual addresses.
1218 */
1219 if (pVM->hm.s.vmx.fVpid)
1220 {
1221 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1222 {
1223 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1224 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1225 }
1226 else
1227 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1228 }
1229 else if (pVM->hm.s.fNestedPaging)
1230 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1231 }
1232
1233 return VINF_SUCCESS;
1234}
1235
1236
1237/**
1238 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1239 * otherwise there is nothing really to invalidate.
1240 *
1241 * @returns VBox status code.
1242 * @param pVM Pointer to the VM.
1243 * @param pVCpu Pointer to the VMCPU.
1244 * @param GCPhys Guest physical address of the page to invalidate.
1245 */
1246VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1247{
1248 LogFlowFunc(("%RGp\n", GCPhys));
1249
1250 /*
1251 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1252 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1253 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1254 */
1255 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1256 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1257 return VINF_SUCCESS;
1258}
1259
1260
1261/**
1262 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1263 * case where neither EPT nor VPID is supported by the CPU.
1264 *
1265 * @param pVM Pointer to the VM.
1266 * @param pVCpu Pointer to the VMCPU.
1267 *
1268 * @remarks Called with interrupts disabled.
1269 */
1270static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1271{
1272 NOREF(pVM);
1273 AssertPtr(pVCpu);
1274 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1275 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1276
1277 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
1278 AssertPtr(pCpu);
1279
1280 pVCpu->hm.s.TlbShootdown.cPages = 0;
1281 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1282 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1283 pVCpu->hm.s.fForceTLBFlush = false;
1284 return;
1285}
1286
1287
1288/**
1289 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1290 *
1291 * @param pVM Pointer to the VM.
1292 * @param pVCpu Pointer to the VMCPU.
1293 * @remarks All references to "ASID" in this function pertains to "VPID" in
1294 * Intel's nomenclature. The reason is, to avoid confusion in compare
1295 * statements since the host-CPU copies are named "ASID".
1296 *
1297 * @remarks Called with interrupts disabled.
1298 */
1299static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1300{
1301#ifdef VBOX_WITH_STATISTICS
1302 bool fTlbFlushed = false;
1303# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1304# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1305 if (!fTlbFlushed) \
1306 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1307 } while (0)
1308#else
1309# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1310# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1311#endif
1312
1313 AssertPtr(pVM);
1314 AssertPtr(pVCpu);
1315 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1316 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1317 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1318
1319 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
1320 AssertPtr(pCpu);
1321
1322 /*
1323 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1324 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1325 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1326 */
1327 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1328 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1329 {
1330 ++pCpu->uCurrentAsid;
1331 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1332 {
1333 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1334 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1335 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1336 }
1337
1338 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1339 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1340 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1341
1342 /*
1343 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1344 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1345 */
1346 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1347 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1348 HMVMX_SET_TAGGED_TLB_FLUSHED();
1349 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1350 }
1351
1352 /* Check for explicit TLB shootdowns. */
1353 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1354 {
1355 /*
1356 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1357 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1358 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1359 * but not guest-physical mappings.
1360 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1361 */
1362 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1363 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1364 HMVMX_SET_TAGGED_TLB_FLUSHED();
1365 }
1366
1367 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1368 * not be executed. See hmQueueInvlPage() where it is commented
1369 * out. Support individual entry flushing someday. */
1370 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1371 {
1372 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1373
1374 /*
1375 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1376 * as supported by the CPU.
1377 */
1378 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1379 {
1380 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1381 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1382 }
1383 else
1384 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1385
1386 HMVMX_SET_TAGGED_TLB_FLUSHED();
1387 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1388 }
1389
1390 pVCpu->hm.s.TlbShootdown.cPages = 0;
1391 pVCpu->hm.s.fForceTLBFlush = false;
1392
1393 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1394
1395 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1396 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1397 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1398 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1399 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1400 ("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1401 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1402 ("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1403
1404 /* Update VMCS with the VPID. */
1405 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1406 AssertRC(rc);
1407
1408#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1409}
1410
1411
1412/**
1413 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
1414 *
1415 * @returns VBox status code.
1416 * @param pVM Pointer to the VM.
1417 * @param pVCpu Pointer to the VMCPU.
1418 *
1419 * @remarks Called with interrupts disabled.
1420 */
1421static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1422{
1423 AssertPtr(pVM);
1424 AssertPtr(pVCpu);
1425 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1426 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1427
1428 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
1429 AssertPtr(pCpu);
1430
1431 /*
1432 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1433 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1434 */
1435 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1436 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1437 {
1438 pVCpu->hm.s.fForceTLBFlush = true;
1439 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1440 }
1441
1442 /* Check for explicit TLB shootdown flushes. */
1443 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1444 {
1445 pVCpu->hm.s.fForceTLBFlush = true;
1446 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1447 }
1448
1449 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1450 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1451
1452 if (pVCpu->hm.s.fForceTLBFlush)
1453 {
1454 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1455 pVCpu->hm.s.fForceTLBFlush = false;
1456 }
1457 else
1458 {
1459 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1460 * not be executed. See hmQueueInvlPage() where it is commented
1461 * out. Support individual entry flushing someday. */
1462 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1463 {
1464 /* We cannot flush individual entries without VPID support. Flush using EPT. */
1465 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1466 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1467 }
1468 else
1469 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1470 }
1471
1472 pVCpu->hm.s.TlbShootdown.cPages = 0;
1473 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1474}
1475
1476
1477/**
1478 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
1479 *
1480 * @returns VBox status code.
1481 * @param pVM Pointer to the VM.
1482 * @param pVCpu Pointer to the VMCPU.
1483 *
1484 * @remarks Called with interrupts disabled.
1485 */
1486static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1487{
1488 AssertPtr(pVM);
1489 AssertPtr(pVCpu);
1490 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1491 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1492
1493 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
1494
1495 /*
1496 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1497 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1498 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1499 */
1500 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1501 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1502 {
1503 pVCpu->hm.s.fForceTLBFlush = true;
1504 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1505 }
1506
1507 /* Check for explicit TLB shootdown flushes. */
1508 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1509 {
1510 /*
1511 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1512 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1513 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1514 */
1515 pVCpu->hm.s.fForceTLBFlush = true;
1516 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1517 }
1518
1519 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1520 if (pVCpu->hm.s.fForceTLBFlush)
1521 {
1522 ++pCpu->uCurrentAsid;
1523 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1524 {
1525 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1526 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1527 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1528 }
1529
1530 pVCpu->hm.s.fForceTLBFlush = false;
1531 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1532 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1533 if (pCpu->fFlushAsidBeforeUse)
1534 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1535 }
1536 else
1537 {
1538 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1539 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1540 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1541 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1542
1543 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1544 * not be executed. See hmQueueInvlPage() where it is commented
1545 * out. Support individual entry flushing someday. */
1546 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1547 {
1548 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1549 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1550 {
1551 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1552 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1553 }
1554 else
1555 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1556 }
1557 else
1558 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1559 }
1560
1561 pVCpu->hm.s.TlbShootdown.cPages = 0;
1562 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1563
1564 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1565 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1566 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1567 ("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1568 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1569 ("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1570
1571 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1572 AssertRC(rc);
1573}
1574
1575
1576/**
1577 * Flushes the guest TLB entry based on CPU capabilities.
1578 *
1579 * @param pVCpu Pointer to the VMCPU.
1580 */
1581DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1582{
1583 PVM pVM = pVCpu->CTX_SUFF(pVM);
1584 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1585 {
1586 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1587 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1588 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1589 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1590 default:
1591 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1592 break;
1593 }
1594}
1595
1596
1597/**
1598 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1599 * TLB entries from the host TLB before VM-entry.
1600 *
1601 * @returns VBox status code.
1602 * @param pVM Pointer to the VM.
1603 */
1604static int hmR0VmxSetupTaggedTlb(PVM pVM)
1605{
1606 /*
1607 * Determine optimal flush type for Nested Paging.
1608 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1609 * guest execution (see hmR3InitFinalizeR0()).
1610 */
1611 if (pVM->hm.s.fNestedPaging)
1612 {
1613 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1614 {
1615 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1616 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1617 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1618 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1619 else
1620 {
1621 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1622 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1623 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1624 }
1625
1626 /* Make sure the write-back cacheable memory type for EPT is supported. */
1627 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1628 {
1629 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
1630 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1631 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1632 }
1633 }
1634 else
1635 {
1636 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1637 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1638 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1639 }
1640 }
1641
1642 /*
1643 * Determine optimal flush type for VPID.
1644 */
1645 if (pVM->hm.s.vmx.fVpid)
1646 {
1647 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1648 {
1649 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1650 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1651 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1652 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1653 else
1654 {
1655 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1656 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1657 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1658 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1659 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1660 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1661 pVM->hm.s.vmx.fVpid = false;
1662 }
1663 }
1664 else
1665 {
1666 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1667 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1668 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1669 pVM->hm.s.vmx.fVpid = false;
1670 }
1671 }
1672
1673 /*
1674 * Setup the handler for flushing tagged-TLBs.
1675 */
1676 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1677 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1678 else if (pVM->hm.s.fNestedPaging)
1679 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1680 else if (pVM->hm.s.vmx.fVpid)
1681 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1682 else
1683 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1684 return VINF_SUCCESS;
1685}
1686
1687
1688/**
1689 * Sets up pin-based VM-execution controls in the VMCS.
1690 *
1691 * @returns VBox status code.
1692 * @param pVM Pointer to the VM.
1693 * @param pVCpu Pointer to the VMCPU.
1694 */
1695static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1696{
1697 AssertPtr(pVM);
1698 AssertPtr(pVCpu);
1699
1700 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
1701 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
1702
1703 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1704 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1705 Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1706
1707 /* Enable the VMX preemption timer. */
1708 if (pVM->hm.s.vmx.fUsePreemptTimer)
1709 {
1710 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1711 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1712 }
1713
1714 if ((val & zap) != val)
1715 {
1716 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1717 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
1718 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
1719 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1720 }
1721
1722 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1723 AssertRCReturn(rc, rc);
1724
1725 /* Update VCPU with the currently set pin-based VM-execution controls. */
1726 pVCpu->hm.s.vmx.u32PinCtls = val;
1727 return rc;
1728}
1729
1730
1731/**
1732 * Sets up processor-based VM-execution controls in the VMCS.
1733 *
1734 * @returns VBox status code.
1735 * @param pVM Pointer to the VM.
1736 * @param pVMCPU Pointer to the VMCPU.
1737 */
1738static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1739{
1740 AssertPtr(pVM);
1741 AssertPtr(pVCpu);
1742
1743 int rc = VERR_INTERNAL_ERROR_5;
1744 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1745 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1746
1747 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1748 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1749 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1750 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1751 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1752 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1753 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1754
1755 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1756 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1757 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1758 {
1759 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1760 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
1761 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1762 }
1763
1764 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1765 if (!pVM->hm.s.fNestedPaging)
1766 {
1767 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1768 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1769 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1770 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1771 }
1772
1773 /* Use TPR shadowing if supported by the CPU. */
1774 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1775 {
1776 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1777 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1778 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1779 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1780 AssertRCReturn(rc, rc);
1781
1782 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1783 /* CR8 writes causes a VM-exit based on TPR threshold. */
1784 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1785 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1786 }
1787 else
1788 {
1789 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1790 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1791 }
1792
1793 /* Use MSR-bitmaps if supported by the CPU. */
1794 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1795 {
1796 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1797
1798 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1799 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1800 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1801 AssertRCReturn(rc, rc);
1802
1803 /*
1804 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1805 * automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1806 */
1807 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1808 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1809 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1810 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1811 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1812 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1813 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1814 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1815 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1816 }
1817
1818 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1819 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1820 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1821
1822 if ((val & zap) != val)
1823 {
1824 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1825 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
1826 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
1827 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1828 }
1829
1830 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1831 AssertRCReturn(rc, rc);
1832
1833 /* Update VCPU with the currently set processor-based VM-execution controls. */
1834 pVCpu->hm.s.vmx.u32ProcCtls = val;
1835
1836 /*
1837 * Secondary processor-based VM-execution controls.
1838 */
1839 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1840 {
1841 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1842 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1843
1844 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1845 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1846
1847 if (pVM->hm.s.fNestedPaging)
1848 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1849 else
1850 {
1851 /*
1852 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1853 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1854 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1855 */
1856 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1857 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1858 }
1859
1860 if (pVM->hm.s.vmx.fVpid)
1861 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1862
1863 if (pVM->hm.s.vmx.fUnrestrictedGuest)
1864 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1865
1866 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1867 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1868 * done dynamically. */
1869 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1870 {
1871 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1872 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1873 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1874 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1875 AssertRCReturn(rc, rc);
1876 }
1877
1878 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1879 {
1880 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1881 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1882 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1883 }
1884
1885 if ((val & zap) != val)
1886 {
1887 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1888 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
1889 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1890 }
1891
1892 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1893 AssertRCReturn(rc, rc);
1894
1895 /* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1896 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1897 }
1898 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
1899 {
1900 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
1901 "available\n"));
1902 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1903 }
1904
1905 return VINF_SUCCESS;
1906}
1907
1908
1909/**
1910 * Sets up miscellaneous (everything other than Pin & Processor-based
1911 * VM-execution) control fields in the VMCS.
1912 *
1913 * @returns VBox status code.
1914 * @param pVM Pointer to the VM.
1915 * @param pVCpu Pointer to the VMCPU.
1916 */
1917static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1918{
1919 AssertPtr(pVM);
1920 AssertPtr(pVCpu);
1921
1922 int rc = VERR_GENERAL_FAILURE;
1923
1924 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1925#if 0
1926 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1927 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1928 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1929
1930 /*
1931 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1932 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1933 * We thus use the exception bitmap to control it rather than use both.
1934 */
1935 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1936 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1937
1938 /** @todo Explore possibility of using IO-bitmaps. */
1939 /* All IO & IOIO instructions cause VM-exits. */
1940 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1941 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1942
1943 /* Initialize the MSR-bitmap area. */
1944 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1945 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
1946 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1947#endif
1948
1949#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1950 /* Setup MSR autoloading/storing. */
1951 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1952 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1953 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1954 AssertRCReturn(rc, rc);
1955 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1956 AssertRCReturn(rc, rc);
1957
1958 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1959 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1960 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1961 AssertRCReturn(rc, rc);
1962#endif
1963
1964 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1965 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1966 AssertRCReturn(rc, rc);
1967
1968 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1969#if 0
1970 /* Setup debug controls */
1971 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
1972 AssertRCReturn(rc, rc);
1973 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1974 AssertRCReturn(rc, rc);
1975#endif
1976
1977 return rc;
1978}
1979
1980
1981/**
1982 * Sets up the initial exception bitmap in the VMCS based on static conditions
1983 * (i.e. conditions that cannot ever change at runtime).
1984 *
1985 * @returns VBox status code.
1986 * @param pVM Pointer to the VM.
1987 * @param pVCpu Pointer to the VMCPU.
1988 */
1989static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1990{
1991 AssertPtr(pVM);
1992 AssertPtr(pVCpu);
1993
1994 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1995
1996 uint32_t u32XcptBitmap = 0;
1997
1998 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1999 if (!pVM->hm.s.fNestedPaging)
2000 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2001
2002 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2003 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2004 AssertRCReturn(rc, rc);
2005 return rc;
2006}
2007
2008
2009/**
2010 * Sets up the initial guest-state mask. The guest-state mask is consulted
2011 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2012 * for the nested virtualization case (as it would cause a VM-exit).
2013 *
2014 * @param pVCpu Pointer to the VMCPU.
2015 */
2016static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2017{
2018 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2019 pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
2020 return VINF_SUCCESS;
2021}
2022
2023
2024/**
2025 * Does per-VM VT-x initialization.
2026 *
2027 * @returns VBox status code.
2028 * @param pVM Pointer to the VM.
2029 */
2030VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2031{
2032 LogFlowFunc(("pVM=%p\n", pVM));
2033
2034 int rc = hmR0VmxStructsAlloc(pVM);
2035 if (RT_FAILURE(rc))
2036 {
2037 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2038 return rc;
2039 }
2040
2041 return VINF_SUCCESS;
2042}
2043
2044
2045/**
2046 * Does per-VM VT-x termination.
2047 *
2048 * @returns VBox status code.
2049 * @param pVM Pointer to the VM.
2050 */
2051VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2052{
2053 LogFlowFunc(("pVM=%p\n", pVM));
2054
2055#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2056 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2057 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2058#endif
2059 hmR0VmxStructsFree(pVM);
2060 return VINF_SUCCESS;
2061}
2062
2063
2064/**
2065 * Sets up the VM for execution under VT-x.
2066 * This function is only called once per-VM during initialization.
2067 *
2068 * @returns VBox status code.
2069 * @param pVM Pointer to the VM.
2070 */
2071VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2072{
2073 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2074 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2075
2076 LogFlowFunc(("pVM=%p\n", pVM));
2077
2078 /*
2079 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2080 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2081 */
2082 /* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2083 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2084 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2085 || !pVM->hm.s.vmx.pRealModeTSS))
2086 {
2087 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2088 return VERR_INTERNAL_ERROR;
2089 }
2090
2091#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2092 /*
2093 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2094 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2095 */
2096 if ( pVM->hm.s.fAllow64BitGuests
2097 && !HMVMX_IS_64BIT_HOST_MODE())
2098 {
2099 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2100 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2101 }
2102#endif
2103
2104 /* Initialize these always, see hmR3InitFinalizeR0().*/
2105 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2106 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2107
2108 /* Setup the tagged-TLB flush handlers. */
2109 int rc = hmR0VmxSetupTaggedTlb(pVM);
2110 if (RT_FAILURE(rc))
2111 {
2112 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2113 return rc;
2114 }
2115
2116 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2117 {
2118 PVMCPU pVCpu = &pVM->aCpus[i];
2119 AssertPtr(pVCpu);
2120 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2121
2122 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2123 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2124
2125 /* Set revision dword at the beginning of the VMCS structure. */
2126 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2127
2128 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2129 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2130 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2131 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2132
2133 /* Load this VMCS as the current VMCS. */
2134 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2135 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2136 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2137
2138 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2139 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2140 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2141
2142 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2143 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2144 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2145
2146 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2147 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2148 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2149
2150 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2151 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2152 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2153
2154 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2155 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2156 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2157
2158#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2159 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2160 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2161 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2162#endif
2163
2164 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2165 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2166 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2167 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2168
2169 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2170
2171 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2172 }
2173
2174 return VINF_SUCCESS;
2175}
2176
2177
2178/**
2179 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2180 * the VMCS.
2181 *
2182 * @returns VBox status code.
2183 * @param pVM Pointer to the VM.
2184 * @param pVCpu Pointer to the VMCPU.
2185 */
2186DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2187{
2188 RTCCUINTREG uReg = ASMGetCR0();
2189 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2190 AssertRCReturn(rc, rc);
2191
2192#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2193 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2194 if (HMVMX_IS_64BIT_HOST_MODE())
2195 {
2196 uint64_t uRegCR3 = HMR0Get64bitCR3();
2197 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2198 }
2199 else
2200#endif
2201 {
2202 uReg = ASMGetCR3();
2203 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2204 }
2205 AssertRCReturn(rc, rc);
2206
2207 uReg = ASMGetCR4();
2208 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2209 AssertRCReturn(rc, rc);
2210 return rc;
2211}
2212
2213
2214/**
2215 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2216 * the host-state area in the VMCS.
2217 *
2218 * @returns VBox status code.
2219 * @param pVM Pointer to the VM.
2220 * @param pVCpu Pointer to the VMCPU.
2221 */
2222DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2223{
2224 int rc = VERR_INTERNAL_ERROR_5;
2225 RTSEL uSelDS = 0;
2226 RTSEL uSelES = 0;
2227 RTSEL uSelFS = 0;
2228 RTSEL uSelGS = 0;
2229 RTSEL uSelTR = 0;
2230
2231 /*
2232 * Host DS, ES, FS and GS segment registers.
2233 */
2234#if HC_ARCH_BITS == 64
2235 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2236 uSelDS = ASMGetDS();
2237 uSelES = ASMGetES();
2238 uSelFS = ASMGetFS();
2239 uSelGS = ASMGetGS();
2240#endif
2241
2242 /*
2243 * Host CS and SS segment registers.
2244 */
2245 RTSEL uSelCS;
2246 RTSEL uSelSS;
2247#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2248 if (HMVMX_IS_64BIT_HOST_MODE())
2249 {
2250 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2251 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2252 }
2253 else
2254 {
2255 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2256 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2257 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2258 }
2259#else
2260 uSelCS = ASMGetCS();
2261 uSelSS = ASMGetSS();
2262#endif
2263
2264 /*
2265 * Host TR segment register.
2266 */
2267 uSelTR = ASMGetTR();
2268
2269#if HC_ARCH_BITS == 64
2270 /*
2271 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2272 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2273 */
2274 if (uSelDS & (X86_SEL_RPL | X86_SEL_LDT))
2275 {
2276 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_DS;
2277 pVCpu->hm.s.vmx.RestoreHost.uHostSelDS = uSelDS;
2278 uSelDS = 0;
2279 }
2280 if (uSelES & (X86_SEL_RPL | X86_SEL_LDT))
2281 {
2282 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_ES;
2283 pVCpu->hm.s.vmx.RestoreHost.uHostSelES = uSelES;
2284 uSelES = 0;
2285 }
2286 if (uSelFS & (X86_SEL_RPL | X86_SEL_LDT))
2287 {
2288 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_FS;
2289 pVCpu->hm.s.vmx.RestoreHost.uHostSelFS = uSelFS;
2290 uSelFS = 0;
2291 }
2292 if (uSelGS & (X86_SEL_RPL | X86_SEL_LDT))
2293 {
2294 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_GS;
2295 pVCpu->hm.s.vmx.RestoreHost.uHostSelGS = uSelGS;
2296 uSelGS = 0;
2297 }
2298#endif
2299
2300 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2301 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2302 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2303 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2304 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2305 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2306 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2307 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2308 Assert(uSelCS);
2309 Assert(uSelTR);
2310
2311 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2312#if 0
2313 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2314 Assert(uSelSS != 0);
2315#endif
2316
2317 /* Write these host selector fields into the host-state area in the VMCS. */
2318 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2319 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2320#if HC_ARCH_BITS == 64
2321 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2322 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2323 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2324 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2325#endif
2326 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2327
2328 /*
2329 * Host GDTR and IDTR.
2330 */
2331 RTGDTR Gdtr;
2332 RT_ZERO(Gdtr);
2333#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2334 if (HMVMX_IS_64BIT_HOST_MODE())
2335 {
2336 X86XDTR64 Gdtr64;
2337 X86XDTR64 Idtr64;
2338 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2339 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2340 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2341
2342 Gdtr.cbGdt = Gdtr64.cb;
2343 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2344 }
2345 else
2346#endif
2347 {
2348 RTIDTR Idtr;
2349 ASMGetGDTR(&Gdtr);
2350 ASMGetIDTR(&Idtr);
2351 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2352 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2353
2354#if HC_ARCH_BITS == 64
2355 /*
2356 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2357 * maximum limit (0xffff) on every VM-exit.
2358 */
2359 if (Gdtr.cbGdt != 0xffff)
2360 {
2361 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2362 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2363 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2364 }
2365
2366 /*
2367 * IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2368 * is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2369 * Intel spec. 6.2 "Exception and Interrupt Vectors".
2370 */
2371 if (Idtr.cbIdt < 0x0fff)
2372 {
2373 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
2374 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2375 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2376 }
2377#endif
2378 }
2379
2380 /*
2381 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2382 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2383 */
2384 if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2385 {
2386 AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2387 return VERR_VMX_INVALID_HOST_STATE;
2388 }
2389
2390 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2391#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2392 if (HMVMX_IS_64BIT_HOST_MODE())
2393 {
2394 /* We need the 64-bit TR base for hybrid darwin. */
2395 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2396 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2397 }
2398 else
2399#endif
2400 {
2401 uintptr_t uTRBase;
2402#if HC_ARCH_BITS == 64
2403 uTRBase = X86DESC64_BASE(pDesc);
2404
2405 /*
2406 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2407 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2408 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2409 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2410 *
2411 * [1] See Intel spec. 3.5 "System Descriptor Types".
2412 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2413 */
2414 Assert(pDesc->System.u4Type == 11);
2415 if ( pDesc->System.u16LimitLow != 0x67
2416 || pDesc->System.u4LimitHigh)
2417 {
2418 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
2419 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2420
2421 /* Store the GDTR here as we need it while restoring TR. */
2422 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2423 }
2424#else
2425 uTRBase = X86DESC_BASE(pDesc);
2426#endif
2427 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2428 }
2429 AssertRCReturn(rc, rc);
2430
2431 /*
2432 * Host FS base and GS base.
2433 */
2434#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2435 if (HMVMX_IS_64BIT_HOST_MODE())
2436 {
2437 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2438 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2439 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2440 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2441
2442# if HC_ARCH_BITS == 64
2443 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2444 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2445 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2446 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2447 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2448# endif
2449 }
2450#endif
2451 return rc;
2452}
2453
2454
2455/**
2456 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2457 * host-state area of the VMCS. Theses MSRs will be automatically restored on
2458 * the host after every successful VM exit.
2459 *
2460 * @returns VBox status code.
2461 * @param pVM Pointer to the VM.
2462 * @param pVCpu Pointer to the VMCPU.
2463 */
2464DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2465{
2466 AssertPtr(pVCpu);
2467 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2468
2469 int rc = VINF_SUCCESS;
2470#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2471 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
2472 uint32_t cHostMsrs = 0;
2473 uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2474
2475 if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX | X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2476 {
2477 uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2478
2479# if HC_ARCH_BITS == 64
2480 /* Paranoia. 64-bit code requires these bits to be set always. */
2481 Assert((u64HostEfer & (MSR_K6_EFER_LMA | MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA | MSR_K6_EFER_LME));
2482
2483 /*
2484 * We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2485 * All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2486 * some reason (e.g. allow transparent reads) we would activate the code below.
2487 */
2488# if 0
2489 /* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2490 Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2491 /* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2492 See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2493 if (CPUMIsGuestInLongMode(pVCpu))
2494 {
2495 uint64_t u64GuestEfer;
2496 rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2497 AssertRC(rc);
2498
2499 if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2500 {
2501 pHostMsr->u32Msr = MSR_K6_EFER;
2502 pHostMsr->u32Reserved = 0;
2503 pHostMsr->u64Value = u64HostEfer;
2504 pHostMsr++; cHostMsrs++;
2505 }
2506 }
2507# endif
2508# else /* HC_ARCH_BITS != 64 */
2509 pHostMsr->u32Msr = MSR_K6_EFER;
2510 pHostMsr->u32Reserved = 0;
2511# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2512 if (CPUMIsGuestInLongMode(pVCpu))
2513 {
2514 /* Must match the EFER value in our 64 bits switcher. */
2515 pHostMsr->u64Value = u64HostEfer | MSR_K6_EFER_LME | MSR_K6_EFER_SCE | MSR_K6_EFER_NXE;
2516 }
2517 else
2518# endif
2519 pHostMsr->u64Value = u64HostEfer;
2520 pHostMsr++; cHostMsrs++;
2521# endif /* HC_ARCH_BITS == 64 */
2522 }
2523
2524# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2525 if (HMVMX_IS_64BIT_HOST_MODE())
2526 {
2527 pHostMsr->u32Msr = MSR_K6_STAR;
2528 pHostMsr->u32Reserved = 0;
2529 pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2530 pHostMsr++; cHostMsrs++;
2531 pHostMsr->u32Msr = MSR_K8_LSTAR;
2532 pHostMsr->u32Reserved = 0;
2533 pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2534 pHostMsr++; cHostMsrs++;
2535 pHostMsr->u32Msr = MSR_K8_SF_MASK;
2536 pHostMsr->u32Reserved = 0;
2537 pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2538 pHostMsr++; cHostMsrs++;
2539 pHostMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
2540 pHostMsr->u32Reserved = 0;
2541 pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2542 pHostMsr++; cHostMsrs++;
2543 }
2544# endif
2545
2546 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2547 if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)))
2548 {
2549 LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)));
2550 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_HOST_MSR_STORAGE;
2551 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2552 }
2553
2554 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2555#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2556
2557 /*
2558 * Host Sysenter MSRs.
2559 */
2560 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2561 AssertRCReturn(rc, rc);
2562#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2563 if (HMVMX_IS_64BIT_HOST_MODE())
2564 {
2565 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2566 AssertRCReturn(rc, rc);
2567 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2568 }
2569 else
2570 {
2571 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2572 AssertRCReturn(rc, rc);
2573 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2574 }
2575#elif HC_ARCH_BITS == 32
2576 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2577 AssertRCReturn(rc, rc);
2578 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2579#else
2580 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2581 AssertRCReturn(rc, rc);
2582 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2583#endif
2584 AssertRCReturn(rc, rc);
2585
2586 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2587 * hmR0VmxSetupExitCtls() !! */
2588 return rc;
2589}
2590
2591
2592/**
2593 * Sets up VM-entry controls in the VMCS. These controls can affect things done
2594 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2595 * controls".
2596 *
2597 * @returns VBox status code.
2598 * @param pVCpu Pointer to the VMCPU.
2599 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2600 * out-of-sync. Make sure to update the required fields
2601 * before using them.
2602 *
2603 * @remarks No-long-jump zone!!!
2604 */
2605DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2606{
2607 int rc = VINF_SUCCESS;
2608 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2609 {
2610 PVM pVM = pVCpu->CTX_SUFF(pVM);
2611 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2612 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2613
2614 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2615 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2616
2617 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2618 if (CPUMIsGuestInLongModeEx(pMixedCtx))
2619 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2620 else
2621 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2622
2623 /*
2624 * The following should not be set (since we're not in SMM mode):
2625 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2626 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2627 */
2628
2629 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2630 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2631 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2632
2633 if ((val & zap) != val)
2634 {
2635 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2636 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
2637 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
2638 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2639 }
2640
2641 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2642 AssertRCReturn(rc, rc);
2643
2644 /* Update VCPU with the currently set VM-exit controls. */
2645 pVCpu->hm.s.vmx.u32EntryCtls = val;
2646 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2647 }
2648 return rc;
2649}
2650
2651
2652/**
2653 * Sets up the VM-exit controls in the VMCS.
2654 *
2655 * @returns VBox status code.
2656 * @param pVM Pointer to the VM.
2657 * @param pVCpu Pointer to the VMCPU.
2658 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2659 * out-of-sync. Make sure to update the required fields
2660 * before using them.
2661 *
2662 * @remarks requires EFER.
2663 */
2664DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2665{
2666 int rc = VINF_SUCCESS;
2667 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2668 {
2669 PVM pVM = pVCpu->CTX_SUFF(pVM);
2670 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2671 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2672
2673 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2674 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2675
2676 /*
2677 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2678 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2679 */
2680#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2681 if (HMVMX_IS_64BIT_HOST_MODE())
2682 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2683 else
2684 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2685#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2686 if (CPUMIsGuestInLongModeEx(pMixedCtx))
2687 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2688 else
2689 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2690#endif
2691
2692 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2693 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2694
2695 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2696 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2697 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2698 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2699 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2700
2701 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2702 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2703
2704 if ((val & zap) != val)
2705 {
2706 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2707 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
2708 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
2709 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2710 }
2711
2712 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2713 AssertRCReturn(rc, rc);
2714
2715 /* Update VCPU with the currently set VM-exit controls. */
2716 pVCpu->hm.s.vmx.u32ExitCtls = val;
2717 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2718 }
2719 return rc;
2720}
2721
2722
2723/**
2724 * Loads the guest APIC and related state.
2725 *
2726 * @returns VBox status code.
2727 * @param pVM Pointer to the VM.
2728 * @param pVCpu Pointer to the VMCPU.
2729 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2730 * out-of-sync. Make sure to update the required fields
2731 * before using them.
2732 */
2733DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2734{
2735 int rc = VINF_SUCCESS;
2736 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2737 {
2738 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2739 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2740 {
2741 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2742
2743 bool fPendingIntr = false;
2744 uint8_t u8Tpr = 0;
2745 uint8_t u8PendingIntr = 0;
2746 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2747 AssertRCReturn(rc, rc);
2748
2749 /*
2750 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
2751 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
2752 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2753 * the interrupt when we VM-exit for other reasons.
2754 */
2755 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2756 uint32_t u32TprThreshold = 0;
2757 if (fPendingIntr)
2758 {
2759 /* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2760 const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
2761 const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
2762 if (u8PendingPriority <= u8TprPriority)
2763 u32TprThreshold = u8PendingPriority;
2764 else
2765 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
2766 }
2767 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2768
2769 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2770 AssertRCReturn(rc, rc);
2771 }
2772
2773 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2774 }
2775 return rc;
2776}
2777
2778
2779/**
2780 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2781 *
2782 * @returns Guest's interruptibility-state.
2783 * @param pVCpu Pointer to the VMCPU.
2784 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2785 * out-of-sync. Make sure to update the required fields
2786 * before using them.
2787 *
2788 * @remarks No-long-jump zone!!!
2789 * @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2790 */
2791DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2792{
2793 /*
2794 * Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2795 * inhibit interrupts or clear any existing interrupt-inhibition.
2796 */
2797 uint32_t uIntrState = 0;
2798 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2799 {
2800 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2801 AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS))
2802 == (HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2803 if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2804 {
2805 /*
2806 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2807 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
2808 */
2809 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2810 }
2811 else if (pMixedCtx->eflags.Bits.u1IF)
2812 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2813 else
2814 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2815 }
2816 return uIntrState;
2817}
2818
2819
2820/**
2821 * Loads the guest's interruptibility-state into the guest-state area in the
2822 * VMCS.
2823 *
2824 * @returns VBox status code.
2825 * @param pVCpu Pointer to the VMCPU.
2826 * @param uIntrState The interruptibility-state to set.
2827 */
2828static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2829{
2830 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2831 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2832 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2833 AssertRCReturn(rc, rc);
2834 return rc;
2835}
2836
2837
2838/**
2839 * Loads the guest's RIP into the guest-state area in the VMCS.
2840 *
2841 * @returns VBox status code.
2842 * @param pVCpu Pointer to the VMCPU.
2843 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2844 * out-of-sync. Make sure to update the required fields
2845 * before using them.
2846 *
2847 * @remarks No-long-jump zone!!!
2848 */
2849static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2850{
2851 int rc = VINF_SUCCESS;
2852 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2853 {
2854 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2855 AssertRCReturn(rc, rc);
2856 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2857 Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#x\n", pMixedCtx->rip, pVCpu->hm.s.fContextUseFlags));
2858 }
2859 return rc;
2860}
2861
2862
2863/**
2864 * Loads the guest's RSP into the guest-state area in the VMCS.
2865 *
2866 * @returns VBox status code.
2867 * @param pVCpu Pointer to the VMCPU.
2868 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2869 * out-of-sync. Make sure to update the required fields
2870 * before using them.
2871 *
2872 * @remarks No-long-jump zone!!!
2873 */
2874static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2875{
2876 int rc = VINF_SUCCESS;
2877 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2878 {
2879 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2880 AssertRCReturn(rc, rc);
2881 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2882 Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
2883 }
2884 return rc;
2885}
2886
2887
2888/**
2889 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
2890 *
2891 * @returns VBox status code.
2892 * @param pVCpu Pointer to the VMCPU.
2893 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2894 * out-of-sync. Make sure to update the required fields
2895 * before using them.
2896 *
2897 * @remarks No-long-jump zone!!!
2898 */
2899static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2900{
2901 int rc = VINF_SUCCESS;
2902 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2903 {
2904 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2905 Let us assert it as such and use 32-bit VMWRITE. */
2906 Assert(!(pMixedCtx->rflags.u64 >> 32));
2907 X86EFLAGS Eflags = pMixedCtx->eflags;
2908 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2909 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2910
2911 /*
2912 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2913 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2914 */
2915 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2916 {
2917 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2918 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2919 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
2920 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2921 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2922 }
2923
2924 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
2925 AssertRCReturn(rc, rc);
2926
2927 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2928 Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
2929 }
2930 return rc;
2931}
2932
2933
2934/**
2935 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2936 *
2937 * @returns VBox status code.
2938 * @param pVCpu Pointer to the VMCPU.
2939 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2940 * out-of-sync. Make sure to update the required fields
2941 * before using them.
2942 *
2943 * @remarks No-long-jump zone!!!
2944 */
2945DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2946{
2947 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2948 AssertRCReturn(rc, rc);
2949 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2950 AssertRCReturn(rc, rc);
2951 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2952 AssertRCReturn(rc, rc);
2953 return rc;
2954}
2955
2956
2957/**
2958 * Loads the guest CR0 control register into the guest-state area in the VMCS.
2959 * CR0 is partially shared with the host and we have to consider the FPU bits.
2960 *
2961 * @returns VBox status code.
2962 * @param pVM Pointer to the VM.
2963 * @param pVCpu Pointer to the VMCPU.
2964 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2965 * out-of-sync. Make sure to update the required fields
2966 * before using them.
2967 *
2968 * @remarks No-long-jump zone!!!
2969 */
2970static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2971{
2972 /*
2973 * Guest CR0.
2974 * Guest FPU.
2975 */
2976 int rc = VINF_SUCCESS;
2977 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2978 {
2979 Assert(!(pMixedCtx->cr0 >> 32));
2980 uint32_t u32GuestCR0 = pMixedCtx->cr0;
2981 PVM pVM = pVCpu->CTX_SUFF(pVM);
2982
2983 /* The guest's view (read access) of its CR0 is unblemished. */
2984 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2985 AssertRCReturn(rc, rc);
2986 Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2987
2988 /* Setup VT-x's view of the guest CR0. */
2989 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2990 if (pVM->hm.s.fNestedPaging)
2991 {
2992 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
2993 {
2994 /* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2995 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2996 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
2997 }
2998 else
2999 {
3000 /* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
3001 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3002 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3003 }
3004
3005 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3006 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3007 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3008
3009 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3010 AssertRCReturn(rc, rc);
3011 }
3012 else
3013 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3014
3015 /*
3016 * Guest FPU bits.
3017 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3018 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3019 */
3020 u32GuestCR0 |= X86_CR0_NE;
3021 bool fInterceptNM = false;
3022 if (CPUMIsGuestFPUStateActive(pVCpu))
3023 {
3024 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3025 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3026 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3027 }
3028 else
3029 {
3030 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3031 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3032 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3033 }
3034
3035 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3036 bool fInterceptMF = false;
3037 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3038 fInterceptMF = true;
3039
3040 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3041 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3042 {
3043 Assert(PDMVmmDevHeapIsEnabled(pVM));
3044 Assert(pVM->hm.s.vmx.pRealModeTSS);
3045 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3046 fInterceptNM = true;
3047 fInterceptMF = true;
3048 }
3049 else
3050 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3051
3052 if (fInterceptNM)
3053 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3054 else
3055 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3056
3057 if (fInterceptMF)
3058 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3059 else
3060 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3061
3062 /* Additional intercepts for debugging, define these yourself explicitly. */
3063#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3064 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3065 | RT_BIT(X86_XCPT_BP)
3066 | RT_BIT(X86_XCPT_DB)
3067 | RT_BIT(X86_XCPT_DE)
3068 | RT_BIT(X86_XCPT_NM)
3069 | RT_BIT(X86_XCPT_UD)
3070 | RT_BIT(X86_XCPT_NP)
3071 | RT_BIT(X86_XCPT_SS)
3072 | RT_BIT(X86_XCPT_GP)
3073 | RT_BIT(X86_XCPT_PF)
3074 | RT_BIT(X86_XCPT_MF)
3075 ;
3076#elif defined(HMVMX_ALWAYS_TRAP_PF)
3077 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3078#endif
3079
3080 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3081
3082 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3083 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3084 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3085 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3086 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3087 else
3088 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3089
3090 u32GuestCR0 |= uSetCR0;
3091 u32GuestCR0 &= uZapCR0;
3092 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3093
3094 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3095 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3096 AssertRCReturn(rc, rc);
3097 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3098 AssertRCReturn(rc, rc);
3099 Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3100
3101 /*
3102 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3103 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3104 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3105 */
3106 uint32_t u32CR0Mask = 0;
3107 u32CR0Mask = X86_CR0_PE
3108 | X86_CR0_NE
3109 | X86_CR0_WP
3110 | X86_CR0_PG
3111 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3112 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3113 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3114 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3115 u32CR0Mask &= ~X86_CR0_PE;
3116 if (pVM->hm.s.fNestedPaging)
3117 u32CR0Mask &= ~X86_CR0_WP;
3118
3119 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3120 if (fInterceptNM)
3121 u32CR0Mask |= (X86_CR0_TS | X86_CR0_MP);
3122 else
3123 u32CR0Mask &= ~(X86_CR0_TS | X86_CR0_MP);
3124
3125 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3126 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3127 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3128 AssertRCReturn(rc, rc);
3129
3130 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
3131 }
3132 return rc;
3133}
3134
3135
3136/**
3137 * Loads the guest control registers (CR3, CR4) into the guest-state area
3138 * in the VMCS.
3139 *
3140 * @returns VBox status code.
3141 * @param pVM Pointer to the VM.
3142 * @param pVCpu Pointer to the VMCPU.
3143 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3144 * out-of-sync. Make sure to update the required fields
3145 * before using them.
3146 *
3147 * @remarks No-long-jump zone!!!
3148 */
3149static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3150{
3151 int rc = VINF_SUCCESS;
3152 PVM pVM = pVCpu->CTX_SUFF(pVM);
3153
3154 /*
3155 * Guest CR2.
3156 * It's always loaded in the assembler code. Nothing to do here.
3157 */
3158
3159 /*
3160 * Guest CR3.
3161 */
3162 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
3163 {
3164 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3165 if (pVM->hm.s.fNestedPaging)
3166 {
3167 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3168
3169 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3170 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3171 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3172 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3173
3174 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3175 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3176 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3177
3178 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3179 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3180 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3181 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3182
3183 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3184 AssertRCReturn(rc, rc);
3185 Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3186
3187 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3188 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3189 {
3190 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3191 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3192 {
3193 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3194 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3195 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3196 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3197 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3198 }
3199
3200 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3201 have Unrestricted Execution to handle the guest when it's not using paging. */
3202 GCPhysGuestCR3 = pMixedCtx->cr3;
3203 }
3204 else
3205 {
3206 /*
3207 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3208 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3209 * EPT takes care of translating it to host-physical addresses.
3210 */
3211 RTGCPHYS GCPhys;
3212 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3213 Assert(PDMVmmDevHeapIsEnabled(pVM));
3214
3215 /* We obtain it here every time as the guest could have relocated this PCI region. */
3216 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3217 AssertRCReturn(rc, rc);
3218
3219 GCPhysGuestCR3 = GCPhys;
3220 }
3221
3222 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3223 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3224 }
3225 else
3226 {
3227 /* Non-nested paging case, just use the hypervisor's CR3. */
3228 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3229
3230 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3231 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3232 }
3233 AssertRCReturn(rc, rc);
3234
3235 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
3236 }
3237
3238 /*
3239 * Guest CR4.
3240 */
3241 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
3242 {
3243 Assert(!(pMixedCtx->cr4 >> 32));
3244 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3245
3246 /* The guest's view of its CR4 is unblemished. */
3247 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3248 AssertRCReturn(rc, rc);
3249 Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3250
3251 /* Setup VT-x's view of the guest CR4. */
3252 /*
3253 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3254 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3255 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3256 */
3257 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3258 {
3259 Assert(pVM->hm.s.vmx.pRealModeTSS);
3260 Assert(PDMVmmDevHeapIsEnabled(pVM));
3261 u32GuestCR4 &= ~X86_CR4_VME;
3262 }
3263
3264 if (pVM->hm.s.fNestedPaging)
3265 {
3266 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3267 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3268 {
3269 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3270 u32GuestCR4 |= X86_CR4_PSE;
3271 /* Our identity mapping is a 32 bits page directory. */
3272 u32GuestCR4 &= ~X86_CR4_PAE;
3273 }
3274 /* else use guest CR4.*/
3275 }
3276 else
3277 {
3278 /*
3279 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3280 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3281 */
3282 switch (pVCpu->hm.s.enmShadowMode)
3283 {
3284 case PGMMODE_REAL: /* Real-mode. */
3285 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3286 case PGMMODE_32_BIT: /* 32-bit paging. */
3287 {
3288 u32GuestCR4 &= ~X86_CR4_PAE;
3289 break;
3290 }
3291
3292 case PGMMODE_PAE: /* PAE paging. */
3293 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3294 {
3295 u32GuestCR4 |= X86_CR4_PAE;
3296 break;
3297 }
3298
3299 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3300 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3301#ifdef VBOX_ENABLE_64_BITS_GUESTS
3302 break;
3303#endif
3304 default:
3305 AssertFailed();
3306 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3307 }
3308 }
3309
3310 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3311 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3312 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3313 u32GuestCR4 |= uSetCR4;
3314 u32GuestCR4 &= uZapCR4;
3315
3316 /* Write VT-x's view of the guest CR4 into the VMCS. */
3317 Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3318 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3319 AssertRCReturn(rc, rc);
3320
3321 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3322 uint32_t u32CR4Mask = 0;
3323 u32CR4Mask = X86_CR4_VME
3324 | X86_CR4_PAE
3325 | X86_CR4_PGE
3326 | X86_CR4_PSE
3327 | X86_CR4_VMXE;
3328 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3329 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3330 AssertRCReturn(rc, rc);
3331
3332 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3333 }
3334 return rc;
3335}
3336
3337
3338/**
3339 * Loads the guest debug registers into the guest-state area in the VMCS.
3340 * This also sets up whether #DB and MOV DRx accesses cause VM exits.
3341 *
3342 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3343 *
3344 * @returns VBox status code.
3345 * @param pVCpu Pointer to the VMCPU.
3346 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3347 * out-of-sync. Make sure to update the required fields
3348 * before using them.
3349 *
3350 * @remarks No-long-jump zone!!!
3351 */
3352static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3353{
3354 if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3355 return VINF_SUCCESS;
3356
3357#ifdef VBOX_STRICT
3358 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3359 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3360 {
3361 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3362 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3363 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3364 }
3365#endif
3366
3367 int rc;
3368 PVM pVM = pVCpu->CTX_SUFF(pVM);
3369 bool fInterceptDB = false;
3370 bool fInterceptMovDRx = false;
3371 if (pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu))
3372 {
3373 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3374 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3375 {
3376 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3377 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3378 AssertRCReturn(rc, rc);
3379 Assert(fInterceptDB == false);
3380 }
3381 else
3382 {
3383 pMixedCtx->eflags.u32 |= X86_EFL_TF;
3384 pVCpu->hm.s.fClearTrapFlag = true;
3385 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RFLAGS;
3386 fInterceptDB = true;
3387 }
3388 }
3389
3390 if (fInterceptDB || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3391 {
3392 /*
3393 * Use the combined guest and host DRx values found in the hypervisor
3394 * register set because the debugger has breakpoints active or someone
3395 * is single stepping on the host side without a monitor trap flag.
3396 *
3397 * Note! DBGF expects a clean DR6 state before executing guest code.
3398 */
3399 if (!CPUMIsHyperDebugStateActive(pVCpu))
3400 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3401 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3402 Assert(CPUMIsHyperDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
3403
3404 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3405 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3406 AssertRCReturn(rc, rc);
3407
3408 fInterceptDB = true;
3409 fInterceptMovDRx = true;
3410 }
3411 else
3412 {
3413 /*
3414 * If the guest has enabled debug registers, we need to load them prior to
3415 * executing guest code so they'll trigger at the right time.
3416 */
3417 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
3418 {
3419 if (!CPUMIsGuestDebugStateActive(pVCpu))
3420 {
3421 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3422 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3423 }
3424 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3425 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
3426 }
3427 /*
3428 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3429 * must intercept #DB in order to maintain a correct DR6 guest value.
3430 */
3431 else if (!CPUMIsGuestDebugStateActive(pVCpu))
3432 {
3433 fInterceptMovDRx = true;
3434 fInterceptDB = true;
3435 }
3436
3437 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3438 AssertRCReturn(rc, rc);
3439 }
3440
3441 /*
3442 * Update the exception bitmap regarding intercepting #DB generated by the guest.
3443 */
3444 if (fInterceptDB)
3445 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
3446 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3447 {
3448#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3449 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3450#endif
3451 }
3452 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3453 AssertRCReturn(rc, rc);
3454
3455 /*
3456 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3457 */
3458 if (fInterceptMovDRx)
3459 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3460 else
3461 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3462 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3463 AssertRCReturn(rc, rc);
3464
3465 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3466 return VINF_SUCCESS;
3467}
3468
3469
3470#ifdef VBOX_STRICT
3471/**
3472 * Strict function to validate segment registers.
3473 *
3474 * @remarks ASSUMES CR0 is up to date.
3475 */
3476static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3477{
3478 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3479 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3480 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3481 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3482 && ( !CPUMIsGuestInRealModeEx(pCtx)
3483 && !CPUMIsGuestInV86ModeEx(pCtx)))
3484 {
3485 /* Protected mode checks */
3486 /* CS */
3487 Assert(pCtx->cs.Attr.n.u1Present);
3488 Assert(!(pCtx->cs.Attr.u & 0xf00));
3489 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3490 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3491 || !(pCtx->cs.Attr.n.u1Granularity));
3492 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3493 || (pCtx->cs.Attr.n.u1Granularity));
3494 /* CS cannot be loaded with NULL in protected mode. */
3495 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3496 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
3497 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3498 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
3499 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3500 else
3501 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3502 /* SS */
3503 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3504 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3505 if ( !(pCtx->cr0 & X86_CR0_PE)
3506 || pCtx->cs.Attr.n.u4Type == 3)
3507 {
3508 Assert(!pCtx->ss.Attr.n.u2Dpl);
3509 }
3510 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
3511 {
3512 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3513 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
3514 Assert(pCtx->ss.Attr.n.u1Present);
3515 Assert(!(pCtx->ss.Attr.u & 0xf00));
3516 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3517 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3518 || !(pCtx->ss.Attr.n.u1Granularity));
3519 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3520 || (pCtx->ss.Attr.n.u1Granularity));
3521 }
3522 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3523 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
3524 {
3525 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3526 Assert(pCtx->ds.Attr.n.u1Present);
3527 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3528 Assert(!(pCtx->ds.Attr.u & 0xf00));
3529 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3530 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3531 || !(pCtx->ds.Attr.n.u1Granularity));
3532 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3533 || (pCtx->ds.Attr.n.u1Granularity));
3534 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3535 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3536 }
3537 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
3538 {
3539 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3540 Assert(pCtx->es.Attr.n.u1Present);
3541 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3542 Assert(!(pCtx->es.Attr.u & 0xf00));
3543 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3544 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3545 || !(pCtx->es.Attr.n.u1Granularity));
3546 Assert( !(pCtx->es.u32Limit & 0xfff00000)
3547 || (pCtx->es.Attr.n.u1Granularity));
3548 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3549 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3550 }
3551 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
3552 {
3553 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3554 Assert(pCtx->fs.Attr.n.u1Present);
3555 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3556 Assert(!(pCtx->fs.Attr.u & 0xf00));
3557 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3558 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3559 || !(pCtx->fs.Attr.n.u1Granularity));
3560 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3561 || (pCtx->fs.Attr.n.u1Granularity));
3562 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3563 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3564 }
3565 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
3566 {
3567 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3568 Assert(pCtx->gs.Attr.n.u1Present);
3569 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3570 Assert(!(pCtx->gs.Attr.u & 0xf00));
3571 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3572 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3573 || !(pCtx->gs.Attr.n.u1Granularity));
3574 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3575 || (pCtx->gs.Attr.n.u1Granularity));
3576 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3577 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3578 }
3579 /* 64-bit capable CPUs. */
3580# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3581 Assert(!(pCtx->cs.u64Base >> 32));
3582 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
3583 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
3584 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
3585# endif
3586 }
3587 else if ( CPUMIsGuestInV86ModeEx(pCtx)
3588 || ( CPUMIsGuestInRealModeEx(pCtx)
3589 && !pVM->hm.s.vmx.fUnrestrictedGuest))
3590 {
3591 /* Real and v86 mode checks. */
3592 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3593 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3594 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3595 {
3596 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3597 }
3598 else
3599 {
3600 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3601 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3602 }
3603
3604 /* CS */
3605 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3606 Assert(pCtx->cs.u32Limit == 0xffff);
3607 Assert(u32CSAttr == 0xf3);
3608 /* SS */
3609 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3610 Assert(pCtx->ss.u32Limit == 0xffff);
3611 Assert(u32SSAttr == 0xf3);
3612 /* DS */
3613 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3614 Assert(pCtx->ds.u32Limit == 0xffff);
3615 Assert(u32DSAttr == 0xf3);
3616 /* ES */
3617 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3618 Assert(pCtx->es.u32Limit == 0xffff);
3619 Assert(u32ESAttr == 0xf3);
3620 /* FS */
3621 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3622 Assert(pCtx->fs.u32Limit == 0xffff);
3623 Assert(u32FSAttr == 0xf3);
3624 /* GS */
3625 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3626 Assert(pCtx->gs.u32Limit == 0xffff);
3627 Assert(u32GSAttr == 0xf3);
3628 /* 64-bit capable CPUs. */
3629# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3630 Assert(!(pCtx->cs.u64Base >> 32));
3631 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
3632 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
3633 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
3634# endif
3635 }
3636}
3637#endif /* VBOX_STRICT */
3638
3639
3640/**
3641 * Writes a guest segment register into the guest-state area in the VMCS.
3642 *
3643 * @returns VBox status code.
3644 * @param pVCpu Pointer to the VMCPU.
3645 * @param idxSel Index of the selector in the VMCS.
3646 * @param idxLimit Index of the segment limit in the VMCS.
3647 * @param idxBase Index of the segment base in the VMCS.
3648 * @param idxAccess Index of the access rights of the segment in the VMCS.
3649 * @param pSelReg Pointer to the segment selector.
3650 * @param pCtx Pointer to the guest-CPU context.
3651 *
3652 * @remarks No-long-jump zone!!!
3653 */
3654static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3655 uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3656{
3657 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3658 AssertRCReturn(rc, rc);
3659 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3660 AssertRCReturn(rc, rc);
3661 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3662 AssertRCReturn(rc, rc);
3663
3664 uint32_t u32Access = pSelReg->Attr.u;
3665 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3666 {
3667 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3668 u32Access = 0xf3;
3669 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3670 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3671 }
3672 else
3673 {
3674 /*
3675 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3676 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3677 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3678 * loaded in protected-mode have their attribute as 0.
3679 */
3680 if (!u32Access)
3681 u32Access = X86DESCATTR_UNUSABLE;
3682 }
3683
3684 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3685 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
3686 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3687
3688 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3689 AssertRCReturn(rc, rc);
3690 return rc;
3691}
3692
3693
3694/**
3695 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3696 * into the guest-state area in the VMCS.
3697 *
3698 * @returns VBox status code.
3699 * @param pVM Pointer to the VM.
3700 * @param pVCPU Pointer to the VMCPU.
3701 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3702 * out-of-sync. Make sure to update the required fields
3703 * before using them.
3704 *
3705 * @remarks ASSUMES CR0 is up to date (strict builds validation).
3706 * @remarks No-long-jump zone!!!
3707 */
3708static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3709{
3710 int rc = VERR_INTERNAL_ERROR_5;
3711 PVM pVM = pVCpu->CTX_SUFF(pVM);
3712
3713 /*
3714 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
3715 */
3716 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3717 {
3718 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3719 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3720 {
3721 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
3722 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
3723 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
3724 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
3725 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
3726 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
3727 }
3728
3729#ifdef VBOX_WITH_REM
3730 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3731 {
3732 Assert(pVM->hm.s.vmx.pRealModeTSS);
3733 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3734 if ( pVCpu->hm.s.vmx.fWasInRealMode
3735 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3736 {
3737 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3738 in real-mode (e.g. OpenBSD 4.0) */
3739 REMFlushTBs(pVM);
3740 Log4(("Load: Switch to protected mode detected!\n"));
3741 pVCpu->hm.s.vmx.fWasInRealMode = false;
3742 }
3743 }
3744#endif
3745 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3746 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3747 AssertRCReturn(rc, rc);
3748 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3749 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3750 AssertRCReturn(rc, rc);
3751 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3752 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3753 AssertRCReturn(rc, rc);
3754 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3755 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3756 AssertRCReturn(rc, rc);
3757 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3758 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3759 AssertRCReturn(rc, rc);
3760 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3761 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3762 AssertRCReturn(rc, rc);
3763
3764 Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
3765 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
3766#ifdef VBOX_STRICT
3767 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3768#endif
3769 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3770 }
3771
3772 /*
3773 * Guest TR.
3774 */
3775 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3776 {
3777 /*
3778 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3779 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3780 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3781 */
3782 uint16_t u16Sel = 0;
3783 uint32_t u32Limit = 0;
3784 uint64_t u64Base = 0;
3785 uint32_t u32AccessRights = 0;
3786
3787 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3788 {
3789 u16Sel = pMixedCtx->tr.Sel;
3790 u32Limit = pMixedCtx->tr.u32Limit;
3791 u64Base = pMixedCtx->tr.u64Base;
3792 u32AccessRights = pMixedCtx->tr.Attr.u;
3793 }
3794 else
3795 {
3796 Assert(pVM->hm.s.vmx.pRealModeTSS);
3797 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3798
3799 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3800 RTGCPHYS GCPhys;
3801 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3802 AssertRCReturn(rc, rc);
3803
3804 X86DESCATTR DescAttr;
3805 DescAttr.u = 0;
3806 DescAttr.n.u1Present = 1;
3807 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3808
3809 u16Sel = 0;
3810 u32Limit = HM_VTX_TSS_SIZE;
3811 u64Base = GCPhys; /* in real-mode phys = virt. */
3812 u32AccessRights = DescAttr.u;
3813 }
3814
3815 /* Validate. */
3816 Assert(!(u16Sel & RT_BIT(2)));
3817 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3818 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3819 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3820 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3821 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3822 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3823 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3824 Assert( (u32Limit & 0xfff) == 0xfff
3825 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3826 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3827 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3828
3829 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3830 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3831 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3832 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3833
3834 Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3835 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3836 }
3837
3838 /*
3839 * Guest GDTR.
3840 */
3841 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3842 {
3843 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3844 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3845
3846 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3847 Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3848 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3849 }
3850
3851 /*
3852 * Guest LDTR.
3853 */
3854 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3855 {
3856 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3857 uint32_t u32Access = 0;
3858 if (!pMixedCtx->ldtr.Attr.u)
3859 u32Access = X86DESCATTR_UNUSABLE;
3860 else
3861 u32Access = pMixedCtx->ldtr.Attr.u;
3862
3863 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3864 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3865 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3866 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3867
3868 /* Validate. */
3869 if (!(u32Access & X86DESCATTR_UNUSABLE))
3870 {
3871 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3872 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3873 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3874 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3875 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3876 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3877 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3878 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3879 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3880 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3881 }
3882
3883 Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3884 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3885 }
3886
3887 /*
3888 * Guest IDTR.
3889 */
3890 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3891 {
3892 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
3893 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
3894
3895 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3896 Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3897 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3898 }
3899
3900 return VINF_SUCCESS;
3901}
3902
3903
3904/**
3905 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3906 * areas. These MSRs will automatically be loaded to the host CPU on every
3907 * successful VM entry and stored from the host CPU on every successful VM exit.
3908 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
3909 *
3910 * @returns VBox status code.
3911 * @param pVCpu Pointer to the VMCPU.
3912 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3913 * out-of-sync. Make sure to update the required fields
3914 * before using them.
3915 *
3916 * @remarks No-long-jump zone!!!
3917 */
3918static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3919{
3920 AssertPtr(pVCpu);
3921 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3922
3923 /*
3924 * MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3925 */
3926 int rc = VINF_SUCCESS;
3927 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3928 {
3929#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3930 PVM pVM = pVCpu->CTX_SUFF(pVM);
3931 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3932 uint32_t cGuestMsrs = 0;
3933
3934 /* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3935 /** @todo r=ramshankar: Optimize this further to do lazy restoration and only
3936 * when the guest really is in 64-bit mode. */
3937 bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3938 if (fSupportsLongMode)
3939 {
3940 pGuestMsr->u32Msr = MSR_K8_LSTAR;
3941 pGuestMsr->u32Reserved = 0;
3942 pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3943 pGuestMsr++; cGuestMsrs++;
3944 pGuestMsr->u32Msr = MSR_K6_STAR;
3945 pGuestMsr->u32Reserved = 0;
3946 pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3947 pGuestMsr++; cGuestMsrs++;
3948 pGuestMsr->u32Msr = MSR_K8_SF_MASK;
3949 pGuestMsr->u32Reserved = 0;
3950 pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3951 pGuestMsr++; cGuestMsrs++;
3952 pGuestMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
3953 pGuestMsr->u32Reserved = 0;
3954 pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3955 pGuestMsr++; cGuestMsrs++;
3956 }
3957
3958 /*
3959 * RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3960 * load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3961 */
3962 if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3963 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3964 {
3965 pGuestMsr->u32Msr = MSR_K8_TSC_AUX;
3966 pGuestMsr->u32Reserved = 0;
3967 rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3968 AssertRCReturn(rc, rc);
3969 pGuestMsr++; cGuestMsrs++;
3970 }
3971
3972 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3973 if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc))
3974 {
3975 LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3976 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
3977 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3978 }
3979
3980 /* Update the VCPU's copy of the guest MSR count. */
3981 pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3982 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3983 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3984#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
3985
3986 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3987 }
3988
3989 /*
3990 * Guest Sysenter MSRs.
3991 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3992 * VM-exits on WRMSRs for these MSRs.
3993 */
3994 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3995 {
3996 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
3997 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3998 }
3999 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4000 {
4001 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4002 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
4003 }
4004 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4005 {
4006 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4007 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
4008 }
4009
4010 return rc;
4011}
4012
4013
4014/**
4015 * Loads the guest activity state into the guest-state area in the VMCS.
4016 *
4017 * @returns VBox status code.
4018 * @param pVCpu Pointer to the VMCPU.
4019 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4020 * out-of-sync. Make sure to update the required fields
4021 * before using them.
4022 *
4023 * @remarks No-long-jump zone!!!
4024 */
4025static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4026{
4027 /** @todo See if we can make use of other states, e.g.
4028 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4029 int rc = VINF_SUCCESS;
4030 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
4031 {
4032 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4033 AssertRCReturn(rc, rc);
4034 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
4035 }
4036 return rc;
4037}
4038
4039
4040/**
4041 * Sets up the appropriate function to run guest code.
4042 *
4043 * @returns VBox status code.
4044 * @param pVCpu Pointer to the VMCPU.
4045 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4046 * out-of-sync. Make sure to update the required fields
4047 * before using them.
4048 *
4049 * @remarks No-long-jump zone!!!
4050 */
4051static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4052{
4053 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4054 {
4055#ifndef VBOX_ENABLE_64_BITS_GUESTS
4056 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4057#endif
4058 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4059#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4060 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4061 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4062#else
4063 /* 64-bit host or hybrid host. */
4064 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4065#endif
4066 }
4067 else
4068 {
4069 /* Guest is not in long mode, use the 32-bit handler. */
4070#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4071 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4072 {
4073 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4074 /** @todo r=bird: Don't we need to set up the host resume (after
4075 * vmlaunch/vmresume) state here?? I'm forcing a trip to ring-3 now
4076 * in the hope that it will prevent crashing the host. A better
4077 * fix should be found as the guest may be going back and forth
4078 * between 16/32-bit and long mode frequently at times. */
4079 VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3);
4080 }
4081#else
4082 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4083#endif
4084 }
4085 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4086 return VINF_SUCCESS;
4087}
4088
4089
4090/**
4091 * Wrapper for running the guest code in VT-x.
4092 *
4093 * @returns VBox strict status code.
4094 * @param pVM Pointer to the VM.
4095 * @param pVCpu Pointer to the VMCPU.
4096 * @param pCtx Pointer to the guest-CPU context.
4097 *
4098 * @remarks No-long-jump zone!!!
4099 */
4100DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4101{
4102 /*
4103 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4104 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4105 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4106 */
4107 const bool fResumeVM = !!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4108 /** @todo Add stats for resume vs launch. */
4109#ifdef VBOX_WITH_KERNEL_USING_XMM
4110 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4111#else
4112 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4113#endif
4114}
4115
4116
4117/**
4118 * Reports world-switch error and dumps some useful debug info.
4119 *
4120 * @param pVM Pointer to the VM.
4121 * @param pVCpu Pointer to the VMCPU.
4122 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4123 * @param pCtx Pointer to the guest-CPU context.
4124 * @param pVmxTransient Pointer to the VMX transient structure (only
4125 * exitReason updated).
4126 */
4127static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4128{
4129 Assert(pVM);
4130 Assert(pVCpu);
4131 Assert(pCtx);
4132 Assert(pVmxTransient);
4133 HMVMX_ASSERT_PREEMPT_SAFE();
4134
4135 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4136 switch (rcVMRun)
4137 {
4138 case VERR_VMX_INVALID_VMXON_PTR:
4139 AssertFailed();
4140 break;
4141 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4142 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4143 {
4144 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4145 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4146 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4147 AssertRC(rc);
4148
4149 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4150 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4151 Cannot do it here as we may have been long preempted. */
4152
4153#ifdef VBOX_STRICT
4154 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4155 pVmxTransient->uExitReason));
4156 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4157 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4158 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4159 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4160 else
4161 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4162 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4163 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4164
4165 /* VMX control bits. */
4166 uint32_t u32Val;
4167 uint64_t u64Val;
4168 HMVMXHCUINTREG uHCReg;
4169 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4170 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4171 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4172 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4173 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4174 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4175 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4176 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4177 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4178 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4179 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4180 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4181 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4182 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4183 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4184 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4185 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4186 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4187 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4188 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4189 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4190 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4191 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4192 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4193 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4194 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4195 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4196 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4197 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4198 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4199 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4200 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4201 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4202 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4203 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4204 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4205 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4206 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4207 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4208 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4209 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4210 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4211
4212 /* Guest bits. */
4213 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4214 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4215 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4216 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4217 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4218 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4219 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4220 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4221
4222 /* Host bits. */
4223 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4224 Log4(("Host CR0 %#RHr\n", uHCReg));
4225 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4226 Log4(("Host CR3 %#RHr\n", uHCReg));
4227 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4228 Log4(("Host CR4 %#RHr\n", uHCReg));
4229
4230 RTGDTR HostGdtr;
4231 PCX86DESCHC pDesc;
4232 ASMGetGDTR(&HostGdtr);
4233 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4234 Log4(("Host CS %#08x\n", u32Val));
4235 if (u32Val < HostGdtr.cbGdt)
4236 {
4237 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4238 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4239 }
4240
4241 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4242 Log4(("Host DS %#08x\n", u32Val));
4243 if (u32Val < HostGdtr.cbGdt)
4244 {
4245 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4246 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4247 }
4248
4249 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4250 Log4(("Host ES %#08x\n", u32Val));
4251 if (u32Val < HostGdtr.cbGdt)
4252 {
4253 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4254 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4255 }
4256
4257 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4258 Log4(("Host FS %#08x\n", u32Val));
4259 if (u32Val < HostGdtr.cbGdt)
4260 {
4261 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4262 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4263 }
4264
4265 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4266 Log4(("Host GS %#08x\n", u32Val));
4267 if (u32Val < HostGdtr.cbGdt)
4268 {
4269 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4270 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4271 }
4272
4273 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4274 Log4(("Host SS %#08x\n", u32Val));
4275 if (u32Val < HostGdtr.cbGdt)
4276 {
4277 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4278 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4279 }
4280
4281 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4282 Log4(("Host TR %#08x\n", u32Val));
4283 if (u32Val < HostGdtr.cbGdt)
4284 {
4285 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4286 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4287 }
4288
4289 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4290 Log4(("Host TR Base %#RHv\n", uHCReg));
4291 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4292 Log4(("Host GDTR Base %#RHv\n", uHCReg));
4293 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4294 Log4(("Host IDTR Base %#RHv\n", uHCReg));
4295 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4296 Log4(("Host SYSENTER CS %#08x\n", u32Val));
4297 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4298 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4299 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4300 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4301 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4302 Log4(("Host RSP %#RHv\n", uHCReg));
4303 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4304 Log4(("Host RIP %#RHv\n", uHCReg));
4305# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4306 if (HMVMX_IS_64BIT_HOST_MODE())
4307 {
4308 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4309 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4310 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4311 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4312 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4313 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4314 }
4315# endif
4316#endif /* VBOX_STRICT */
4317 break;
4318 }
4319
4320 default:
4321 /* Impossible */
4322 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4323 break;
4324 }
4325 NOREF(pVM);
4326}
4327
4328
4329#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4330#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4331# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4332#endif
4333#ifdef VBOX_STRICT
4334static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4335{
4336 switch (idxField)
4337 {
4338 case VMX_VMCS_GUEST_RIP:
4339 case VMX_VMCS_GUEST_RSP:
4340 case VMX_VMCS_GUEST_SYSENTER_EIP:
4341 case VMX_VMCS_GUEST_SYSENTER_ESP:
4342 case VMX_VMCS_GUEST_GDTR_BASE:
4343 case VMX_VMCS_GUEST_IDTR_BASE:
4344 case VMX_VMCS_GUEST_CS_BASE:
4345 case VMX_VMCS_GUEST_DS_BASE:
4346 case VMX_VMCS_GUEST_ES_BASE:
4347 case VMX_VMCS_GUEST_FS_BASE:
4348 case VMX_VMCS_GUEST_GS_BASE:
4349 case VMX_VMCS_GUEST_SS_BASE:
4350 case VMX_VMCS_GUEST_LDTR_BASE:
4351 case VMX_VMCS_GUEST_TR_BASE:
4352 case VMX_VMCS_GUEST_CR3:
4353 return true;
4354 }
4355 return false;
4356}
4357
4358static bool hmR0VmxIsValidReadField(uint32_t idxField)
4359{
4360 switch (idxField)
4361 {
4362 /* Read-only fields. */
4363 case VMX_VMCS_RO_EXIT_QUALIFICATION:
4364 return true;
4365 }
4366 /* Remaining readable fields should also be writable. */
4367 return hmR0VmxIsValidWriteField(idxField);
4368}
4369#endif /* VBOX_STRICT */
4370
4371
4372/**
4373 * Executes the specified handler in 64-bit mode.
4374 *
4375 * @returns VBox status code.
4376 * @param pVM Pointer to the VM.
4377 * @param pVCpu Pointer to the VMCPU.
4378 * @param pCtx Pointer to the guest CPU context.
4379 * @param enmOp The operation to perform.
4380 * @param cbParam Number of parameters.
4381 * @param paParam Array of 32-bit parameters.
4382 */
4383VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4384 uint32_t *paParam)
4385{
4386 int rc, rc2;
4387 PHMGLOBALCPUINFO pCpu;
4388 RTHCPHYS HCPhysCpuPage;
4389 RTCCUINTREG uOldEflags;
4390
4391 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4392 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4393 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4394 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4395
4396#ifdef VBOX_STRICT
4397 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4398 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4399
4400 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4401 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4402#endif
4403
4404 /* Disable interrupts. */
4405 uOldEflags = ASMIntDisableFlags();
4406
4407#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4408 RTCPUID idHostCpu = RTMpCpuId();
4409 CPUMR0SetLApic(pVCpu, idHostCpu);
4410#endif
4411
4412 pCpu = HMR0GetCurrentCpu();
4413 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4414
4415 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4416 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4417
4418 /* Leave VMX Root Mode. */
4419 VMXDisable();
4420
4421 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4422
4423 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4424 CPUMSetHyperEIP(pVCpu, enmOp);
4425 for (int i = (int)cbParam - 1; i >= 0; i--)
4426 CPUMPushHyper(pVCpu, paParam[i]);
4427
4428 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4429
4430 /* Call the switcher. */
4431 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4432 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4433
4434 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4435 /* Make sure the VMX instructions don't cause #UD faults. */
4436 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
4437
4438 /* Re-enter VMX Root Mode */
4439 rc2 = VMXEnable(HCPhysCpuPage);
4440 if (RT_FAILURE(rc2))
4441 {
4442 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4443 ASMSetFlags(uOldEflags);
4444 return rc2;
4445 }
4446
4447 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4448 AssertRC(rc2);
4449 Assert(!(ASMGetFlags() & X86_EFL_IF));
4450 ASMSetFlags(uOldEflags);
4451 return rc;
4452}
4453
4454
4455/**
4456 * Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4457 * supporting 64-bit guests.
4458 *
4459 * @returns VBox status code.
4460 * @param fResume Whether to VMLAUNCH or VMRESUME.
4461 * @param pCtx Pointer to the guest-CPU context.
4462 * @param pCache Pointer to the VMCS cache.
4463 * @param pVM Pointer to the VM.
4464 * @param pVCpu Pointer to the VMCPU.
4465 */
4466DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4467{
4468 uint32_t aParam[6];
4469 PHMGLOBALCPUINFO pCpu = NULL;
4470 RTHCPHYS HCPhysCpuPage = 0;
4471 int rc = VERR_INTERNAL_ERROR_5;
4472
4473 pCpu = HMR0GetCurrentCpu();
4474 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4475
4476#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4477 pCache->uPos = 1;
4478 pCache->interPD = PGMGetInterPaeCR3(pVM);
4479 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4480#endif
4481
4482#ifdef VBOX_STRICT
4483 pCache->TestIn.HCPhysCpuPage = 0;
4484 pCache->TestIn.HCPhysVmcs = 0;
4485 pCache->TestIn.pCache = 0;
4486 pCache->TestOut.HCPhysVmcs = 0;
4487 pCache->TestOut.pCache = 0;
4488 pCache->TestOut.pCtx = 0;
4489 pCache->TestOut.eflags = 0;
4490#endif
4491
4492 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4493 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4494 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4495 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4496 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4497 aParam[5] = 0;
4498
4499#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4500 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4501 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4502#endif
4503 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4504
4505#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4506 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4507 Assert(pCtx->dr[4] == 10);
4508 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4509#endif
4510
4511#ifdef VBOX_STRICT
4512 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4513 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4514 pVCpu->hm.s.vmx.HCPhysVmcs));
4515 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4516 pCache->TestOut.HCPhysVmcs));
4517 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4518 pCache->TestOut.pCache));
4519 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4520 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4521 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4522 pCache->TestOut.pCtx));
4523 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4524#endif
4525 return rc;
4526}
4527
4528
4529/**
4530 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4531 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4532 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4533 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4534 *
4535 * @returns VBox status code.
4536 * @param pVM Pointer to the VM.
4537 * @param pVCpu Pointer to the VMCPU.
4538 */
4539static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4540{
4541#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4542{ \
4543 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4544 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4545 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4546 ++cReadFields; \
4547}
4548
4549 AssertPtr(pVM);
4550 AssertPtr(pVCpu);
4551 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4552 uint32_t cReadFields = 0;
4553
4554 /*
4555 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
4556 * and serve to indicate exceptions to the rules.
4557 */
4558
4559 /* Guest-natural selector base fields. */
4560#if 0
4561 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4562 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4563 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4564#endif
4565 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4566 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4567 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4568 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4569 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4570 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4571 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4572 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4573 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4574 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4575 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4576 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4577#if 0
4578 /* Unused natural width guest-state fields. */
4579 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4580 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4581#endif
4582 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4583 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4584
4585 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4586#if 0
4587 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4588 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4589 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4590 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4591 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4592 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4593 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4594 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4595 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4596#endif
4597
4598 /* Natural width guest-state fields. */
4599 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4600#if 0
4601 /* Currently unused field. */
4602 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4603#endif
4604
4605 if (pVM->hm.s.fNestedPaging)
4606 {
4607 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4608 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4609 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4610 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4611 }
4612 else
4613 {
4614 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4615 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4616 }
4617
4618#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4619 return VINF_SUCCESS;
4620}
4621
4622
4623/**
4624 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4625 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4626 * darwin, running 64-bit guests).
4627 *
4628 * @returns VBox status code.
4629 * @param pVCpu Pointer to the VMCPU.
4630 * @param idxField The VMCS field encoding.
4631 * @param u64Val 16, 32 or 64 bits value.
4632 */
4633VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4634{
4635 int rc;
4636 switch (idxField)
4637 {
4638 /*
4639 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4640 */
4641 /* 64-bit Control fields. */
4642 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4643 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4644 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4645 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4646 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4647 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4648 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4649 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4650 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4651 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4652 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4653 case VMX_VMCS64_CTRL_EPTP_FULL:
4654 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4655 /* 64-bit Guest-state fields. */
4656 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4657 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4658 case VMX_VMCS64_GUEST_PAT_FULL:
4659 case VMX_VMCS64_GUEST_EFER_FULL:
4660 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4661 case VMX_VMCS64_GUEST_PDPTE0_FULL:
4662 case VMX_VMCS64_GUEST_PDPTE1_FULL:
4663 case VMX_VMCS64_GUEST_PDPTE2_FULL:
4664 case VMX_VMCS64_GUEST_PDPTE3_FULL:
4665 /* 64-bit Host-state fields. */
4666 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4667 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4668 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4669 {
4670 rc = VMXWriteVmcs32(idxField, u64Val);
4671 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4672 break;
4673 }
4674
4675 /*
4676 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4677 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4678 */
4679 /* Natural-width Guest-state fields. */
4680 case VMX_VMCS_GUEST_CR3:
4681 case VMX_VMCS_GUEST_ES_BASE:
4682 case VMX_VMCS_GUEST_CS_BASE:
4683 case VMX_VMCS_GUEST_SS_BASE:
4684 case VMX_VMCS_GUEST_DS_BASE:
4685 case VMX_VMCS_GUEST_FS_BASE:
4686 case VMX_VMCS_GUEST_GS_BASE:
4687 case VMX_VMCS_GUEST_LDTR_BASE:
4688 case VMX_VMCS_GUEST_TR_BASE:
4689 case VMX_VMCS_GUEST_GDTR_BASE:
4690 case VMX_VMCS_GUEST_IDTR_BASE:
4691 case VMX_VMCS_GUEST_RSP:
4692 case VMX_VMCS_GUEST_RIP:
4693 case VMX_VMCS_GUEST_SYSENTER_ESP:
4694 case VMX_VMCS_GUEST_SYSENTER_EIP:
4695 {
4696 if (!(u64Val >> 32))
4697 {
4698 /* If this field is 64-bit, VT-x will zero out the top bits. */
4699 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4700 }
4701 else
4702 {
4703 /* Assert that only the 32->64 switcher case should ever come here. */
4704 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4705 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4706 }
4707 break;
4708 }
4709
4710 default:
4711 {
4712 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4713 rc = VERR_INVALID_PARAMETER;
4714 break;
4715 }
4716 }
4717 AssertRCReturn(rc, rc);
4718 return rc;
4719}
4720
4721
4722/**
4723 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4724 * hosts (except darwin) for 64-bit guests.
4725 *
4726 * @param pVCpu Pointer to the VMCPU.
4727 * @param idxField The VMCS field encoding.
4728 * @param u64Val 16, 32 or 64 bits value.
4729 */
4730VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4731{
4732 AssertPtr(pVCpu);
4733 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4734
4735 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4736 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4737
4738 /* Make sure there are no duplicates. */
4739 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4740 {
4741 if (pCache->Write.aField[i] == idxField)
4742 {
4743 pCache->Write.aFieldVal[i] = u64Val;
4744 return VINF_SUCCESS;
4745 }
4746 }
4747
4748 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4749 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4750 pCache->Write.cValidEntries++;
4751 return VINF_SUCCESS;
4752}
4753
4754/* Enable later when the assembly code uses these as callbacks. */
4755#if 0
4756/*
4757 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4758 *
4759 * @param pVCpu Pointer to the VMCPU.
4760 * @param pCache Pointer to the VMCS cache.
4761 *
4762 * @remarks No-long-jump zone!!!
4763 */
4764VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4765{
4766 AssertPtr(pCache);
4767 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4768 {
4769 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4770 AssertRC(rc);
4771 }
4772 pCache->Write.cValidEntries = 0;
4773}
4774
4775
4776/**
4777 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
4778 *
4779 * @param pVCpu Pointer to the VMCPU.
4780 * @param pCache Pointer to the VMCS cache.
4781 *
4782 * @remarks No-long-jump zone!!!
4783 */
4784VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4785{
4786 AssertPtr(pCache);
4787 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4788 {
4789 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4790 AssertRC(rc);
4791 }
4792}
4793#endif
4794#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4795
4796
4797/**
4798 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4799 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4800 * timer.
4801 *
4802 * @returns VBox status code.
4803 * @param pVCpu Pointer to the VMCPU.
4804 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4805 * out-of-sync. Make sure to update the required fields
4806 * before using them.
4807 * @remarks No-long-jump zone!!!
4808 */
4809static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4810{
4811 int rc = VERR_INTERNAL_ERROR_5;
4812 bool fOffsettedTsc = false;
4813 PVM pVM = pVCpu->CTX_SUFF(pVM);
4814 if (pVM->hm.s.vmx.fUsePreemptTimer)
4815 {
4816 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4817
4818 /* Make sure the returned values have sane upper and lower boundaries. */
4819 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4820 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4821 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4822 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4823
4824 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4825 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4826 }
4827 else
4828 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4829
4830 if (fOffsettedTsc)
4831 {
4832 uint64_t u64CurTSC = ASMReadTSC();
4833 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4834 {
4835 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4836 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4837
4838 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4839 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4840 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4841 }
4842 else
4843 {
4844 /* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4845 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4846 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4847 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4848 }
4849 }
4850 else
4851 {
4852 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4853 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4854 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4855 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4856 }
4857}
4858
4859
4860/**
4861 * Determines if an exception is a contributory exception. Contributory
4862 * exceptions are ones which can cause double-faults. Page-fault is
4863 * intentionally not included here as it's a conditional contributory exception.
4864 *
4865 * @returns true if the exception is contributory, false otherwise.
4866 * @param uVector The exception vector.
4867 */
4868DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4869{
4870 switch (uVector)
4871 {
4872 case X86_XCPT_GP:
4873 case X86_XCPT_SS:
4874 case X86_XCPT_NP:
4875 case X86_XCPT_TS:
4876 case X86_XCPT_DE:
4877 return true;
4878 default:
4879 break;
4880 }
4881 return false;
4882}
4883
4884
4885/**
4886 * Sets an event as a pending event to be injected into the guest.
4887 *
4888 * @param pVCpu Pointer to the VMCPU.
4889 * @param u32IntrInfo The VM-entry interruption-information field.
4890 * @param cbInstr The VM-entry instruction length in bytes (for software
4891 * interrupts, exceptions and privileged software
4892 * exceptions).
4893 * @param u32ErrCode The VM-entry exception error code.
4894 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4895 * page-fault.
4896 *
4897 * @remarks Statistics counter assumes this is a guest event being injected or
4898 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
4899 * always incremented.
4900 */
4901DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4902 RTGCUINTPTR GCPtrFaultAddress)
4903{
4904 Assert(!pVCpu->hm.s.Event.fPending);
4905 pVCpu->hm.s.Event.fPending = true;
4906 pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4907 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4908 pVCpu->hm.s.Event.cbInstr = cbInstr;
4909 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4910
4911 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
4912}
4913
4914
4915/**
4916 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4917 *
4918 * @param pVCpu Pointer to the VMCPU.
4919 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4920 * out-of-sync. Make sure to update the required fields
4921 * before using them.
4922 */
4923DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4924{
4925 uint32_t u32IntrInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
4926 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4927 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4928 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
4929}
4930
4931
4932/**
4933 * Handle a condition that occurred while delivering an event through the guest
4934 * IDT.
4935 *
4936 * @returns VBox status code (informational error codes included).
4937 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
4938 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
4939 * continue execution of the guest which will delivery the #DF.
4940 * @retval VINF_EM_RESET if we detected a triple-fault condition.
4941 *
4942 * @param pVCpu Pointer to the VMCPU.
4943 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4944 * out-of-sync. Make sure to update the required fields
4945 * before using them.
4946 * @param pVmxTransient Pointer to the VMX transient structure.
4947 *
4948 * @remarks No-long-jump zone!!!
4949 */
4950static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4951{
4952 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4953 AssertRC(rc);
4954 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4955 {
4956 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4957 AssertRCReturn(rc, rc);
4958
4959 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4960 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4961 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4962
4963 typedef enum
4964 {
4965 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4966 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4967 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4968 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4969 } VMXREFLECTXCPT;
4970
4971 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4972 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4973 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo))
4974 {
4975 if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4976 {
4977 enmReflect = VMXREFLECTXCPT_XCPT;
4978#ifdef VBOX_STRICT
4979 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4980 && uExitVector == X86_XCPT_PF)
4981 {
4982 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4983 }
4984#endif
4985 if ( uExitVector == X86_XCPT_PF
4986 && uIdtVector == X86_XCPT_PF)
4987 {
4988 pVmxTransient->fVectoringPF = true;
4989 Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4990 }
4991 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4992 && hmR0VmxIsContributoryXcpt(uExitVector)
4993 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
4994 || uIdtVector == X86_XCPT_PF))
4995 {
4996 enmReflect = VMXREFLECTXCPT_DF;
4997 }
4998 else if (uIdtVector == X86_XCPT_DF)
4999 enmReflect = VMXREFLECTXCPT_TF;
5000 }
5001 else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5002 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5003 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5004 {
5005 /*
5006 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5007 * (whatever they are) as they reoccur when restarting the instruction.
5008 */
5009 enmReflect = VMXREFLECTXCPT_XCPT;
5010 }
5011 }
5012 else
5013 {
5014 /*
5015 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5016 * interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5017 * original exception to the guest after handling the VM-exit.
5018 */
5019 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5020 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5021 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5022 {
5023 enmReflect = VMXREFLECTXCPT_XCPT;
5024 }
5025 }
5026
5027 switch (enmReflect)
5028 {
5029 case VMXREFLECTXCPT_XCPT:
5030 {
5031 Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5032 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5033 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5034
5035 uint32_t u32ErrCode = 0;
5036 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5037 {
5038 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5039 AssertRCReturn(rc, rc);
5040 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5041 }
5042
5043 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5044 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5045 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5046 rc = VINF_SUCCESS;
5047 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5048 pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
5049
5050 break;
5051 }
5052
5053 case VMXREFLECTXCPT_DF:
5054 {
5055 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5056 rc = VINF_HM_DOUBLE_FAULT;
5057 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5058 pVCpu->hm.s.Event.u64IntrInfo, uIdtVector, uExitVector));
5059
5060 break;
5061 }
5062
5063 case VMXREFLECTXCPT_TF:
5064 {
5065 rc = VINF_EM_RESET;
5066 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5067 uExitVector));
5068 break;
5069 }
5070
5071 default:
5072 Assert(rc == VINF_SUCCESS);
5073 break;
5074 }
5075 }
5076 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5077 return rc;
5078}
5079
5080
5081/**
5082 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5083 *
5084 * @returns VBox status code.
5085 * @param pVCpu Pointer to the VMCPU.
5086 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5087 * out-of-sync. Make sure to update the required fields
5088 * before using them.
5089 *
5090 * @remarks No-long-jump zone!!!
5091 */
5092static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5093{
5094 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
5095 {
5096 uint32_t uVal = 0;
5097 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5098 AssertRCReturn(rc, rc);
5099 uint32_t uShadow = 0;
5100 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5101 AssertRCReturn(rc, rc);
5102
5103 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5104 CPUMSetGuestCR0(pVCpu, uVal);
5105 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR0;
5106 }
5107 return VINF_SUCCESS;
5108}
5109
5110
5111/**
5112 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5113 *
5114 * @returns VBox status code.
5115 * @param pVCpu Pointer to the VMCPU.
5116 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5117 * out-of-sync. Make sure to update the required fields
5118 * before using them.
5119 *
5120 * @remarks No-long-jump zone!!!
5121 */
5122static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5123{
5124 int rc = VINF_SUCCESS;
5125 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
5126 {
5127 uint32_t uVal = 0;
5128 uint32_t uShadow = 0;
5129 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5130 AssertRCReturn(rc, rc);
5131 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5132 AssertRCReturn(rc, rc);
5133
5134 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5135 CPUMSetGuestCR4(pVCpu, uVal);
5136 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR4;
5137 }
5138 return rc;
5139}
5140
5141
5142/**
5143 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5144 *
5145 * @returns VBox status code.
5146 * @param pVCpu Pointer to the VMCPU.
5147 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5148 * out-of-sync. Make sure to update the required fields
5149 * before using them.
5150 *
5151 * @remarks No-long-jump zone!!!
5152 */
5153static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5154{
5155 int rc = VINF_SUCCESS;
5156 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5157 {
5158 uint64_t u64Val = 0;
5159 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5160 AssertRCReturn(rc, rc);
5161
5162 pMixedCtx->rip = u64Val;
5163 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RIP;
5164 }
5165 return rc;
5166}
5167
5168
5169/**
5170 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
5171 *
5172 * @returns VBox status code.
5173 * @param pVCpu Pointer to the VMCPU.
5174 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5175 * out-of-sync. Make sure to update the required fields
5176 * before using them.
5177 *
5178 * @remarks No-long-jump zone!!!
5179 */
5180static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5181{
5182 int rc = VINF_SUCCESS;
5183 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5184 {
5185 uint64_t u64Val = 0;
5186 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5187 AssertRCReturn(rc, rc);
5188
5189 pMixedCtx->rsp = u64Val;
5190 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RSP;
5191 }
5192 return rc;
5193}
5194
5195
5196/**
5197 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5198 *
5199 * @returns VBox status code.
5200 * @param pVCpu Pointer to the VMCPU.
5201 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5202 * out-of-sync. Make sure to update the required fields
5203 * before using them.
5204 *
5205 * @remarks No-long-jump zone!!!
5206 */
5207static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5208{
5209 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5210 {
5211 uint32_t uVal = 0;
5212 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5213 AssertRCReturn(rc, rc);
5214
5215 pMixedCtx->eflags.u32 = uVal;
5216 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5217 {
5218 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5219 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5220
5221 pMixedCtx->eflags.Bits.u1VM = 0;
5222 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5223 }
5224
5225 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RFLAGS;
5226 }
5227 return VINF_SUCCESS;
5228}
5229
5230
5231/**
5232 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5233 * guest-CPU context.
5234 */
5235DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5236{
5237 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5238 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5239 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5240 return rc;
5241}
5242
5243
5244/**
5245 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5246 * from the guest-state area in the VMCS.
5247 *
5248 * @param pVCpu Pointer to the VMCPU.
5249 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5250 * out-of-sync. Make sure to update the required fields
5251 * before using them.
5252 *
5253 * @remarks No-long-jump zone!!!
5254 */
5255static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5256{
5257 uint32_t uIntrState = 0;
5258 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5259 AssertRC(rc);
5260
5261 if (!uIntrState)
5262 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5263 else
5264 {
5265 Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5266 || uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5267 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5268 AssertRC(rc);
5269 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5270 AssertRC(rc);
5271
5272 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5273 Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5274 }
5275}
5276
5277
5278/**
5279 * Saves the guest's activity state.
5280 *
5281 * @returns VBox status code.
5282 * @param pVCpu Pointer to the VMCPU.
5283 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5284 * out-of-sync. Make sure to update the required fields
5285 * before using them.
5286 *
5287 * @remarks No-long-jump zone!!!
5288 */
5289static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5290{
5291 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5292 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5293 return VINF_SUCCESS;
5294}
5295
5296
5297/**
5298 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5299 * the current VMCS into the guest-CPU context.
5300 *
5301 * @returns VBox status code.
5302 * @param pVCpu Pointer to the VMCPU.
5303 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5304 * out-of-sync. Make sure to update the required fields
5305 * before using them.
5306 *
5307 * @remarks No-long-jump zone!!!
5308 */
5309static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5310{
5311 int rc = VINF_SUCCESS;
5312 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5313 {
5314 uint32_t u32Val = 0;
5315 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5316 pMixedCtx->SysEnter.cs = u32Val;
5317 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5318 }
5319
5320 uint64_t u64Val = 0;
5321 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5322 {
5323 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5324 pMixedCtx->SysEnter.eip = u64Val;
5325 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5326 }
5327 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5328 {
5329 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5330 pMixedCtx->SysEnter.esp = u64Val;
5331 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5332 }
5333 return rc;
5334}
5335
5336
5337/**
5338 * Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5339 * context.
5340 *
5341 * @returns VBox status code.
5342 * @param pVCpu Pointer to the VMCPU.
5343 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5344 * out-of-sync. Make sure to update the required fields
5345 * before using them.
5346 *
5347 * @remarks No-long-jump zone!!!
5348 */
5349static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5350{
5351 int rc = VINF_SUCCESS;
5352 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5353 {
5354 uint64_t u64Val = 0;
5355 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5356 pMixedCtx->fs.u64Base = u64Val;
5357 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5358 }
5359 return rc;
5360}
5361
5362
5363/**
5364 * Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5365 * context.
5366 *
5367 * @returns VBox status code.
5368 * @param pVCpu Pointer to the VMCPU.
5369 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5370 * out-of-sync. Make sure to update the required fields
5371 * before using them.
5372 *
5373 * @remarks No-long-jump zone!!!
5374 */
5375static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5376{
5377 int rc = VINF_SUCCESS;
5378 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5379 {
5380 uint64_t u64Val = 0;
5381 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5382 pMixedCtx->gs.u64Base = u64Val;
5383 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5384 }
5385 return rc;
5386}
5387
5388
5389/**
5390 * Saves the auto load/store'd guest MSRs from the current VMCS into the
5391 * guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5392 * and TSC_AUX.
5393 *
5394 * @returns VBox status code.
5395 * @param pVCpu Pointer to the VMCPU.
5396 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5397 * out-of-sync. Make sure to update the required fields
5398 * before using them.
5399 *
5400 * @remarks No-long-jump zone!!!
5401 */
5402static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5403{
5404 if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5405 return VINF_SUCCESS;
5406
5407#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5408 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5409 {
5410 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5411 pMsr += i;
5412 switch (pMsr->u32Msr)
5413 {
5414 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5415 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5416 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5417 case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5418 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5419 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5420 default:
5421 {
5422 AssertFailed();
5423 return VERR_HM_UNEXPECTED_LD_ST_MSR;
5424 }
5425 }
5426 }
5427#endif
5428
5429 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5430 return VINF_SUCCESS;
5431}
5432
5433
5434/**
5435 * Saves the guest control registers from the current VMCS into the guest-CPU
5436 * context.
5437 *
5438 * @returns VBox status code.
5439 * @param pVCpu Pointer to the VMCPU.
5440 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5441 * out-of-sync. Make sure to update the required fields
5442 * before using them.
5443 *
5444 * @remarks No-long-jump zone!!!
5445 */
5446static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5447{
5448 /* Guest CR0. Guest FPU. */
5449 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5450 AssertRCReturn(rc, rc);
5451
5452 /* Guest CR4. */
5453 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5454 AssertRCReturn(rc, rc);
5455
5456 /* Guest CR2 - updated always during the world-switch or in #PF. */
5457 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5458 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5459 {
5460 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5461 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5462
5463 PVM pVM = pVCpu->CTX_SUFF(pVM);
5464 if ( pVM->hm.s.vmx.fUnrestrictedGuest
5465 || ( pVM->hm.s.fNestedPaging
5466 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5467 {
5468 uint64_t u64Val = 0;
5469 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5470 if (pMixedCtx->cr3 != u64Val)
5471 {
5472 CPUMSetGuestCR3(pVCpu, u64Val);
5473 if (VMMRZCallRing3IsEnabled(pVCpu))
5474 {
5475 PGMUpdateCR3(pVCpu, u64Val);
5476 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5477 }
5478 else
5479 {
5480 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5481 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5482 }
5483 }
5484
5485 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5486 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5487 {
5488 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5489 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5490 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5491 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5492
5493 if (VMMRZCallRing3IsEnabled(pVCpu))
5494 {
5495 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5496 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5497 }
5498 else
5499 {
5500 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5501 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5502 }
5503 }
5504 }
5505
5506 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR3;
5507 }
5508
5509 /*
5510 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5511 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5512 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5513 *
5514 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5515 */
5516 if (VMMRZCallRing3IsEnabled(pVCpu))
5517 {
5518 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5519 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5520
5521 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5522 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5523
5524 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5525 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5526 }
5527
5528 return rc;
5529}
5530
5531
5532/**
5533 * Reads a guest segment register from the current VMCS into the guest-CPU
5534 * context.
5535 *
5536 * @returns VBox status code.
5537 * @param pVCpu Pointer to the VMCPU.
5538 * @param idxSel Index of the selector in the VMCS.
5539 * @param idxLimit Index of the segment limit in the VMCS.
5540 * @param idxBase Index of the segment base in the VMCS.
5541 * @param idxAccess Index of the access rights of the segment in the VMCS.
5542 * @param pSelReg Pointer to the segment selector.
5543 *
5544 * @remarks No-long-jump zone!!!
5545 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5546 * macro as that takes care of whether to read from the VMCS cache or
5547 * not.
5548 */
5549DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5550 PCPUMSELREG pSelReg)
5551{
5552 uint32_t u32Val = 0;
5553 int rc = VMXReadVmcs32(idxSel, &u32Val);
5554 AssertRCReturn(rc, rc);
5555 pSelReg->Sel = (uint16_t)u32Val;
5556 pSelReg->ValidSel = (uint16_t)u32Val;
5557 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5558
5559 rc = VMXReadVmcs32(idxLimit, &u32Val);
5560 AssertRCReturn(rc, rc);
5561 pSelReg->u32Limit = u32Val;
5562
5563 uint64_t u64Val = 0;
5564 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5565 AssertRCReturn(rc, rc);
5566 pSelReg->u64Base = u64Val;
5567
5568 rc = VMXReadVmcs32(idxAccess, &u32Val);
5569 AssertRCReturn(rc, rc);
5570 pSelReg->Attr.u = u32Val;
5571
5572 /*
5573 * If VT-x marks the segment as unusable, most other bits remain undefined:
5574 * - For CS the L, D and G bits have meaning.
5575 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
5576 * - For the remaining data segments no bits are defined.
5577 *
5578 * The present bit and the unusable bit has been observed to be set at the
5579 * same time (the selector was supposed to invalid as we started executing
5580 * a V8086 interrupt in ring-0).
5581 *
5582 * What should be important for the rest of the VBox code that the P bit is
5583 * cleared. Some of the other VBox code recognizes the unusable bit, but
5584 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
5585 * safe side here, we'll strip off P and other bits we don't care about. If
5586 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
5587 *
5588 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5589 */
5590 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
5591 {
5592 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5593
5594 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
5595 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
5596 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
5597
5598 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
5599#ifdef DEBUG_bird
5600 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
5601 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
5602 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
5603#endif
5604 }
5605 return VINF_SUCCESS;
5606}
5607
5608
5609#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5610# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5611 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5612 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5613#else
5614# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5615 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5616 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5617#endif
5618
5619
5620/**
5621 * Saves the guest segment registers from the current VMCS into the guest-CPU
5622 * context.
5623 *
5624 * @returns VBox status code.
5625 * @param pVCpu Pointer to the VMCPU.
5626 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5627 * out-of-sync. Make sure to update the required fields
5628 * before using them.
5629 *
5630 * @remarks No-long-jump zone!!!
5631 */
5632static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5633{
5634 /* Guest segment registers. */
5635 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5636 {
5637 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5638 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5639 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5640 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5641 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5642 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5643 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5644
5645 /* Restore segment attributes for real-on-v86 mode hack. */
5646 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5647 {
5648 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
5649 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
5650 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
5651 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
5652 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
5653 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
5654 }
5655 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5656 }
5657
5658 return VINF_SUCCESS;
5659}
5660
5661
5662/**
5663 * Saves the guest descriptor table registers and task register from the current
5664 * VMCS into the guest-CPU context.
5665 *
5666 * @returns VBox status code.
5667 * @param pVCpu Pointer to the VMCPU.
5668 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5669 * out-of-sync. Make sure to update the required fields
5670 * before using them.
5671 *
5672 * @remarks No-long-jump zone!!!
5673 */
5674static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5675{
5676 int rc = VINF_SUCCESS;
5677
5678 /* Guest LDTR. */
5679 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5680 {
5681 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5682 AssertRCReturn(rc, rc);
5683 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_LDTR;
5684 }
5685
5686 /* Guest GDTR. */
5687 uint64_t u64Val = 0;
5688 uint32_t u32Val = 0;
5689 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5690 {
5691 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5692 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5693 pMixedCtx->gdtr.pGdt = u64Val;
5694 pMixedCtx->gdtr.cbGdt = u32Val;
5695 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_GDTR;
5696 }
5697
5698 /* Guest IDTR. */
5699 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5700 {
5701 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5702 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5703 pMixedCtx->idtr.pIdt = u64Val;
5704 pMixedCtx->idtr.cbIdt = u32Val;
5705 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_IDTR;
5706 }
5707
5708 /* Guest TR. */
5709 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5710 {
5711 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5712 AssertRCReturn(rc, rc);
5713
5714 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5715 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5716 {
5717 rc = VMXLOCAL_READ_SEG(TR, tr);
5718 AssertRCReturn(rc, rc);
5719 }
5720 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_TR;
5721 }
5722 return rc;
5723}
5724
5725#undef VMXLOCAL_READ_SEG
5726
5727
5728/**
5729 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
5730 * context.
5731 *
5732 * @returns VBox status code.
5733 * @param pVCpu Pointer to the VMCPU.
5734 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5735 * out-of-sync. Make sure to update the required fields
5736 * before using them.
5737 *
5738 * @remarks No-long-jump zone!!!
5739 */
5740static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5741{
5742 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5743 {
5744 if (!CPUMIsHyperDebugStateActive(pVCpu))
5745 {
5746 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
5747 uint32_t u32Val;
5748 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
5749 pMixedCtx->dr[7] = u32Val;
5750 }
5751
5752 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_DEBUG;
5753 }
5754 return VINF_SUCCESS;
5755}
5756
5757
5758/**
5759 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
5760 *
5761 * @returns VBox status code.
5762 * @param pVCpu Pointer to the VMCPU.
5763 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5764 * out-of-sync. Make sure to update the required fields
5765 * before using them.
5766 *
5767 * @remarks No-long-jump zone!!!
5768 */
5769static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5770{
5771 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5772 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_APIC_STATE;
5773 return VINF_SUCCESS;
5774}
5775
5776
5777/**
5778 * Saves the entire guest state from the currently active VMCS into the
5779 * guest-CPU context. This essentially VMREADs all guest-data.
5780 *
5781 * @returns VBox status code.
5782 * @param pVCpu Pointer to the VMCPU.
5783 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5784 * out-of-sync. Make sure to update the required fields
5785 * before using them.
5786 */
5787static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5788{
5789 Assert(pVCpu);
5790 Assert(pMixedCtx);
5791
5792 if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5793 return VINF_SUCCESS;
5794
5795 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled again on the ring-3 callback path,
5796 there is no real need to. */
5797 if (VMMRZCallRing3IsEnabled(pVCpu))
5798 VMMR0LogFlushDisable(pVCpu);
5799 else
5800 Assert(VMMR0IsLogFlushDisabled(pVCpu));
5801 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
5802
5803 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5804 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5805
5806 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5807 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5808
5809 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5810 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5811
5812 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5813 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5814
5815 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
5816 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5817
5818 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5819 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5820
5821 rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5822 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5823
5824 rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5825 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5826
5827 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5828 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5829
5830 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5831 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5832
5833 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5834 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5835
5836 AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5837 ("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5838
5839 if (VMMRZCallRing3IsEnabled(pVCpu))
5840 VMMR0LogFlushEnable(pVCpu);
5841
5842 return rc;
5843}
5844
5845
5846/**
5847 * Check per-VM and per-VCPU force flag actions that require us to go back to
5848 * ring-3 for one reason or another.
5849 *
5850 * @returns VBox status code (information status code included).
5851 * @retval VINF_SUCCESS if we don't have any actions that require going back to
5852 * ring-3.
5853 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5854 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5855 * interrupts)
5856 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5857 * all EMTs to be in ring-3.
5858 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5859 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5860 * to the EM loop.
5861 *
5862 * @param pVM Pointer to the VM.
5863 * @param pVCpu Pointer to the VMCPU.
5864 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5865 * out-of-sync. Make sure to update the required fields
5866 * before using them.
5867 */
5868static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5869{
5870 Assert(VMMRZCallRing3IsEnabled(pVCpu));
5871
5872 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
5873 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
5874 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
5875 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
5876 {
5877 /* We need the control registers now, make sure the guest-CPU context is updated. */
5878 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5879 AssertRCReturn(rc3, rc3);
5880
5881 /* Pending HM CR3 sync. */
5882 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5883 {
5884 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5885 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
5886 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
5887 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5888 }
5889
5890 /* Pending HM PAE PDPEs. */
5891 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5892 {
5893 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5894 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5895 }
5896
5897 /* Pending PGM C3 sync. */
5898 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5899 {
5900 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5901 if (rc2 != VINF_SUCCESS)
5902 {
5903 AssertRC(rc2);
5904 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
5905 return rc2;
5906 }
5907 }
5908
5909 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5910 /* -XXX- what was that about single stepping? */
5911 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5912 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5913 {
5914 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5915 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5916 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
5917 return rc2;
5918 }
5919
5920 /* Pending VM request packets, such as hardware interrupts. */
5921 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5922 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5923 {
5924 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5925 return VINF_EM_PENDING_REQUEST;
5926 }
5927
5928 /* Pending PGM pool flushes. */
5929 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5930 {
5931 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5932 return VINF_PGM_POOL_FLUSH_PENDING;
5933 }
5934
5935 /* Pending DMA requests. */
5936 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5937 {
5938 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5939 return VINF_EM_RAW_TO_R3;
5940 }
5941 }
5942
5943 /* Paranoia. */
5944 return VINF_SUCCESS;
5945}
5946
5947
5948/**
5949 * Converts any TRPM trap into a pending HM event. This is typically used when
5950 * entering from ring-3 (not longjmp returns).
5951 *
5952 * @param pVCpu Pointer to the VMCPU.
5953 */
5954static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
5955{
5956 Assert(TRPMHasTrap(pVCpu));
5957 Assert(!pVCpu->hm.s.Event.fPending);
5958
5959 uint8_t uVector;
5960 TRPMEVENT enmTrpmEvent;
5961 RTGCUINT uErrCode;
5962 RTGCUINTPTR GCPtrFaultAddress;
5963 uint8_t cbInstr;
5964
5965 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5966 AssertRC(rc);
5967
5968 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5969 uint32_t u32IntrInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
5970 if (enmTrpmEvent == TRPM_TRAP)
5971 {
5972 switch (uVector)
5973 {
5974 case X86_XCPT_BP:
5975 case X86_XCPT_OF:
5976 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5977 break;
5978
5979 case X86_XCPT_PF:
5980 case X86_XCPT_DF:
5981 case X86_XCPT_TS:
5982 case X86_XCPT_NP:
5983 case X86_XCPT_SS:
5984 case X86_XCPT_GP:
5985 case X86_XCPT_AC:
5986 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5987 /* no break! */
5988 default:
5989 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5990 break;
5991 }
5992 }
5993 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5994 {
5995 if (uVector == X86_XCPT_NMI)
5996 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5997 else
5998 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5999 }
6000 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6001 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6002 else
6003 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6004
6005 rc = TRPMResetTrap(pVCpu);
6006 AssertRC(rc);
6007 Log4(("TRPM->HM event: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6008 u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6009
6010 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6011 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6012}
6013
6014
6015/**
6016 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6017 * VT-x to execute any instruction.
6018 *
6019 * @param pvCpu Pointer to the VMCPU.
6020 */
6021static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6022{
6023 Assert(pVCpu->hm.s.Event.fPending);
6024
6025 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6026 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
6027 bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
6028 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6029
6030 /* If a trap was already pending, we did something wrong! */
6031 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6032
6033 TRPMEVENT enmTrapType;
6034 switch (uVectorType)
6035 {
6036 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6037 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6038 enmTrapType = TRPM_HARDWARE_INT;
6039 break;
6040
6041 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6042 enmTrapType = TRPM_SOFTWARE_INT;
6043 break;
6044
6045 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6046 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6047 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6048 enmTrapType = TRPM_TRAP;
6049 break;
6050
6051 default:
6052 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6053 enmTrapType = TRPM_32BIT_HACK;
6054 break;
6055 }
6056
6057 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6058
6059 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6060 AssertRC(rc);
6061
6062 if (fErrorCodeValid)
6063 TRPMSetErrorCode(pVCpu, uErrorCode);
6064
6065 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6066 && uVector == X86_XCPT_PF)
6067 {
6068 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6069 }
6070 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6071 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6072 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6073 {
6074 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6075 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6076 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6077 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6078 }
6079 pVCpu->hm.s.Event.fPending = false;
6080}
6081
6082
6083/**
6084 * Does the necessary state syncing before returning to ring-3 for any reason
6085 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6086 *
6087 * @param pVM Pointer to the VM.
6088 * @param pVCpu Pointer to the VMCPU.
6089 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6090 * out-of-sync. Make sure to update the required fields
6091 * before using them.
6092 *
6093 * @remarks No-long-jmp zone!!!
6094 */
6095static void hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6096{
6097 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6098 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6099
6100 RTCPUID idCpu = RTMpCpuId();
6101 Log4Func(("HostCpuId=%u\n", idCpu));
6102
6103 /* Save the guest state if necessary. */
6104 if (pVCpu->hm.s.vmx.fUpdatedGuestState != HMVMX_UPDATED_GUEST_ALL)
6105 {
6106 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6107 AssertRC(rc);
6108 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
6109 }
6110
6111 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6112 if (CPUMIsGuestFPUStateActive(pVCpu))
6113 {
6114 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6115 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6116 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
6117 }
6118
6119 /* Restore host debug registers if necessary and resync on next R0 reentry. */
6120#ifdef VBOX_STRICT
6121 if (CPUMIsHyperDebugStateActive(pVCpu))
6122 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6123#endif
6124 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6125 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
6126 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
6127 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
6128
6129 /* Restore host-state bits that VT-x only restores partially. */
6130 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6131 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6132 {
6133 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6134 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6135 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6136 }
6137
6138 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6139 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6140 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6141 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6142 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6143 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6144 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6145 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6146
6147 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6148
6149 /** @todo This kinda defeats the purpose of having preemption hooks.
6150 * The problem is, deregistering the hooks should be moved to a place that
6151 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6152 * context.
6153 */
6154 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6155 {
6156 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6157 AssertRC(rc);
6158 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6159 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6160 }
6161 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6162 NOREF(idCpu);
6163}
6164
6165
6166/**
6167 * Leaves the VT-x session.
6168 *
6169 * @param pVM Pointer to the VM.
6170 * @param pVCpu Pointer to the VMCPU.
6171 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6172 * out-of-sync. Make sure to update the required fields
6173 * before using them.
6174 *
6175 * @remarks No-long-jmp zone!!!
6176 */
6177DECLINLINE(void) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6178{
6179 HM_DISABLE_PREEMPT_IF_NEEDED();
6180 HMVMX_ASSERT_CPU_SAFE();
6181 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6182 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6183
6184 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6185 and done this from the VMXR0ThreadCtxCallback(). */
6186 if (!pVCpu->hm.s.fLeaveDone)
6187 {
6188 hmR0VmxLeave(pVM, pVCpu, pMixedCtx);
6189 pVCpu->hm.s.fLeaveDone = true;
6190 }
6191
6192 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6193 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
6194 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6195 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6196 VMMR0ThreadCtxHooksDeregister(pVCpu);
6197
6198 /* Leave HM context. This takes care of local init (term). */
6199 int rc = HMR0LeaveCpu(pVCpu);
6200 AssertRC(rc); NOREF(rc);
6201
6202 HM_RESTORE_PREEMPT_IF_NEEDED();
6203}
6204
6205
6206/**
6207 * Does the necessary state syncing before doing a longjmp to ring-3.
6208 *
6209 * @param pVM Pointer to the VM.
6210 * @param pVCpu Pointer to the VMCPU.
6211 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6212 * out-of-sync. Make sure to update the required fields
6213 * before using them.
6214 *
6215 * @remarks No-long-jmp zone!!!
6216 */
6217DECLINLINE(void) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6218{
6219 hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6220}
6221
6222
6223/**
6224 * Take necessary actions before going back to ring-3.
6225 *
6226 * An action requires us to go back to ring-3. This function does the necessary
6227 * steps before we can safely return to ring-3. This is not the same as longjmps
6228 * to ring-3, this is voluntary and prepares the guest so it may continue
6229 * executing outside HM (recompiler/IEM).
6230 *
6231 * @param pVM Pointer to the VM.
6232 * @param pVCpu Pointer to the VMCPU.
6233 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6234 * out-of-sync. Make sure to update the required fields
6235 * before using them.
6236 * @param rcExit The reason for exiting to ring-3. Can be
6237 * VINF_VMM_UNKNOWN_RING3_CALL.
6238 */
6239static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6240{
6241 Assert(pVM);
6242 Assert(pVCpu);
6243 Assert(pMixedCtx);
6244 HMVMX_ASSERT_PREEMPT_SAFE();
6245
6246 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
6247 {
6248 /* We've done what is required in hmR0VmxExitErrInvalidGuestState(). We're not going to continue guest execution... */
6249 return;
6250 }
6251 else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6252 {
6253 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6254 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6255 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6256 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6257 return;
6258 }
6259
6260 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6261 VMMRZCallRing3Disable(pVCpu);
6262 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6263
6264 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6265 if (pVCpu->hm.s.Event.fPending)
6266 {
6267 hmR0VmxPendingEventToTrpmTrap(pVCpu);
6268 Assert(!pVCpu->hm.s.Event.fPending);
6269 }
6270
6271 /* Save guest state and restore host state bits. */
6272 hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6273 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6274
6275 /* Sync recompiler state. */
6276 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6277 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6278 | CPUM_CHANGED_LDTR
6279 | CPUM_CHANGED_GDTR
6280 | CPUM_CHANGED_IDTR
6281 | CPUM_CHANGED_TR
6282 | CPUM_CHANGED_HIDDEN_SEL_REGS);
6283 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
6284 if ( pVM->hm.s.fNestedPaging
6285 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
6286 {
6287 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6288 }
6289
6290 /*
6291 * Clear the X86_EFL_TF if necessary .
6292 */
6293 if (pVCpu->hm.s.fClearTrapFlag)
6294 {
6295 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6296 pMixedCtx->eflags.Bits.u1TF = 0;
6297 pVCpu->hm.s.fClearTrapFlag = false;
6298 }
6299 /** @todo there seems to be issues with the resume flag when the monitor trap
6300 * flag is pending without being used. Seen early in bios init when
6301 * accessing APIC page in prot mode. */
6302
6303 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6304 if (rcExit != VINF_EM_RAW_INTERRUPT)
6305 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_ALL_GUEST;
6306
6307 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6308
6309 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6310 VMMRZCallRing3RemoveNotification(pVCpu);
6311 VMMRZCallRing3Enable(pVCpu);
6312}
6313
6314
6315/**
6316 * VMMRZCallRing3() callback wrapper which saves the guest state before we
6317 * longjump to ring-3 and possibly get preempted.
6318 *
6319 * @param pVCpu Pointer to the VMCPU.
6320 * @param enmOperation The operation causing the ring-3 longjump.
6321 * @param pvUser The user argument (pointer to the possibly
6322 * out-of-date guest-CPU context).
6323 *
6324 * @remarks Must never be called with @a enmOperation ==
6325 * VMMCALLRING3_VM_R0_ASSERTION.
6326 */
6327DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6328{
6329 /* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion. */
6330 Assert(pVCpu);
6331 Assert(pvUser);
6332 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6333 HMVMX_ASSERT_PREEMPT_SAFE();
6334
6335 VMMRZCallRing3Disable(pVCpu);
6336 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6337
6338 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
6339 hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6340
6341 VMMRZCallRing3Enable(pVCpu);
6342}
6343
6344
6345/**
6346 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6347 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
6348 *
6349 * @param pVCpu Pointer to the VMCPU.
6350 */
6351DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6352{
6353 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6354 {
6355 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6356 {
6357 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6358 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6359 AssertRC(rc);
6360 }
6361 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6362}
6363
6364
6365/**
6366 * Evaluates the event to be delivered to the guest and sets it as the pending
6367 * event.
6368 *
6369 * @param pVCpu Pointer to the VMCPU.
6370 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6371 * out-of-sync. Make sure to update the required fields
6372 * before using them.
6373 */
6374static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6375{
6376 Assert(!pVCpu->hm.s.Event.fPending);
6377
6378 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6379 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6380 bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6381 bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6382
6383 Assert(!fBlockSti || (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6384 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6385 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6386 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6387 Assert(!TRPMHasTrap(pVCpu));
6388
6389 /** @todo SMI. SMIs take priority over NMIs. */
6390 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6391 {
6392 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6393 if ( !fBlockMovSS
6394 && !fBlockSti)
6395 {
6396 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6397 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
6398 uint32_t u32IntrInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
6399 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6400
6401 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddres */);
6402 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6403 }
6404 else
6405 hmR0VmxSetIntWindowExitVmcs(pVCpu);
6406 }
6407 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
6408 && !pVCpu->hm.s.fSingleInstruction)
6409 {
6410 /*
6411 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
6412 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
6413 * evaluated here and not set as pending, solely based on the force-flags.
6414 */
6415 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6416 AssertRC(rc);
6417 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6418 if ( !fBlockInt
6419 && !fBlockSti
6420 && !fBlockMovSS)
6421 {
6422 uint8_t u8Interrupt;
6423 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6424 if (RT_SUCCESS(rc))
6425 {
6426 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
6427 uint32_t u32IntrInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
6428 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6429
6430 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
6431 }
6432 else
6433 {
6434 /** @todo Does this actually happen? If not turn it into an assertion. */
6435 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
6436 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6437 }
6438 }
6439 else
6440 hmR0VmxSetIntWindowExitVmcs(pVCpu);
6441 }
6442}
6443
6444
6445/**
6446 * Injects any pending events into the guest if the guest is in a state to
6447 * receive them.
6448 *
6449 * @returns VBox status code (informational status codes included).
6450 * @param pVCpu Pointer to the VMCPU.
6451 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6452 * out-of-sync. Make sure to update the required fields
6453 * before using them.
6454 *
6455 * @remarks No-long-jump zone!!!
6456 */
6457static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6458{
6459 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6460 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6461 bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6462 bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6463
6464 Assert(!fBlockSti || (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6465 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6466 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6467 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6468 Assert(!TRPMHasTrap(pVCpu));
6469
6470 int rc = VINF_SUCCESS;
6471 if (pVCpu->hm.s.Event.fPending)
6472 {
6473#if defined(VBOX_STRICT) || defined(VBOX_WITH_STATISTICS)
6474 uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6475 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6476 {
6477 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6478 AssertRCReturn(rc, rc);
6479 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6480 Assert(!fBlockInt);
6481 Assert(!fBlockSti);
6482 Assert(!fBlockMovSS);
6483 }
6484 else if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6485 {
6486 Assert(!fBlockSti);
6487 Assert(!fBlockMovSS);
6488 }
6489#endif
6490 Log4(("Injecting pending event vcpu[%RU32] u64IntrInfo=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntrInfo));
6491 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
6492 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6493 AssertRCReturn(rc, rc);
6494
6495 pVCpu->hm.s.Event.fPending = false;
6496
6497 /* Update the interruptibility-state as it could have been changed by
6498 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
6499 fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6500 fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6501
6502#ifdef VBOX_WITH_STATISTICS
6503 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6504 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6505 else
6506 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6507#endif
6508 }
6509
6510 /* Delivery pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
6511 int rc2 = VINF_SUCCESS;
6512 if ( fBlockSti
6513 || fBlockMovSS)
6514 {
6515 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
6516 {
6517 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6518 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6519 {
6520 /*
6521 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
6522 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
6523 * See Intel spec. 27.3.4 "Saving Non-Register State".
6524 */
6525 rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6526 AssertRCReturn(rc, rc);
6527 }
6528 }
6529 else
6530 {
6531 /* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6532 delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6533 uIntrState = 0;
6534 }
6535 }
6536
6537 /*
6538 * There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6539 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6540 */
6541 rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6542 AssertRC(rc2);
6543
6544 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
6545 return rc;
6546}
6547
6548
6549/**
6550 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6551 *
6552 * @param pVCpu Pointer to the VMCPU.
6553 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6554 * out-of-sync. Make sure to update the required fields
6555 * before using them.
6556 */
6557DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6558{
6559 uint32_t u32IntrInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
6560 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6561}
6562
6563
6564/**
6565 * Injects a double-fault (#DF) exception into the VM.
6566 *
6567 * @returns VBox status code (informational status code included).
6568 * @param pVCpu Pointer to the VMCPU.
6569 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6570 * out-of-sync. Make sure to update the required fields
6571 * before using them.
6572 */
6573DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6574{
6575 uint32_t u32IntrInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
6576 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6577 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6578 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
6579 puIntrState);
6580}
6581
6582
6583/**
6584 * Sets a debug (#DB) exception as pending-for-injection into the VM.
6585 *
6586 * @param pVCpu Pointer to the VMCPU.
6587 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6588 * out-of-sync. Make sure to update the required fields
6589 * before using them.
6590 */
6591DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6592{
6593 uint32_t u32IntrInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
6594 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6595 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6596}
6597
6598
6599/**
6600 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
6601 *
6602 * @param pVCpu Pointer to the VMCPU.
6603 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6604 * out-of-sync. Make sure to update the required fields
6605 * before using them.
6606 * @param cbInstr The value of RIP that is to be pushed on the guest
6607 * stack.
6608 */
6609DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6610{
6611 uint32_t u32IntrInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
6612 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6613 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6614}
6615
6616
6617/**
6618 * Injects a general-protection (#GP) fault into the VM.
6619 *
6620 * @returns VBox status code (informational status code included).
6621 * @param pVCpu Pointer to the VMCPU.
6622 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6623 * out-of-sync. Make sure to update the required fields
6624 * before using them.
6625 * @param u32ErrorCode The error code associated with the #GP.
6626 */
6627DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6628 uint32_t *puIntrState)
6629{
6630 uint32_t u32IntrInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
6631 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6632 if (fErrorCodeValid)
6633 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6634 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
6635 puIntrState);
6636}
6637
6638
6639/**
6640 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
6641 *
6642 * @param pVCpu Pointer to the VMCPU.
6643 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6644 * out-of-sync. Make sure to update the required fields
6645 * before using them.
6646 * @param uVector The software interrupt vector number.
6647 * @param cbInstr The value of RIP that is to be pushed on the guest
6648 * stack.
6649 */
6650DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6651{
6652 uint32_t u32IntrInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6653 if ( uVector == X86_XCPT_BP
6654 || uVector == X86_XCPT_OF)
6655 {
6656 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6657 }
6658 else
6659 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6660 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6661}
6662
6663
6664/**
6665 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6666 * stack.
6667 *
6668 * @returns VBox status code (information status code included).
6669 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6670 * @param pVM Pointer to the VM.
6671 * @param pMixedCtx Pointer to the guest-CPU context.
6672 * @param uValue The value to push to the guest stack.
6673 */
6674DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6675{
6676 /*
6677 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6678 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6679 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6680 */
6681 if (pMixedCtx->sp == 1)
6682 return VINF_EM_RESET;
6683 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6684 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6685 AssertRCReturn(rc, rc);
6686 return rc;
6687}
6688
6689
6690/**
6691 * Injects an event into the guest upon VM-entry by updating the relevant fields
6692 * in the VM-entry area in the VMCS.
6693 *
6694 * @returns VBox status code (informational error codes included).
6695 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6696 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6697 *
6698 * @param pVCpu Pointer to the VMCPU.
6699 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6700 * be out-of-sync. Make sure to update the required
6701 * fields before using them.
6702 * @param u64IntrInfo The VM-entry interruption-information field.
6703 * @param cbInstr The VM-entry instruction length in bytes (for
6704 * software interrupts, exceptions and privileged
6705 * software exceptions).
6706 * @param u32ErrCode The VM-entry exception error code.
6707 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6708 * @param puIntrState Pointer to the current guest interruptibility-state.
6709 * This interruptibility-state will be updated if
6710 * necessary. This cannot not be NULL.
6711 *
6712 * @remarks No-long-jump zone!!!
6713 * @remarks Requires CR0!
6714 */
6715static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6716 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6717{
6718 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6719 AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6720 Assert(puIntrState);
6721 uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6722
6723 const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6724 const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6725
6726#ifdef VBOX_STRICT
6727 /* Validate the error-code-valid bit for hardware exceptions. */
6728 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
6729 {
6730 switch (uVector)
6731 {
6732 case X86_XCPT_PF:
6733 case X86_XCPT_DF:
6734 case X86_XCPT_TS:
6735 case X86_XCPT_NP:
6736 case X86_XCPT_SS:
6737 case X86_XCPT_GP:
6738 case X86_XCPT_AC:
6739 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo),
6740 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
6741 /* fallthru */
6742 default:
6743 break;
6744 }
6745 }
6746#endif
6747
6748 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
6749 Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6750 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6751
6752 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6753
6754 /* We require CR0 to check if the guest is in real-mode. */
6755 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6756 AssertRCReturn(rc, rc);
6757
6758 /*
6759 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6760 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6761 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6762 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6763 */
6764 if (CPUMIsGuestInRealModeEx(pMixedCtx))
6765 {
6766 PVM pVM = pVCpu->CTX_SUFF(pVM);
6767 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6768 {
6769 Assert(PDMVmmDevHeapIsEnabled(pVM));
6770 Assert(pVM->hm.s.vmx.pRealModeTSS);
6771
6772 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6773 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6774 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6775 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6776 AssertRCReturn(rc, rc);
6777 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6778
6779 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6780 const size_t cbIdtEntry = 4;
6781 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6782 {
6783 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6784 if (uVector == X86_XCPT_DF)
6785 return VINF_EM_RESET;
6786 else if (uVector == X86_XCPT_GP)
6787 {
6788 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6789 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6790 }
6791
6792 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6793 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6794 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, puIntrState);
6795 }
6796
6797 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6798 uint16_t uGuestIp = pMixedCtx->ip;
6799 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6800 {
6801 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
6802 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
6803 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6804 }
6805 else if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6806 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6807
6808 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6809 uint16_t offIdtEntry = 0;
6810 RTSEL selIdtEntry = 0;
6811 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6812 rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6813 rc |= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6814 AssertRCReturn(rc, rc);
6815
6816 /* Construct the stack frame for the interrupt/exception handler. */
6817 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6818 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6819 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6820 AssertRCReturn(rc, rc);
6821
6822 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
6823 if (rc == VINF_SUCCESS)
6824 {
6825 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
6826 pMixedCtx->rip = offIdtEntry;
6827 pMixedCtx->cs.Sel = selIdtEntry;
6828 pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6829 if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6830 && uVector == X86_XCPT_PF)
6831 {
6832 pMixedCtx->cr2 = GCPtrFaultAddress;
6833 }
6834
6835 /* If any other guest-state bits are changed here, make sure to update
6836 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
6837 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SEGMENT_REGS
6838 | HM_CHANGED_GUEST_RIP
6839 | HM_CHANGED_GUEST_RFLAGS
6840 | HM_CHANGED_GUEST_RSP;
6841
6842 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6843 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6844 {
6845 Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6846 && uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6847 Log4(("Clearing inhibition due to STI.\n"));
6848 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6849 }
6850 Log4(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6851 }
6852 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
6853 return rc;
6854 }
6855 else
6856 {
6857 /*
6858 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6859 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6860 */
6861 u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6862 }
6863 }
6864
6865 /* Validate. */
6866 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6867 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6868 Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6869
6870 /* Inject. */
6871 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6872 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6873 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6874 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6875
6876 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6877 && uVector == X86_XCPT_PF)
6878 {
6879 pMixedCtx->cr2 = GCPtrFaultAddress;
6880 }
6881
6882 Log4(("Injecting vcpu[%RU32] u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
6883 u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6884
6885 AssertRCReturn(rc, rc);
6886 return rc;
6887}
6888
6889
6890/**
6891 * Enters the VT-x session.
6892 *
6893 * @returns VBox status code.
6894 * @param pVM Pointer to the VM.
6895 * @param pVCpu Pointer to the VMCPU.
6896 * @param pCpu Pointer to the CPU info struct.
6897 */
6898VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
6899{
6900 AssertPtr(pVM);
6901 AssertPtr(pVCpu);
6902 Assert(pVM->hm.s.vmx.fSupported);
6903 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6904 NOREF(pCpu);
6905
6906 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6907 Assert(pVCpu->hm.s.fContextUseFlags & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
6908
6909#ifdef VBOX_STRICT
6910 /* Make sure we're in VMX root mode. */
6911 RTCCUINTREG u32HostCR4 = ASMGetCR4();
6912 if (!(u32HostCR4 & X86_CR4_VMXE))
6913 {
6914 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6915 return VERR_VMX_X86_CR4_VMXE_CLEARED;
6916 }
6917#endif
6918
6919 /*
6920 * Load the VCPU's VMCS as the current (and active) one.
6921 */
6922 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
6923 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6924 if (RT_FAILURE(rc))
6925 return rc;
6926
6927 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
6928 pVCpu->hm.s.fLeaveDone = false;
6929 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
6930
6931 return VINF_SUCCESS;
6932}
6933
6934
6935/**
6936 * The thread-context callback (only on platforms which support it).
6937 *
6938 * @param enmEvent The thread-context event.
6939 * @param pVCpu Pointer to the VMCPU.
6940 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
6941 * @thread EMT.
6942 */
6943VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
6944{
6945 switch (enmEvent)
6946 {
6947 case RTTHREADCTXEVENT_PREEMPTING:
6948 {
6949 /** @todo Stats. */
6950 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6951 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
6952 VMCPU_ASSERT_EMT(pVCpu);
6953
6954 PVM pVM = pVCpu->CTX_SUFF(pVM);
6955 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
6956
6957 /* No longjmps (logger flushes, locks) in this fragile context. */
6958 VMMRZCallRing3Disable(pVCpu);
6959 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
6960
6961 /* Save the guest-state, restore host-state (FPU, debug etc.). */
6962 if (!pVCpu->hm.s.fLeaveDone)
6963 {
6964 hmR0VmxLeave(pVM, pVCpu, pMixedCtx);
6965 pVCpu->hm.s.fLeaveDone = true;
6966 }
6967
6968 /* Leave HM context, takes care of local init (term). */
6969 int rc = HMR0LeaveCpu(pVCpu);
6970 AssertRC(rc); NOREF(rc);
6971
6972 /* Restore longjmp state. */
6973 VMMRZCallRing3Enable(pVCpu);
6974 break;
6975 }
6976
6977 case RTTHREADCTXEVENT_RESUMED:
6978 {
6979 /** @todo Stats. */
6980 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6981 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
6982 VMCPU_ASSERT_EMT(pVCpu);
6983
6984 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
6985 VMMRZCallRing3Disable(pVCpu);
6986 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
6987
6988 /* Initialize the bare minimum state required for HM. This takes care of
6989 initializing VT-x if necessary (onlined CPUs, local init etc.) */
6990 int rc = HMR0EnterCpu(pVCpu);
6991 AssertRC(rc);
6992 Assert(pVCpu->hm.s.fContextUseFlags & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
6993
6994 /* Load the active VMCS as the current one. */
6995 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
6996 {
6997 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6998 AssertRC(rc); NOREF(rc);
6999 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7000 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7001 }
7002 pVCpu->hm.s.fLeaveDone = false;
7003 VMMRZCallRing3Enable(pVCpu);
7004 break;
7005 }
7006
7007 default:
7008 break;
7009 }
7010}
7011
7012
7013/**
7014 * Saves the host state in the VMCS host-state.
7015 * Sets up the VM-exit MSR-load area.
7016 *
7017 * The CPU state will be loaded from these fields on every successful VM-exit.
7018 *
7019 * @returns VBox status code.
7020 * @param pVM Pointer to the VM.
7021 * @param pVCpu Pointer to the VMCPU.
7022 *
7023 * @remarks No-long-jump zone!!!
7024 */
7025static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7026{
7027 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7028
7029 if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
7030 return VINF_SUCCESS;
7031
7032 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7033 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7034
7035 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7036 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7037
7038 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7039 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7040
7041 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
7042 return rc;
7043}
7044
7045
7046/**
7047 * Saves the host state in the VMCS host-state.
7048 *
7049 * @returns VBox status code.
7050 * @param pVM Pointer to the VM.
7051 * @param pVCpu Pointer to the VMCPU.
7052 *
7053 * @remarks No-long-jump zone!!!
7054 */
7055VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7056{
7057 AssertPtr(pVM);
7058 AssertPtr(pVCpu);
7059
7060 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7061
7062 /* When thread-context hooks are available, this is done later (when preemption/interrupts are disabled). */
7063 if (!VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7064 {
7065 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7066 return hmR0VmxSaveHostState(pVM, pVCpu);
7067 }
7068 return VINF_SUCCESS;
7069}
7070
7071
7072/**
7073 * Loads the guest state into the VMCS guest-state area. The CPU state will be
7074 * loaded from these fields on every successful VM-entry.
7075 *
7076 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7077 * Sets up the VM-entry controls.
7078 * Sets up the appropriate VMX non-root function to execute guest code based on
7079 * the guest CPU mode.
7080 *
7081 * @returns VBox status code.
7082 * @param pVM Pointer to the VM.
7083 * @param pVCpu Pointer to the VMCPU.
7084 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7085 * out-of-sync. Make sure to update the required fields
7086 * before using them.
7087 *
7088 * @remarks No-long-jump zone!!!
7089 */
7090static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7091{
7092 AssertPtr(pVM);
7093 AssertPtr(pVCpu);
7094 AssertPtr(pMixedCtx);
7095 HMVMX_ASSERT_PREEMPT_SAFE();
7096
7097#ifdef LOG_ENABLED
7098 /** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7099 * probably not initialized yet? Anyway this will do for now.
7100 *
7101 * Update: Should be possible once VMXR0LoadGuestState() is removed as an
7102 * interface and disable ring-3 calls when thread-context hooks are not
7103 * available. */
7104 bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7105 VMMR0LogFlushDisable(pVCpu);
7106#endif
7107
7108 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7109
7110 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7111
7112 /* Determine real-on-v86 mode. */
7113 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7114 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7115 && CPUMIsGuestInRealModeEx(pMixedCtx))
7116 {
7117 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7118 }
7119
7120 /*
7121 * Load the guest-state into the VMCS.
7122 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7123 * Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7124 */
7125 int rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7126 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7127
7128 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7129 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7130
7131 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7132 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7133
7134 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7135 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7136
7137 /* Assumes CR0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7138 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7139 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7140
7141 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7142 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7143
7144 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7145 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7146
7147 /*
7148 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7149 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7150 */
7151 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7152 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7153
7154 rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7155 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7156
7157 /* Clear any unused and reserved bits. */
7158 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR2;
7159
7160#ifdef LOG_ENABLED
7161 /* Only reenable log-flushing if the caller has it enabled. */
7162 if (!fCallerDisabledLogFlush)
7163 VMMR0LogFlushEnable(pVCpu);
7164#endif
7165
7166 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7167 return rc;
7168}
7169
7170
7171/**
7172 * Loads the state shared between the host and guest into the VMCS.
7173 *
7174 * @param pVM Pointer to the VM.
7175 * @param pVCpu Pointer to the VMCPU.
7176 * @param pCtx Pointer to the guest-CPU context.
7177 *
7178 * @remarks No-long-jump zone!!!
7179 */
7180static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7181{
7182 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7183 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7184
7185 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
7186 {
7187 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7188 AssertRC(rc);
7189 }
7190
7191 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG)
7192 {
7193 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7194 AssertRC(rc);
7195
7196 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7197 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
7198 {
7199 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7200 AssertRC(rc);
7201 }
7202 }
7203
7204 AssertMsg(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_GUEST_SHARED_STATE), ("fContextUseFlags=%#x\n",
7205 pVCpu->hm.s.fContextUseFlags));
7206}
7207
7208
7209/**
7210 * Worker for loading the guest-state bits in the inner VT-x execution loop.
7211 *
7212 * @param pVM Pointer to the VM.
7213 * @param pVCpu Pointer to the VMCPU.
7214 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7215 * out-of-sync. Make sure to update the required fields
7216 * before using them.
7217 */
7218DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7219{
7220 HMVMX_ASSERT_PREEMPT_SAFE();
7221
7222 Log5(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
7223#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7224 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_ALL_GUEST;
7225#endif
7226
7227 if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
7228 {
7229 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7230 AssertRC(rc);
7231 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7232 }
7233 else if (pVCpu->hm.s.fContextUseFlags)
7234 {
7235 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7236 AssertRC(rc);
7237 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7238 }
7239
7240 /* All the guest state bits should be loaded except maybe the host context and shared host/guest bits. */
7241 AssertMsg( !(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_ALL_GUEST)
7242 || !(pVCpu->hm.s.fContextUseFlags & ~(HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE)),
7243 ("fContextUseFlags=%#x\n", pVCpu->hm.s.fContextUseFlags));
7244
7245#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7246 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7247 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7248 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7249#endif
7250}
7251
7252
7253/**
7254 * Does the preparations before executing guest code in VT-x.
7255 *
7256 * This may cause longjmps to ring-3 and may even result in rescheduling to the
7257 * recompiler. We must be cautious what we do here regarding committing
7258 * guest-state information into the VMCS assuming we assuredly execute the
7259 * guest in VT-x. If we fall back to the recompiler after updating the VMCS and
7260 * clearing the common-state (TRPM/forceflags), we must undo those changes so
7261 * that the recompiler can (and should) use them when it resumes guest
7262 * execution. Otherwise such operations must be done when we can no longer
7263 * exit to ring-3.
7264 *
7265 * @returns VBox status code (informational status codes included).
7266 * @retval VINF_SUCCESS if we can proceed with running the guest.
7267 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
7268 * into the guest.
7269 * @retval VINF_* scheduling changes, we have to go back to ring-3.
7270 *
7271 * @param pVM Pointer to the VM.
7272 * @param pVCpu Pointer to the VMCPU.
7273 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7274 * out-of-sync. Make sure to update the required fields
7275 * before using them.
7276 * @param pVmxTransient Pointer to the VMX transient structure.
7277 *
7278 * @remarks Called with preemption disabled.
7279 */
7280static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7281{
7282 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7283
7284#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7285 PGMRZDynMapFlushAutoSet(pVCpu);
7286#endif
7287
7288 /* Check force flag actions that might require us to go back to ring-3. */
7289 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7290 if (rc != VINF_SUCCESS)
7291 return rc;
7292
7293#ifndef IEM_VERIFICATION_MODE_FULL
7294 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7295 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7296 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7297 {
7298 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7299 RTGCPHYS GCPhysApicBase;
7300 GCPhysApicBase = pMixedCtx->msrApicBase;
7301 GCPhysApicBase &= PAGE_BASE_GC_MASK;
7302
7303 /* Unalias any existing mapping. */
7304 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7305 AssertRCReturn(rc, rc);
7306
7307 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7308 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7309 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
7310 AssertRCReturn(rc, rc);
7311
7312 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7313 }
7314#endif /* !IEM_VERIFICATION_MODE_FULL */
7315
7316 /* Load the guest state bits, we can handle longjmps/getting preempted here. */
7317 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7318
7319 /*
7320 * Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
7321 * we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
7322 */
7323 if (TRPMHasTrap(pVCpu))
7324 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7325 else if (!pVCpu->hm.s.Event.fPending)
7326 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
7327
7328 /*
7329 * No longjmps to ring-3 from this point on!!!
7330 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7331 * This also disables flushing of the R0-logger instance (if any).
7332 */
7333 VMMRZCallRing3Disable(pVCpu);
7334
7335 /*
7336 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
7337 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
7338 *
7339 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
7340 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
7341 *
7342 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
7343 * executing guest code.
7344 */
7345 pVmxTransient->uEflags = ASMIntDisableFlags();
7346 if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
7347 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7348 {
7349 ASMSetFlags(pVmxTransient->uEflags);
7350 VMMRZCallRing3Enable(pVCpu);
7351 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7352 return VINF_EM_RAW_TO_R3;
7353 }
7354 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
7355 {
7356 ASMSetFlags(pVmxTransient->uEflags);
7357 VMMRZCallRing3Enable(pVCpu);
7358 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
7359 return VINF_EM_RAW_INTERRUPT;
7360 }
7361
7362 /*
7363 * Event injection might result in triple-faulting the VM (real-on-v86 case), which is why it's
7364 * done here and not in hmR0VmxPreRunGuestCommitted() which doesn't expect failures.
7365 */
7366 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7367 if (RT_UNLIKELY(rc != VINF_SUCCESS))
7368 {
7369 ASMSetFlags(pVmxTransient->uEflags);
7370 VMMRZCallRing3Enable(pVCpu);
7371 return rc;
7372 }
7373
7374 return VINF_SUCCESS;
7375}
7376
7377
7378/**
7379 * Prepares to run guest code in VT-x and we've committed to doing so. This
7380 * means there is no backing out to ring-3 or anywhere else at this
7381 * point.
7382 *
7383 * @param pVM Pointer to the VM.
7384 * @param pVCpu Pointer to the VMCPU.
7385 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7386 * out-of-sync. Make sure to update the required fields
7387 * before using them.
7388 * @param pVmxTransient Pointer to the VMX transient structure.
7389 *
7390 * @remarks Called with preemption disabled.
7391 * @remarks No-long-jump zone!!!
7392 */
7393static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7394{
7395 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7396 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7397 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7398
7399 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7400 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
7401
7402 /*
7403 * Load the host state bits as we may've been preempted (only happens when
7404 * thread-context hooks are used).
7405 */
7406 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT)
7407 {
7408 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7409 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
7410 AssertRC(rc);
7411 }
7412 Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
7413
7414 /*
7415 * If we are injecting events to a real-on-v86 mode guest, we may have to update
7416 * RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
7417 * Reload only the necessary state, the assertion will catch if other parts of the code
7418 * change.
7419 */
7420 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7421 {
7422 hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7423 hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7424 }
7425
7426 /* Load the state shared between host and guest (FPU, debug). */
7427 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_GUEST_SHARED_STATE)
7428 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
7429 AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags=%#x\n", pVCpu->hm.s.fContextUseFlags));
7430
7431 /*
7432 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
7433 */
7434 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7435 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
7436
7437 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
7438 || HMR0GetCurrentCpu()->idCpu != pVCpu->hm.s.idLastCpu)
7439 {
7440 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
7441 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
7442 }
7443
7444 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
7445 hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
7446
7447 RTCPUID idCurrentCpu = HMR0GetCurrentCpu()->idCpu;
7448 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
7449 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
7450
7451 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
7452
7453 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
7454 to start executing. */
7455
7456#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7457 /*
7458 * Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
7459 * RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
7460 */
7461 if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7462 && !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7463 {
7464 pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
7465 uint64_t u64HostTscAux = 0;
7466 int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
7467 AssertRC(rc2);
7468 ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
7469 }
7470#endif
7471}
7472
7473
7474/**
7475 * Performs some essential restoration of state after running guest code in
7476 * VT-x.
7477 *
7478 * @param pVM Pointer to the VM.
7479 * @param pVCpu Pointer to the VMCPU.
7480 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7481 * out-of-sync. Make sure to update the required fields
7482 * before using them.
7483 * @param pVmxTransient Pointer to the VMX transient structure.
7484 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
7485 *
7486 * @remarks Called with interrupts disabled.
7487 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
7488 * unconditionally when it is safe to do so.
7489 */
7490static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
7491{
7492 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7493
7494 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
7495 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
7496 pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
7497 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
7498 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
7499
7500 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7501 {
7502#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7503 /* Restore host's TSC_AUX. */
7504 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7505 ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
7506#endif
7507 /** @todo Find a way to fix hardcoding a guestimate. */
7508 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
7509 + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
7510 }
7511
7512 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
7513 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
7514 Assert(!(ASMGetFlags() & X86_EFL_IF));
7515 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7516
7517 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
7518 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
7519 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
7520 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
7521
7522 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
7523 uint32_t uExitReason;
7524 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
7525 rc |= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7526 AssertRC(rc);
7527 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
7528 pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
7529
7530 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
7531 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
7532 {
7533 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
7534 pVmxTransient->fVMEntryFailed));
7535 return;
7536 }
7537
7538 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
7539 {
7540 /* Update the guest interruptibility-state from the VMCS. */
7541 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
7542#if defined(HMVMX_SYNC_FULL_GUEST_STATE) || defined(HMVMX_SAVE_FULL_GUEST_STATE)
7543 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7544 AssertRC(rc);
7545#endif
7546 /*
7547 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
7548 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
7549 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
7550 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
7551 */
7552 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7553 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
7554 {
7555 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
7556 AssertRC(rc);
7557 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
7558 }
7559 }
7560}
7561
7562
7563
7564/**
7565 * Runs the guest code using VT-x the normal way.
7566 *
7567 * @returns VBox status code.
7568 * @param pVM Pointer to the VM.
7569 * @param pVCpu Pointer to the VMCPU.
7570 * @param pCtx Pointer to the guest-CPU context.
7571 *
7572 * @note Mostly the same as hmR0VmxRunGuestCodeStep.
7573 * @remarks Called with preemption disabled.
7574 */
7575static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7576{
7577 VMXTRANSIENT VmxTransient;
7578 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7579 int rc = VERR_INTERNAL_ERROR_5;
7580 uint32_t cLoops = 0;
7581
7582 for (;; cLoops++)
7583 {
7584 Assert(!HMR0SuspendPending());
7585 HMVMX_ASSERT_CPU_SAFE();
7586
7587 /* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
7588 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7589 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7590 if (rc != VINF_SUCCESS)
7591 break;
7592
7593 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7594 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7595 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7596
7597 /* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state. */
7598 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7599
7600 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7601 if (RT_UNLIKELY(rc != VINF_SUCCESS))
7602 {
7603 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7604 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7605 return rc;
7606 }
7607
7608 /* Handle the VM-exit. */
7609 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7610 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7611 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7612 HMVMX_START_EXIT_DISPATCH_PROF();
7613#ifdef HMVMX_USE_FUNCTION_TABLE
7614 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7615#else
7616 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7617#endif
7618 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7619 if (rc != VINF_SUCCESS)
7620 break;
7621 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7622 {
7623 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7624 rc = VINF_EM_RAW_INTERRUPT;
7625 break;
7626 }
7627 }
7628
7629 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7630 return rc;
7631}
7632
7633
7634/**
7635 * Single steps guest code using VT-x.
7636 *
7637 * @returns VBox status code.
7638 * @param pVM Pointer to the VM.
7639 * @param pVCpu Pointer to the VMCPU.
7640 * @param pCtx Pointer to the guest-CPU context.
7641 *
7642 * @note Mostly the same as hmR0VmxRunGuestCodeNormal.
7643 * @remarks Called with preemption disabled.
7644 */
7645static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7646{
7647 VMXTRANSIENT VmxTransient;
7648 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7649 int rc = VERR_INTERNAL_ERROR_5;
7650 uint32_t cLoops = 0;
7651 uint16_t uCsStart = pCtx->cs.Sel;
7652 uint64_t uRipStart = pCtx->rip;
7653
7654 for (;; cLoops++)
7655 {
7656 Assert(!HMR0SuspendPending());
7657 HMVMX_ASSERT_CPU_SAFE();
7658
7659 /* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
7660 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7661 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7662 if (rc != VINF_SUCCESS)
7663 break;
7664
7665 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7666 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7667 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7668
7669 /* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state. */
7670 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7671
7672 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7673 if (RT_UNLIKELY(rc != VINF_SUCCESS))
7674 {
7675 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7676 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7677 return rc;
7678 }
7679
7680 /* Handle the VM-exit. */
7681 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7682 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7683 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7684 HMVMX_START_EXIT_DISPATCH_PROF();
7685#ifdef HMVMX_USE_FUNCTION_TABLE
7686 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7687#else
7688 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7689#endif
7690 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7691 if (rc != VINF_SUCCESS)
7692 break;
7693 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7694 {
7695 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7696 rc = VINF_EM_RAW_INTERRUPT;
7697 break;
7698 }
7699
7700 /*
7701 * Did the RIP change, if so, consider it a single step.
7702 * Otherwise, make sure one of the TFs gets set.
7703 */
7704 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
7705 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
7706 AssertRCReturn(rc2, rc2);
7707 if ( pCtx->rip != uRipStart
7708 || pCtx->cs.Sel != uCsStart)
7709 {
7710 rc = VINF_EM_DBG_STEPPED;
7711 break;
7712 }
7713 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
7714 }
7715
7716 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7717 return rc;
7718}
7719
7720
7721/**
7722 * Runs the guest code using VT-x.
7723 *
7724 * @returns VBox status code.
7725 * @param pVM Pointer to the VM.
7726 * @param pVCpu Pointer to the VMCPU.
7727 * @param pCtx Pointer to the guest-CPU context.
7728 *
7729 * @remarks Called with preemption disabled.
7730 */
7731VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7732{
7733 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7734 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
7735 HMVMX_ASSERT_PREEMPT_SAFE();
7736
7737 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
7738
7739 int rc;
7740 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
7741 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
7742 else
7743 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
7744
7745 if (rc == VERR_EM_INTERPRETER)
7746 rc = VINF_EM_RAW_EMULATE_INSTR;
7747 else if (rc == VINF_EM_RESET)
7748 rc = VINF_EM_TRIPLE_FAULT;
7749
7750 hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
7751 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
7752 return rc;
7753}
7754
7755
7756#ifndef HMVMX_USE_FUNCTION_TABLE
7757DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
7758{
7759 int rc;
7760 switch (rcReason)
7761 {
7762 case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
7763 case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
7764 case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
7765 case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
7766 case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
7767 case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
7768 case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7769 case VMX_EXIT_XCPT_OR_NMI: rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); break;
7770 case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
7771 case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
7772 case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7773 case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
7774 case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
7775 case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
7776 case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
7777 case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7778 case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7779 case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
7780 case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
7781 case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
7782 case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
7783 case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
7784 case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
7785 case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
7786 case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
7787 case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7788 case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7789 case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
7790 case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
7791 case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
7792 case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
7793 case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
7794 case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
7795
7796 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
7797 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7798 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
7799 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
7800 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7801 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7802 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
7803 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
7804 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
7805
7806 case VMX_EXIT_VMCALL:
7807 case VMX_EXIT_VMCLEAR:
7808 case VMX_EXIT_VMLAUNCH:
7809 case VMX_EXIT_VMPTRLD:
7810 case VMX_EXIT_VMPTRST:
7811 case VMX_EXIT_VMREAD:
7812 case VMX_EXIT_VMRESUME:
7813 case VMX_EXIT_VMWRITE:
7814 case VMX_EXIT_VMXOFF:
7815 case VMX_EXIT_VMXON:
7816 case VMX_EXIT_INVEPT:
7817 case VMX_EXIT_INVVPID:
7818 case VMX_EXIT_VMFUNC:
7819 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
7820 break;
7821 default:
7822 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
7823 break;
7824 }
7825 return rc;
7826}
7827#endif
7828
7829#ifdef DEBUG
7830/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
7831# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
7832 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
7833
7834# define HMVMX_ASSERT_PREEMPT_CPUID() \
7835 do \
7836 { \
7837 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
7838 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
7839 } while (0)
7840
7841# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
7842 do { \
7843 AssertPtr(pVCpu); \
7844 AssertPtr(pMixedCtx); \
7845 AssertPtr(pVmxTransient); \
7846 Assert(pVmxTransient->fVMEntryFailed == false); \
7847 Assert(ASMIntAreEnabled()); \
7848 HMVMX_ASSERT_PREEMPT_SAFE(); \
7849 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
7850 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
7851 HMVMX_ASSERT_PREEMPT_SAFE(); \
7852 if (VMMR0IsLogFlushDisabled(pVCpu)) \
7853 HMVMX_ASSERT_PREEMPT_CPUID(); \
7854 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
7855 } while (0)
7856
7857# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
7858 do { \
7859 Log4Func(("\n")); \
7860 } while(0)
7861#else /* Release builds */
7862# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
7863# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
7864#endif
7865
7866
7867/**
7868 * Advances the guest RIP after reading it from the VMCS.
7869 *
7870 * @returns VBox status code.
7871 * @param pVCpu Pointer to the VMCPU.
7872 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7873 * out-of-sync. Make sure to update the required fields
7874 * before using them.
7875 * @param pVmxTransient Pointer to the VMX transient structure.
7876 *
7877 * @remarks No-long-jump zone!!!
7878 */
7879DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7880{
7881 int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7882 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7883 AssertRCReturn(rc, rc);
7884
7885 pMixedCtx->rip += pVmxTransient->cbInstr;
7886 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
7887 return rc;
7888}
7889
7890
7891/**
7892 * Tries to determine what part of the guest-state VT-x has deemed as invalid
7893 * and update error record fields accordingly.
7894 *
7895 * @return VMX_IGS_* return codes.
7896 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
7897 * wrong with the guest state.
7898 *
7899 * @param pVM Pointer to the VM.
7900 * @param pVCpu Pointer to the VMCPU.
7901 * @param pCtx Pointer to the guest-CPU state.
7902 */
7903static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7904{
7905#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
7906#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
7907 uError = (err); \
7908 break; \
7909 } else do {} while (0)
7910/* Duplicate of IEM_IS_CANONICAL(). */
7911#define HMVMX_IS_CANONICAL(a_u64Addr) ((uint64_t)(a_u64Addr) + UINT64_C(0x800000000000) < UINT64_C(0x1000000000000))
7912
7913 int rc;
7914 uint64_t u64Val;
7915 uint32_t u32Val;
7916 uint32_t uError = VMX_IGS_ERROR;
7917 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
7918
7919 do
7920 {
7921 /*
7922 * CR0.
7923 */
7924 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
7925 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
7926 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
7927 See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
7928 if (fUnrestrictedGuest)
7929 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
7930
7931 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
7932 AssertRCBreak(rc);
7933 HMVMX_CHECK_BREAK((u32Val & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
7934 HMVMX_CHECK_BREAK(!(u32Val & ~uZapCR0), VMX_IGS_CR0_FIXED0);
7935 if ( !fUnrestrictedGuest
7936 && (u32Val & X86_CR0_PG)
7937 && !(u32Val & X86_CR0_PE))
7938 {
7939 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
7940 }
7941
7942 /*
7943 * CR4.
7944 */
7945 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
7946 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
7947 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
7948 AssertRCBreak(rc);
7949 HMVMX_CHECK_BREAK((u32Val & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
7950 HMVMX_CHECK_BREAK(!(u32Val & ~uZapCR4), VMX_IGS_CR4_FIXED0);
7951
7952 /*
7953 * IA32_DEBUGCTL MSR.
7954 */
7955 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
7956 AssertRCBreak(rc);
7957 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
7958 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
7959 {
7960 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
7961 }
7962 uint64_t u64DebugCtlMsr = u64Val;
7963
7964#ifdef VBOX_STRICT
7965 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
7966 AssertRCBreak(rc);
7967 Assert(u32Val == pVCpu->hm.s.vmx.u32ProcCtls);
7968#endif
7969 bool const fLongModeGuest = !!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
7970
7971 /*
7972 * RIP and RFLAGS.
7973 */
7974 uint32_t u32Eflags;
7975#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7976 if (HMVMX_IS_64BIT_HOST_MODE())
7977 {
7978 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
7979 AssertRCBreak(rc);
7980 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
7981 if ( !fLongModeGuest
7982 || !pCtx->cs.Attr.n.u1Long)
7983 {
7984 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
7985 }
7986 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
7987 * must be identical if the "IA32e mode guest" VM-entry control is 1
7988 * and CS.L is 1. No check applies if the CPU supports 64
7989 * linear-address bits. */
7990
7991 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
7992 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
7993 AssertRCBreak(rc);
7994 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
7995 VMX_IGS_RFLAGS_RESERVED);
7996 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
7997 u32Eflags = u64Val;
7998 }
7999 else
8000#endif
8001 {
8002 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8003 AssertRCBreak(rc);
8004 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8005 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8006 }
8007
8008 if ( fLongModeGuest
8009 || !(pCtx->cr0 & X86_CR0_PE))
8010 {
8011 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8012 }
8013
8014 uint32_t u32EntryInfo;
8015 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8016 AssertRCBreak(rc);
8017 if ( VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8018 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8019 {
8020 HMVMX_CHECK_BREAK(u32Val & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8021 }
8022
8023 /*
8024 * 64-bit checks.
8025 */
8026#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8027 if (HMVMX_IS_64BIT_HOST_MODE())
8028 {
8029 if ( fLongModeGuest
8030 && !fUnrestrictedGuest)
8031 {
8032 HMVMX_CHECK_BREAK(CPUMIsGuestPagingEnabledEx(pCtx), VMX_IGS_CR0_PG_LONGMODE);
8033 HMVMX_CHECK_BREAK((pCtx->cr4 & X86_CR4_PAE), VMX_IGS_CR4_PAE_LONGMODE);
8034 }
8035
8036 if ( !fLongModeGuest
8037 && (pCtx->cr4 & X86_CR4_PCIDE))
8038 {
8039 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8040 }
8041
8042 /** @todo CR3 field must be such that bits 63:52 and bits in the range
8043 * 51:32 beyond the processor's physical-address width are 0. */
8044
8045 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8046 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8047 {
8048 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8049 }
8050
8051 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8052 AssertRCBreak(rc);
8053 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8054
8055 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8056 AssertRCBreak(rc);
8057 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8058 }
8059#endif
8060
8061 /*
8062 * PERF_GLOBAL MSR.
8063 */
8064 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8065 {
8066 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8067 AssertRCBreak(rc);
8068 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8069 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8070 }
8071
8072 /*
8073 * PAT MSR.
8074 */
8075 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8076 {
8077 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8078 AssertRCBreak(rc);
8079 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8080 for (unsigned i = 0; i < 8; i++)
8081 {
8082 uint8_t u8Val = (u64Val & 0x7);
8083 if ( u8Val != 0 /* UC */
8084 || u8Val != 1 /* WC */
8085 || u8Val != 4 /* WT */
8086 || u8Val != 5 /* WP */
8087 || u8Val != 6 /* WB */
8088 || u8Val != 7 /* UC- */)
8089 {
8090 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8091 }
8092 u64Val >>= 3;
8093 }
8094 }
8095
8096 /*
8097 * EFER MSR.
8098 */
8099 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8100 {
8101 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8102 AssertRCBreak(rc);
8103 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8104 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8105 HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8106 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8107 HMVMX_CHECK_BREAK( fUnrestrictedGuest
8108 || (u64Val & MSR_K6_EFER_LMA) == (pCtx->cr0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8109 }
8110
8111 /*
8112 * Segment registers.
8113 */
8114 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8115 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8116 if (!(u32Eflags & X86_EFL_VM))
8117 {
8118 /* CS */
8119 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8120 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8121 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8122 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8123 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8124 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8125 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8126 /* CS cannot be loaded with NULL in protected mode. */
8127 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8128 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8129 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
8130 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8131 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
8132 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8133 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8134 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8135 else
8136 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8137
8138 /* SS */
8139 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8140 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8141 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8142 if ( !(pCtx->cr0 & X86_CR0_PE)
8143 || pCtx->cs.Attr.n.u4Type == 3)
8144 {
8145 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8146 }
8147 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8148 {
8149 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8150 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8151 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8152 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8153 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8154 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8155 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8156 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8157 }
8158
8159 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8160 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8161 {
8162 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8163 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8164 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8165 || pCtx->ds.Attr.n.u4Type > 11
8166 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8167 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8168 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8169 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8170 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8171 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8172 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8173 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8174 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8175 }
8176 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8177 {
8178 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8179 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8180 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8181 || pCtx->es.Attr.n.u4Type > 11
8182 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8183 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8184 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8185 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8186 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8187 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8188 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8189 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8190 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8191 }
8192 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8193 {
8194 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8195 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8196 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8197 || pCtx->fs.Attr.n.u4Type > 11
8198 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8199 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8200 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8201 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8202 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8203 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8204 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8205 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8206 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8207 }
8208 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8209 {
8210 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8211 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8212 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8213 || pCtx->gs.Attr.n.u4Type > 11
8214 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8215 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8216 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8217 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8218 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8219 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8220 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8221 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8222 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8223 }
8224 /* 64-bit capable CPUs. */
8225#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8226 if (HMVMX_IS_64BIT_HOST_MODE())
8227 {
8228 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8229 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8230 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8231 || HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8232 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8233 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
8234 VMX_IGS_LONGMODE_SS_BASE_INVALID);
8235 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
8236 VMX_IGS_LONGMODE_DS_BASE_INVALID);
8237 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
8238 VMX_IGS_LONGMODE_ES_BASE_INVALID);
8239 }
8240#endif
8241 }
8242 else
8243 {
8244 /* V86 mode checks. */
8245 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8246 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8247 {
8248 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8249 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8250 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8251 }
8252 else
8253 {
8254 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8255 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8256 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8257 }
8258
8259 /* CS */
8260 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
8261 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
8262 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
8263 /* SS */
8264 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
8265 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
8266 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
8267 /* DS */
8268 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
8269 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
8270 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
8271 /* ES */
8272 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
8273 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
8274 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
8275 /* FS */
8276 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
8277 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
8278 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
8279 /* GS */
8280 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
8281 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
8282 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
8283 /* 64-bit capable CPUs. */
8284#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8285 if (HMVMX_IS_64BIT_HOST_MODE())
8286 {
8287 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8288 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8289 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8290 || HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8291 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8292 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
8293 VMX_IGS_LONGMODE_SS_BASE_INVALID);
8294 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
8295 VMX_IGS_LONGMODE_DS_BASE_INVALID);
8296 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
8297 VMX_IGS_LONGMODE_ES_BASE_INVALID);
8298 }
8299#endif
8300 }
8301
8302 /*
8303 * TR.
8304 */
8305 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
8306 /* 64-bit capable CPUs. */
8307#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8308 if (HMVMX_IS_64BIT_HOST_MODE())
8309 {
8310 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
8311 }
8312#endif
8313 if (fLongModeGuest)
8314 {
8315 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
8316 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
8317 }
8318 else
8319 {
8320 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
8321 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
8322 VMX_IGS_TR_ATTR_TYPE_INVALID);
8323 }
8324 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
8325 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
8326 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
8327 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
8328 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8329 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
8330 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8331 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
8332
8333 /*
8334 * GDTR and IDTR.
8335 */
8336#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8337 if (HMVMX_IS_64BIT_HOST_MODE())
8338 {
8339 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8340 AssertRCBreak(rc);
8341 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
8342
8343 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8344 AssertRCBreak(rc);
8345 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
8346 }
8347#endif
8348
8349 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8350 AssertRCBreak(rc);
8351 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8352
8353 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8354 AssertRCBreak(rc);
8355 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8356
8357 /*
8358 * Guest Non-Register State.
8359 */
8360 /* Activity State. */
8361 uint32_t u32ActivityState;
8362 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
8363 AssertRCBreak(rc);
8364 HMVMX_CHECK_BREAK( !u32ActivityState
8365 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
8366 VMX_IGS_ACTIVITY_STATE_INVALID);
8367 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
8368 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
8369 uint32_t u32IntrState;
8370 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
8371 AssertRCBreak(rc);
8372 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
8373 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8374 {
8375 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
8376 }
8377
8378 /** @todo Activity state and injecting interrupts. Left as a todo since we
8379 * currently don't use activity states but ACTIVE. */
8380
8381 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8382 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
8383
8384 /* Guest interruptibility-state. */
8385 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
8386 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8387 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
8388 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8389 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8390 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
8391 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
8392 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8393 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
8394 if (VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo))
8395 {
8396 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8397 {
8398 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8399 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8400 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
8401 }
8402 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8403 {
8404 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8405 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
8406 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8407 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
8408 }
8409 }
8410 /** @todo Assumes the processor is not in SMM. */
8411 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8412 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
8413 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8414 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8415 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
8416 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
8417 && VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8418 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8419 {
8420 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
8421 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
8422 }
8423
8424 /* Pending debug exceptions. */
8425 if (HMVMX_IS_64BIT_HOST_MODE())
8426 {
8427 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
8428 AssertRCBreak(rc);
8429 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
8430 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
8431 u32Val = u64Val; /* For pending debug exceptions checks below. */
8432 }
8433 else
8434 {
8435 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
8436 AssertRCBreak(rc);
8437 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
8438 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
8439 }
8440
8441 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8442 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
8443 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
8444 {
8445 if ( (u32Eflags & X86_EFL_TF)
8446 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8447 {
8448 /* Bit 14 is PendingDebug.BS. */
8449 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
8450 }
8451 if ( !(u32Eflags & X86_EFL_TF)
8452 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8453 {
8454 /* Bit 14 is PendingDebug.BS. */
8455 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
8456 }
8457 }
8458
8459 /* VMCS link pointer. */
8460 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
8461 AssertRCBreak(rc);
8462 if (u64Val != UINT64_C(0xffffffffffffffff))
8463 {
8464 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
8465 /** @todo Bits beyond the processor's physical-address width MBZ. */
8466 /** @todo 32-bit located in memory referenced by value of this field (as a
8467 * physical address) must contain the processor's VMCS revision ID. */
8468 /** @todo SMM checks. */
8469 }
8470
8471 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries. */
8472
8473 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
8474 if (uError == VMX_IGS_ERROR)
8475 uError = VMX_IGS_REASON_NOT_FOUND;
8476 } while (0);
8477
8478 pVCpu->hm.s.u32HMError = uError;
8479 return uError;
8480
8481#undef HMVMX_ERROR_BREAK
8482#undef HMVMX_CHECK_BREAK
8483#undef HMVMX_IS_CANONICAL
8484}
8485
8486/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8487/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
8488/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8489
8490/** @name VM-exit handlers.
8491 * @{
8492 */
8493
8494/**
8495 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
8496 */
8497HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8498{
8499 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8500 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
8501 /* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
8502#if HC_ARCH_BITS == 64
8503 Assert(ASMIntAreEnabled());
8504 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUsePreemptTimer)
8505 return VINF_SUCCESS;
8506#endif
8507 return VINF_EM_RAW_INTERRUPT;
8508}
8509
8510
8511/**
8512 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
8513 */
8514HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8515{
8516 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8517 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
8518
8519 int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8520 AssertRCReturn(rc, rc);
8521
8522 uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
8523 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
8524 && uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8525 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo));
8526
8527 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8528 {
8529 /*
8530 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
8531 * anything we inject is not going to cause a VM-exit directly for the event being injected.
8532 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
8533 *
8534 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
8535 */
8536 VMXDispatchHostNmi();
8537 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmi);
8538 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8539 return VINF_SUCCESS;
8540 }
8541
8542 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8543 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8544 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8545 {
8546 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8547 return VINF_SUCCESS;
8548 }
8549 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8550 {
8551 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8552 return rc;
8553 }
8554
8555 uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
8556 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
8557 switch (uIntrType)
8558 {
8559 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
8560 Assert(uVector == X86_XCPT_DB || uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
8561 /* no break */
8562 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
8563 {
8564 switch (uVector)
8565 {
8566 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
8567 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
8568 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
8569 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
8570 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
8571 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
8572#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8573 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
8574 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8575 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
8576 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8577 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8578 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8579 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
8580 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8581 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
8582 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8583#endif
8584 default:
8585 {
8586 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8587 AssertRCReturn(rc, rc);
8588
8589 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
8590 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8591 {
8592 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
8593 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
8594 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8595
8596 rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8597 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8598 AssertRCReturn(rc, rc);
8599 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
8600 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
8601 0 /* GCPtrFaultAddress */);
8602 AssertRCReturn(rc, rc);
8603 }
8604 else
8605 {
8606 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
8607 pVCpu->hm.s.u32HMError = uVector;
8608 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
8609 }
8610 break;
8611 }
8612 }
8613 break;
8614 }
8615
8616 default:
8617 {
8618 pVCpu->hm.s.u32HMError = uExitIntrInfo;
8619 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
8620 AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
8621 break;
8622 }
8623 }
8624 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8625 return rc;
8626}
8627
8628
8629/**
8630 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
8631 */
8632HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8633{
8634 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8635
8636 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
8637 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
8638 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
8639 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8640 AssertRCReturn(rc, rc);
8641
8642 /* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
8643 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
8644 return VINF_SUCCESS;
8645}
8646
8647
8648/**
8649 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
8650 */
8651HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8652{
8653 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8654 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
8655 pVCpu->hm.s.u32HMError = VMX_EXIT_NMI_WINDOW;
8656 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8657}
8658
8659
8660/**
8661 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
8662 */
8663HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8664{
8665 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8666 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
8667 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8668}
8669
8670
8671/**
8672 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
8673 */
8674HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8675{
8676 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8677 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
8678 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8679}
8680
8681
8682/**
8683 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
8684 */
8685HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8686{
8687 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8688 PVM pVM = pVCpu->CTX_SUFF(pVM);
8689 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8690 if (RT_LIKELY(rc == VINF_SUCCESS))
8691 {
8692 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8693 Assert(pVmxTransient->cbInstr == 2);
8694 }
8695 else
8696 {
8697 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
8698 rc = VERR_EM_INTERPRETER;
8699 }
8700 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
8701 return rc;
8702}
8703
8704
8705/**
8706 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
8707 */
8708HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8709{
8710 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8711 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
8712 AssertRCReturn(rc, rc);
8713
8714 if (pMixedCtx->cr4 & X86_CR4_SMXE)
8715 return VINF_EM_RAW_EMULATE_INSTR;
8716
8717 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
8718 pVCpu->hm.s.u32HMError = VMX_EXIT_GETSEC;
8719 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8720}
8721
8722
8723/**
8724 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
8725 */
8726HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8727{
8728 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8729 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8730 AssertRCReturn(rc, rc);
8731
8732 PVM pVM = pVCpu->CTX_SUFF(pVM);
8733 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8734 if (RT_LIKELY(rc == VINF_SUCCESS))
8735 {
8736 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8737 Assert(pVmxTransient->cbInstr == 2);
8738 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
8739 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
8740 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8741 }
8742 else
8743 {
8744 AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
8745 rc = VERR_EM_INTERPRETER;
8746 }
8747 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
8748 return rc;
8749}
8750
8751
8752/**
8753 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
8754 */
8755HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8756{
8757 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8758 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8759 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
8760 AssertRCReturn(rc, rc);
8761
8762 PVM pVM = pVCpu->CTX_SUFF(pVM);
8763 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
8764 if (RT_LIKELY(rc == VINF_SUCCESS))
8765 {
8766 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8767 Assert(pVmxTransient->cbInstr == 3);
8768 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
8769 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
8770 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8771 }
8772 else
8773 {
8774 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
8775 rc = VERR_EM_INTERPRETER;
8776 }
8777 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
8778 return rc;
8779}
8780
8781
8782/**
8783 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
8784 */
8785HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8786{
8787 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8788 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8789 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
8790 AssertRCReturn(rc, rc);
8791
8792 PVM pVM = pVCpu->CTX_SUFF(pVM);
8793 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8794 if (RT_LIKELY(rc == VINF_SUCCESS))
8795 {
8796 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8797 Assert(pVmxTransient->cbInstr == 2);
8798 }
8799 else
8800 {
8801 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
8802 rc = VERR_EM_INTERPRETER;
8803 }
8804 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
8805 return rc;
8806}
8807
8808
8809/**
8810 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
8811 */
8812HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8813{
8814 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8815 PVM pVM = pVCpu->CTX_SUFF(pVM);
8816 Assert(!pVM->hm.s.fNestedPaging);
8817
8818 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8819 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8820 AssertRCReturn(rc, rc);
8821
8822 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
8823 rc = VBOXSTRICTRC_VAL(rc2);
8824 if (RT_LIKELY(rc == VINF_SUCCESS))
8825 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8826 else
8827 {
8828 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
8829 pVmxTransient->uExitQualification, rc));
8830 }
8831 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
8832 return rc;
8833}
8834
8835
8836/**
8837 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
8838 */
8839HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8840{
8841 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8842 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8843 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8844 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8845 AssertRCReturn(rc, rc);
8846
8847 PVM pVM = pVCpu->CTX_SUFF(pVM);
8848 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8849 if (RT_LIKELY(rc == VINF_SUCCESS))
8850 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8851 else
8852 {
8853 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
8854 rc = VERR_EM_INTERPRETER;
8855 }
8856 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
8857 return rc;
8858}
8859
8860
8861/**
8862 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
8863 */
8864HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8865{
8866 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8867 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8868 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8869 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8870 AssertRCReturn(rc, rc);
8871
8872 PVM pVM = pVCpu->CTX_SUFF(pVM);
8873 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8874 rc = VBOXSTRICTRC_VAL(rc2);
8875 if (RT_LIKELY( rc == VINF_SUCCESS
8876 || rc == VINF_EM_HALT))
8877 {
8878 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8879 AssertRCReturn(rc3, rc3);
8880
8881 if ( rc == VINF_EM_HALT
8882 && EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
8883 {
8884 rc = VINF_SUCCESS;
8885 }
8886 }
8887 else
8888 {
8889 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
8890 rc = VERR_EM_INTERPRETER;
8891 }
8892 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
8893 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
8894 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
8895 return rc;
8896}
8897
8898
8899/**
8900 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
8901 */
8902HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8903{
8904 /*
8905 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
8906 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
8907 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
8908 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
8909 */
8910 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8911 pVCpu->hm.s.u32HMError = VMX_EXIT_RSM;
8912 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8913}
8914
8915
8916/**
8917 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
8918 */
8919HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8920{
8921 /*
8922 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
8923 * root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
8924 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
8925 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
8926 */
8927 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8928 pVCpu->hm.s.u32HMError = VMX_EXIT_SMI;
8929 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8930}
8931
8932
8933/**
8934 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
8935 */
8936HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8937{
8938 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
8939 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8940 pVCpu->hm.s.u32HMError = VMX_EXIT_IO_SMI;
8941 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8942}
8943
8944
8945/**
8946 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
8947 */
8948HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8949{
8950 /*
8951 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
8952 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
8953 * See Intel spec. 25.3 "Other Causes of VM-exits".
8954 */
8955 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8956 pVCpu->hm.s.u32HMError = VMX_EXIT_SIPI;
8957 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8958}
8959
8960
8961/**
8962 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
8963 * VM-exit.
8964 */
8965HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8966{
8967 /*
8968 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
8969 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
8970 *
8971 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
8972 * See Intel spec. "23.8 Restrictions on VMX operation".
8973 */
8974 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8975 return VINF_SUCCESS;
8976}
8977
8978
8979/**
8980 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
8981 * VM-exit.
8982 */
8983HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8984{
8985 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8986 return VINF_EM_RESET;
8987}
8988
8989
8990/**
8991 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
8992 */
8993HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8994{
8995 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8996 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
8997 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8998 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8999 AssertRCReturn(rc, rc);
9000
9001 pMixedCtx->rip++;
9002 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
9003 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9004 rc = VINF_SUCCESS;
9005 else
9006 rc = VINF_EM_HALT;
9007
9008 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9009 return rc;
9010}
9011
9012
9013/**
9014 * VM-exit handler for instructions that result in a #UD exception delivered to
9015 * the guest.
9016 */
9017HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9018{
9019 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9020 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9021 return VINF_SUCCESS;
9022}
9023
9024
9025/**
9026 * VM-exit handler for expiry of the VMX preemption timer.
9027 */
9028HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9029{
9030 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9031
9032 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9033 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9034
9035 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9036 PVM pVM = pVCpu->CTX_SUFF(pVM);
9037 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9038 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9039 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9040}
9041
9042
9043/**
9044 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9045 */
9046HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9047{
9048 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9049
9050 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9051 /** @todo check if XSETBV is supported by the recompiler. */
9052 return VERR_EM_INTERPRETER;
9053}
9054
9055
9056/**
9057 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9058 */
9059HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9060{
9061 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9062
9063 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9064 /** @todo implement EMInterpretInvpcid() */
9065 return VERR_EM_INTERPRETER;
9066}
9067
9068
9069/**
9070 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9071 * Error VM-exit.
9072 */
9073HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9074{
9075 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9076 AssertRCReturn(rc, rc);
9077
9078 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9079 NOREF(uInvalidReason);
9080
9081#ifdef VBOX_STRICT
9082 uint32_t uIntrState;
9083 HMVMXHCUINTREG uHCReg;
9084 uint64_t u64Val;
9085 uint32_t u32Val;
9086
9087 rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
9088 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9089 rc |= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
9090 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9091 AssertRCReturn(rc, rc);
9092
9093 Log4(("uInvalidReason %u\n", uInvalidReason));
9094 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
9095 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9096 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9097 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9098
9099 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9100 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9101 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9102 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9103 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9104 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9105 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9106 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9107 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9108 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9109 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9110 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9111#endif
9112
9113 PVM pVM = pVCpu->CTX_SUFF(pVM);
9114 HMDumpRegs(pVM, pVCpu, pMixedCtx);
9115
9116 return VERR_VMX_INVALID_GUEST_STATE;
9117}
9118
9119
9120/**
9121 * VM-exit handler for VM-entry failure due to an MSR-load
9122 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9123 */
9124HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9125{
9126 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9127 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9128}
9129
9130
9131/**
9132 * VM-exit handler for VM-entry failure due to a machine-check event
9133 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9134 */
9135HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9136{
9137 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9138 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9139}
9140
9141
9142/**
9143 * VM-exit handler for all undefined reasons. Should never ever happen.. in
9144 * theory.
9145 */
9146HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9147{
9148 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9149 return VERR_VMX_UNDEFINED_EXIT_CODE;
9150}
9151
9152
9153/**
9154 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9155 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9156 * Conditional VM-exit.
9157 */
9158HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9159{
9160 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9161
9162 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9163 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9164 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9165 return VERR_EM_INTERPRETER;
9166 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9167 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9168}
9169
9170
9171/**
9172 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9173 */
9174HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9175{
9176 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9177
9178 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9179 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9180 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9181 return VERR_EM_INTERPRETER;
9182 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9183 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9184}
9185
9186
9187/**
9188 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9189 */
9190HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9191{
9192 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9193
9194 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9195 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9196 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9197 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9198 AssertRCReturn(rc, rc);
9199 Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
9200
9201 PVM pVM = pVCpu->CTX_SUFF(pVM);
9202 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9203 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
9204 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
9205 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
9206
9207 if (RT_LIKELY(rc == VINF_SUCCESS))
9208 {
9209 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9210 Assert(pVmxTransient->cbInstr == 2);
9211 }
9212 return rc;
9213}
9214
9215
9216/**
9217 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
9218 */
9219HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9220{
9221 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9222 PVM pVM = pVCpu->CTX_SUFF(pVM);
9223 int rc = VINF_SUCCESS;
9224
9225 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
9226 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9227 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9228 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9229 AssertRCReturn(rc, rc);
9230 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
9231
9232 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9233 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
9234 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
9235
9236 if (RT_LIKELY(rc == VINF_SUCCESS))
9237 {
9238 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9239
9240 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
9241 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
9242 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
9243 {
9244 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
9245 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
9246 EMInterpretWrmsr() changes it. */
9247 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
9248 }
9249 else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
9250 {
9251 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9252 AssertRCReturn(rc, rc);
9253 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
9254 }
9255 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
9256 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9257
9258 /* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
9259 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9260 {
9261 switch (pMixedCtx->ecx)
9262 {
9263 case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
9264 case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
9265 case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
9266 case MSR_K8_FS_BASE: /* no break */
9267 case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SEGMENT_REGS; break;
9268 case MSR_K8_KERNEL_GS_BASE: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_AUTO_MSRS; break;
9269 }
9270 }
9271#ifdef VBOX_STRICT
9272 else
9273 {
9274 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
9275 switch (pMixedCtx->ecx)
9276 {
9277 case MSR_IA32_SYSENTER_CS:
9278 case MSR_IA32_SYSENTER_EIP:
9279 case MSR_IA32_SYSENTER_ESP:
9280 case MSR_K8_FS_BASE:
9281 case MSR_K8_GS_BASE:
9282 {
9283 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9284 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9285 }
9286
9287 case MSR_K8_LSTAR:
9288 case MSR_K6_STAR:
9289 case MSR_K8_SF_MASK:
9290 case MSR_K8_TSC_AUX:
9291 case MSR_K8_KERNEL_GS_BASE:
9292 {
9293 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
9294 pMixedCtx->ecx));
9295 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9296 }
9297 }
9298 }
9299#endif /* VBOX_STRICT */
9300 }
9301 return rc;
9302}
9303
9304
9305/**
9306 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
9307 */
9308HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9309{
9310 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9311
9312 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
9313 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
9314 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
9315 return VERR_EM_INTERPRETER;
9316 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9317 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9318}
9319
9320
9321/**
9322 * VM-exit handler for when the TPR value is lowered below the specified
9323 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
9324 */
9325HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9326{
9327 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9328 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
9329
9330 /*
9331 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
9332 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
9333 * resume guest execution.
9334 */
9335 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
9336 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
9337 return VINF_SUCCESS;
9338}
9339
9340
9341/**
9342 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
9343 * VM-exit.
9344 *
9345 * @retval VINF_SUCCESS when guest execution can continue.
9346 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
9347 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
9348 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
9349 * recompiler.
9350 */
9351HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9352{
9353 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9354 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
9355 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9356 AssertRCReturn(rc, rc);
9357
9358 const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
9359 const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
9360 PVM pVM = pVCpu->CTX_SUFF(pVM);
9361 switch (uAccessType)
9362 {
9363 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
9364 {
9365#if 0
9366 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
9367 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9368#else
9369 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9370 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9371 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9372#endif
9373 AssertRCReturn(rc, rc);
9374
9375 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9376 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
9377 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
9378 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
9379
9380 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
9381 {
9382 case 0: /* CR0 */
9383 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
9384 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
9385 break;
9386 case 2: /* C2 **/
9387 /* Nothing to do here, CR2 it's not part of the VMCS. */
9388 break;
9389 case 3: /* CR3 */
9390 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
9391 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
9392 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR3;
9393 break;
9394 case 4: /* CR4 */
9395 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
9396 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR4;
9397 break;
9398 case 8: /* CR8 */
9399 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9400 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
9401 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
9402 break;
9403 default:
9404 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
9405 break;
9406 }
9407
9408 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9409 break;
9410 }
9411
9412 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
9413 {
9414 /* EMInterpretCRxRead() requires EFER MSR, CS. */
9415 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9416 AssertRCReturn(rc, rc);
9417 Assert( !pVM->hm.s.fNestedPaging
9418 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
9419 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
9420
9421 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
9422 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
9423 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9424
9425 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9426 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
9427 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
9428 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
9429 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9430 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
9431 break;
9432 }
9433
9434 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
9435 {
9436 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9437 AssertRCReturn(rc, rc);
9438 rc = EMInterpretCLTS(pVM, pVCpu);
9439 AssertRCReturn(rc, rc);
9440 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
9441 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
9442 Log4(("CRX CLTS write rc=%d\n", rc));
9443 break;
9444 }
9445
9446 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
9447 {
9448 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9449 AssertRCReturn(rc, rc);
9450 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
9451 if (RT_LIKELY(rc == VINF_SUCCESS))
9452 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
9453 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
9454 Log4(("CRX LMSW write rc=%d\n", rc));
9455 break;
9456 }
9457
9458 default:
9459 {
9460 AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
9461 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9462 }
9463 }
9464
9465 /* Validate possible error codes. */
9466 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
9467 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
9468 if (RT_SUCCESS(rc))
9469 {
9470 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9471 AssertRCReturn(rc2, rc2);
9472 }
9473
9474 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
9475 return rc;
9476}
9477
9478
9479/**
9480 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
9481 * VM-exit.
9482 */
9483HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9484{
9485 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9486 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
9487
9488 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9489 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9490 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9491 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
9492 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
9493 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
9494 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
9495 AssertRCReturn(rc2, rc2);
9496
9497 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
9498 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
9499 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
9500 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
9501 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
9502 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
9503 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
9504
9505 /* I/O operation lookup arrays. */
9506 static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
9507 static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
9508
9509 VBOXSTRICTRC rcStrict;
9510 const uint32_t cbValue = s_aIOSizes[uIOWidth];
9511 const uint32_t cbInstr = pVmxTransient->cbInstr;
9512 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
9513 PVM pVM = pVCpu->CTX_SUFF(pVM);
9514 if (fIOString)
9515 {
9516 /*
9517 * INS/OUTS - I/O String instruction.
9518 *
9519 * Use instruction-information if available, otherwise fall back on
9520 * interpreting the instruction.
9521 */
9522 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9523#if 0 /* Not quite ready, seem iSegReg assertion trigger once... Do we perhaps need to always read that in longjmp / preempt scenario? */
9524 AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
9525 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
9526 {
9527 rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9528 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9529 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9530 AssertRCReturn(rc2, rc2);
9531 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
9532 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
9533 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
9534 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
9535 if (fIOWrite)
9536 {
9537 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
9538 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9539 //if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9540 // hmR0SavePendingIOPortWriteStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr,
9541 // pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9542 }
9543 else
9544 {
9545 AssertMsgReturn(pVmxTransient->ExitInstrInfo.StrIo.iSegReg == X86_SREG_ES,
9546 ("%#x (%#llx)\n", pVmxTransient->ExitInstrInfo.StrIo.iSegReg, pVmxTransient->ExitInstrInfo.u),
9547 VERR_HMVMX_IPE_4);
9548 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
9549 //if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9550 // hmR0SavePendingIOPortReadStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr);
9551 }
9552 }
9553 else
9554 {
9555 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9556 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9557 AssertRCReturn(rc2, rc2);
9558 rcStrict = IEMExecOne(pVCpu);
9559 }
9560 /** @todo IEM needs to be setting these flags somehow. */
9561 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
9562 fUpdateRipAlready = true;
9563#else
9564 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
9565 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
9566 if (RT_SUCCESS(rcStrict))
9567 {
9568 if (fIOWrite)
9569 {
9570 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9571 (DISCPUMODE)pDis->uAddrMode, cbValue);
9572 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
9573 }
9574 else
9575 {
9576 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9577 (DISCPUMODE)pDis->uAddrMode, cbValue);
9578 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
9579 }
9580 }
9581 else
9582 {
9583 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
9584 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
9585 }
9586#endif
9587 }
9588 else
9589 {
9590 /*
9591 * IN/OUT - I/O instruction.
9592 */
9593 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9594 const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
9595 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
9596 if (fIOWrite)
9597 {
9598 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
9599 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9600 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9601 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
9602 }
9603 else
9604 {
9605 uint32_t u32Result = 0;
9606 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
9607 if (IOM_SUCCESS(rcStrict))
9608 {
9609 /* Save result of I/O IN instr. in AL/AX/EAX. */
9610 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
9611 }
9612 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9613 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9614 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
9615 }
9616 }
9617
9618 if (IOM_SUCCESS(rcStrict))
9619 {
9620 if (!fUpdateRipAlready)
9621 {
9622 pMixedCtx->rip += cbInstr;
9623 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
9624 }
9625
9626 /*
9627 * If any I/O breakpoints are armed, we need to check if one triggered
9628 * and take appropriate action.
9629 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
9630 */
9631 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
9632 AssertRCReturn(rc2, rc2);
9633
9634 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
9635 * execution engines about whether hyper BPs and such are pending. */
9636 uint32_t const uDr7 = pMixedCtx->dr[7];
9637 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
9638 && X86_DR7_ANY_RW_IO(uDr7)
9639 && (pMixedCtx->cr4 & X86_CR4_DE))
9640 || DBGFBpIsHwIoArmed(pVM)))
9641 {
9642 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
9643
9644 /* We're playing with the host CPU state here, make sure we don't preempt. */
9645 HM_DISABLE_PREEMPT_IF_NEEDED();
9646 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /*fDr6*/);
9647
9648 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
9649 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
9650 {
9651 /* Raise #DB. */
9652 if (fIsGuestDbgActive)
9653 ASMSetDR6(pMixedCtx->dr[6]);
9654 if (pMixedCtx->dr[7] != uDr7)
9655 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
9656
9657 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
9658 }
9659 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
9660 else if ( rcStrict2 != VINF_SUCCESS
9661 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
9662 rcStrict = rcStrict2;
9663
9664 HM_RESTORE_PREEMPT_IF_NEEDED();
9665 }
9666 }
9667
9668#ifdef DEBUG
9669 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9670 Assert(!fIOWrite);
9671 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9672 Assert(fIOWrite);
9673 else
9674 {
9675 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
9676 * statuses, that the VMM device and some others may return. See
9677 * IOM_SUCCESS() for guidance. */
9678 AssertMsg( RT_FAILURE(rcStrict)
9679 || rcStrict == VINF_SUCCESS
9680 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
9681 || rcStrict == VINF_EM_DBG_BREAKPOINT
9682 || rcStrict == VINF_EM_RAW_GUEST_TRAP
9683 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9684 }
9685#endif
9686
9687 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
9688 return VBOXSTRICTRC_TODO(rcStrict);
9689}
9690
9691
9692/**
9693 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
9694 * VM-exit.
9695 */
9696HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9697{
9698 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9699
9700 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
9701 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9702 AssertRCReturn(rc, rc);
9703 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
9704 {
9705 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
9706 AssertRCReturn(rc, rc);
9707 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
9708 {
9709 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
9710
9711 /* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
9712 if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
9713 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
9714 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
9715 {
9716 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
9717 bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
9718
9719 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
9720 Assert(!pVCpu->hm.s.Event.fPending);
9721 pVCpu->hm.s.Event.fPending = true;
9722 pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
9723 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
9724 AssertRCReturn(rc, rc);
9725 if (fErrorCodeValid)
9726 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
9727 else
9728 pVCpu->hm.s.Event.u32ErrCode = 0;
9729 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
9730 && uVector == X86_XCPT_PF)
9731 {
9732 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
9733 }
9734
9735 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
9736 }
9737 }
9738 }
9739
9740 /** @todo Emulate task switch someday, currently just going back to ring-3 for
9741 * emulation. */
9742 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
9743 return VERR_EM_INTERPRETER;
9744}
9745
9746
9747/**
9748 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
9749 */
9750HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9751{
9752 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9753 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
9754 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
9755 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9756 AssertRCReturn(rc, rc);
9757 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
9758 return VINF_EM_DBG_STEPPED;
9759}
9760
9761
9762/**
9763 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
9764 */
9765HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9766{
9767 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9768
9769 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9770 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9771 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9772 return VINF_SUCCESS;
9773 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9774 return rc;
9775
9776#if 0
9777 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
9778 * just sync the whole thing. */
9779 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9780#else
9781 /* Aggressive state sync. for now. */
9782 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9783 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9784 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9785#endif
9786 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9787 AssertRCReturn(rc, rc);
9788
9789 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
9790 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
9791 switch (uAccessType)
9792 {
9793 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
9794 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
9795 {
9796 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9797 && VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
9798 {
9799 AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
9800 }
9801
9802 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
9803 GCPhys &= PAGE_BASE_GC_MASK;
9804 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
9805 PVM pVM = pVCpu->CTX_SUFF(pVM);
9806 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
9807 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
9808
9809 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
9810 (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
9811 CPUMCTX2CORE(pMixedCtx), GCPhys);
9812 rc = VBOXSTRICTRC_VAL(rc2);
9813 Log4(("ApicAccess rc=%d\n", rc));
9814 if ( rc == VINF_SUCCESS
9815 || rc == VERR_PAGE_TABLE_NOT_PRESENT
9816 || rc == VERR_PAGE_NOT_PRESENT)
9817 {
9818 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
9819 | HM_CHANGED_VMX_GUEST_APIC_STATE;
9820 rc = VINF_SUCCESS;
9821 }
9822 break;
9823 }
9824
9825 default:
9826 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
9827 rc = VINF_EM_RAW_EMULATE_INSTR;
9828 break;
9829 }
9830
9831 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
9832 return rc;
9833}
9834
9835
9836/**
9837 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
9838 * VM-exit.
9839 */
9840HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9841{
9842 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9843
9844 /* We should -not- get this VM-exit if the guest is debugging. */
9845 if (CPUMIsGuestDebugStateActive(pVCpu))
9846 {
9847 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9848 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9849 }
9850
9851 int rc = VERR_INTERNAL_ERROR_5;
9852 if ( !DBGFIsStepping(pVCpu)
9853 && !pVCpu->hm.s.fSingleInstruction
9854 && !CPUMIsHyperDebugStateActive(pVCpu))
9855 {
9856 /* Don't intercept MOV DRx and #DB any more. */
9857 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
9858 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9859 AssertRCReturn(rc, rc);
9860
9861 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9862 {
9863#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9864 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
9865 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9866 AssertRCReturn(rc, rc);
9867#endif
9868 }
9869
9870 /* We're playing with the host CPU state here, make sure we can't preempt. */
9871 HM_DISABLE_PREEMPT_IF_NEEDED();
9872
9873 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
9874 PVM pVM = pVCpu->CTX_SUFF(pVM);
9875 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
9876 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
9877
9878 HM_RESTORE_PREEMPT_IF_NEEDED();
9879
9880#ifdef VBOX_WITH_STATISTICS
9881 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9882 AssertRCReturn(rc, rc);
9883 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
9884 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
9885 else
9886 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
9887#endif
9888 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
9889 return VINF_SUCCESS;
9890 }
9891
9892 /*
9893 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
9894 * hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
9895 */
9896 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9897 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9898 AssertRCReturn(rc, rc);
9899 Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
9900
9901 PVM pVM = pVCpu->CTX_SUFF(pVM);
9902 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
9903 {
9904 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9905 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
9906 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
9907 if (RT_SUCCESS(rc))
9908 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
9909 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
9910 }
9911 else
9912 {
9913 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9914 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
9915 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
9916 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
9917 }
9918
9919 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
9920 if (RT_SUCCESS(rc))
9921 {
9922 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9923 AssertRCReturn(rc2, rc2);
9924 }
9925 return rc;
9926}
9927
9928
9929/**
9930 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
9931 * Conditional VM-exit.
9932 */
9933HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9934{
9935 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9936 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
9937
9938 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9939 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9940 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9941 return VINF_SUCCESS;
9942 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9943 return rc;
9944
9945 RTGCPHYS GCPhys = 0;
9946 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
9947
9948#if 0
9949 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
9950#else
9951 /* Aggressive state sync. for now. */
9952 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9953 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9954 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9955#endif
9956 AssertRCReturn(rc, rc);
9957
9958 /*
9959 * If we succeed, resume guest execution.
9960 * If we fail in interpreting the instruction because we couldn't get the guest physical address
9961 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
9962 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
9963 * weird case. See @bugref{6043}.
9964 */
9965 PVM pVM = pVCpu->CTX_SUFF(pVM);
9966 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
9967 rc = VBOXSTRICTRC_VAL(rc2);
9968 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
9969 if ( rc == VINF_SUCCESS
9970 || rc == VERR_PAGE_TABLE_NOT_PRESENT
9971 || rc == VERR_PAGE_NOT_PRESENT)
9972 {
9973 /* Successfully handled MMIO operation. */
9974 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
9975 | HM_CHANGED_VMX_GUEST_APIC_STATE;
9976 rc = VINF_SUCCESS;
9977 }
9978 return rc;
9979}
9980
9981
9982/**
9983 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
9984 * VM-exit.
9985 */
9986HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9987{
9988 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9989 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
9990
9991 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9992 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9993 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9994 return VINF_SUCCESS;
9995 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9996 return rc;
9997
9998 RTGCPHYS GCPhys = 0;
9999 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10000 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10001#if 0
10002 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10003#else
10004 /* Aggressive state sync. for now. */
10005 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10006 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10007 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10008#endif
10009 AssertRCReturn(rc, rc);
10010
10011 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10012 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10013
10014 RTGCUINT uErrorCode = 0;
10015 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10016 uErrorCode |= X86_TRAP_PF_ID;
10017 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10018 uErrorCode |= X86_TRAP_PF_RW;
10019 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10020 uErrorCode |= X86_TRAP_PF_P;
10021
10022 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10023
10024 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10025 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10026
10027 /* Handle the pagefault trap for the nested shadow table. */
10028 PVM pVM = pVCpu->CTX_SUFF(pVM);
10029 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10030 TRPMResetTrap(pVCpu);
10031
10032 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10033 if ( rc == VINF_SUCCESS
10034 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10035 || rc == VERR_PAGE_NOT_PRESENT)
10036 {
10037 /* Successfully synced our nested page tables. */
10038 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10039 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS;
10040 return VINF_SUCCESS;
10041 }
10042
10043 Log4(("EPT return to ring-3 rc=%d\n"));
10044 return rc;
10045}
10046
10047/** @} */
10048
10049/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10050/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10051/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10052
10053/** @name VM-exit exception handlers.
10054 * @{
10055 */
10056
10057/**
10058 * VM-exit exception handler for #MF (Math Fault: floating point exception).
10059 */
10060static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10061{
10062 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10063 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10064
10065 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10066 AssertRCReturn(rc, rc);
10067
10068 if (!(pMixedCtx->cr0 & X86_CR0_NE))
10069 {
10070 /* Old-style FPU error reporting needs some extra work. */
10071 /** @todo don't fall back to the recompiler, but do it manually. */
10072 return VERR_EM_INTERPRETER;
10073 }
10074
10075 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10076 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10077 return rc;
10078}
10079
10080
10081/**
10082 * VM-exit exception handler for #BP (Breakpoint exception).
10083 */
10084static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10085{
10086 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10087 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10088
10089 /** @todo Try optimize this by not saving the entire guest state unless
10090 * really needed. */
10091 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10092 AssertRCReturn(rc, rc);
10093
10094 PVM pVM = pVCpu->CTX_SUFF(pVM);
10095 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10096 if (rc == VINF_EM_RAW_GUEST_TRAP)
10097 {
10098 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10099 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10100 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10101 AssertRCReturn(rc, rc);
10102
10103 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10104 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10105 }
10106
10107 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
10108 return rc;
10109}
10110
10111
10112/**
10113 * VM-exit exception handler for #DB (Debug exception).
10114 */
10115static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10116{
10117 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10118 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10119 Log6(("XcptDB\n"));
10120
10121 /*
10122 * Get the DR6-like values from the exit qualification and pass it to DBGF
10123 * for processing.
10124 */
10125 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10126 AssertRCReturn(rc, rc);
10127
10128 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10129 uint64_t uDR6 = X86_DR6_INIT_VAL;
10130 uDR6 |= ( pVmxTransient->uExitQualification
10131 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
10132
10133 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10134 if (rc == VINF_EM_RAW_GUEST_TRAP)
10135 {
10136 /*
10137 * The exception was for the guest. Update DR6, DR7.GD and
10138 * IA32_DEBUGCTL.LBR before forwarding it.
10139 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10140 */
10141 HM_DISABLE_PREEMPT_IF_NEEDED();
10142
10143 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
10144 pMixedCtx->dr[6] |= uDR6;
10145 if (CPUMIsGuestDebugStateActive(pVCpu))
10146 ASMSetDR6(pMixedCtx->dr[6]);
10147
10148 HM_RESTORE_PREEMPT_IF_NEEDED();
10149
10150 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10151 AssertRCReturn(rc, rc);
10152
10153 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10154 pMixedCtx->dr[7] &= ~X86_DR7_GD;
10155
10156 /* Paranoia. */
10157 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
10158 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
10159
10160 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
10161 AssertRCReturn(rc, rc);
10162
10163 /*
10164 * Raise #DB in the guest.
10165 */
10166 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10167 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10168 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10169 AssertRCReturn(rc, rc);
10170 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10171 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10172 return VINF_SUCCESS;
10173 }
10174
10175 /*
10176 * Not a guest trap, must be a hypervisor related debug event then.
10177 * Update DR6 in case someone is interested in it.
10178 */
10179 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
10180 AssertReturn(CPUMIsHyperDebugStateActive(pVCpu), VERR_HM_IPE_5);
10181 CPUMSetHyperDR6(pVCpu, uDR6);
10182
10183 return rc;
10184}
10185
10186
10187/**
10188 * VM-exit exception handler for #NM (Device-not-available exception: floating
10189 * point exception).
10190 */
10191static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10192{
10193 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10194
10195#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10196 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
10197#endif
10198
10199 /* We require CR0 and EFER. EFER is always up-to-date. */
10200 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10201 AssertRCReturn(rc, rc);
10202
10203 /* We're playing with the host CPU state here, have to disable preemption. */
10204 HM_DISABLE_PREEMPT_IF_NEEDED();
10205
10206 /* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
10207 PVM pVM = pVCpu->CTX_SUFF(pVM);
10208 rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
10209 if (rc == VINF_SUCCESS)
10210 {
10211 Assert(CPUMIsGuestFPUStateActive(pVCpu));
10212 HM_RESTORE_PREEMPT_IF_NEEDED();
10213
10214 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
10215 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
10216 return VINF_SUCCESS;
10217 }
10218
10219 HM_RESTORE_PREEMPT_IF_NEEDED();
10220
10221 /* Forward #NM to the guest. */
10222 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
10223 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10224 AssertRCReturn(rc, rc);
10225 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10226 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
10227 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10228 return rc;
10229}
10230
10231
10232/**
10233 * VM-exit exception handler for #GP (General-protection exception).
10234 *
10235 * @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
10236 */
10237static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10238{
10239 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10240 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
10241
10242 int rc = VERR_INTERNAL_ERROR_5;
10243 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10244 {
10245#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10246 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
10247 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10248 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10249 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10250 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10251 AssertRCReturn(rc, rc);
10252 Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
10253 pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
10254 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10255 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10256 return rc;
10257#else
10258 /* We don't intercept #GP. */
10259 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
10260 return VERR_VMX_UNEXPECTED_EXCEPTION;
10261#endif
10262 }
10263
10264 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
10265 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
10266
10267 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
10268 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10269 AssertRCReturn(rc, rc);
10270
10271 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10272 uint32_t cbOp = 0;
10273 PVM pVM = pVCpu->CTX_SUFF(pVM);
10274 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
10275 if (RT_SUCCESS(rc))
10276 {
10277 rc = VINF_SUCCESS;
10278 Assert(cbOp == pDis->cbInstr);
10279 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10280 switch (pDis->pCurInstr->uOpcode)
10281 {
10282 case OP_CLI:
10283 {
10284 pMixedCtx->eflags.Bits.u1IF = 0;
10285 pMixedCtx->rip += pDis->cbInstr;
10286 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS;
10287 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
10288 break;
10289 }
10290
10291 case OP_STI:
10292 {
10293 pMixedCtx->eflags.Bits.u1IF = 1;
10294 pMixedCtx->rip += pDis->cbInstr;
10295 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
10296 Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
10297 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS;
10298 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
10299 break;
10300 }
10301
10302 case OP_HLT:
10303 {
10304 rc = VINF_EM_HALT;
10305 pMixedCtx->rip += pDis->cbInstr;
10306 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
10307 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10308 break;
10309 }
10310
10311 case OP_POPF:
10312 {
10313 Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10314 uint32_t cbParm = 0;
10315 uint32_t uMask = 0;
10316 if (pDis->fPrefix & DISPREFIX_OPSIZE)
10317 {
10318 cbParm = 4;
10319 uMask = 0xffffffff;
10320 }
10321 else
10322 {
10323 cbParm = 2;
10324 uMask = 0xffff;
10325 }
10326
10327 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
10328 RTGCPTR GCPtrStack = 0;
10329 X86EFLAGS Eflags;
10330 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10331 &GCPtrStack);
10332 if (RT_SUCCESS(rc))
10333 {
10334 Assert(sizeof(Eflags.u32) >= cbParm);
10335 Eflags.u32 = 0;
10336 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
10337 }
10338 if (RT_FAILURE(rc))
10339 {
10340 rc = VERR_EM_INTERPRETER;
10341 break;
10342 }
10343 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
10344 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10345 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
10346 /* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
10347 pMixedCtx->eflags.Bits.u1RF = 0;
10348 pMixedCtx->esp += cbParm;
10349 pMixedCtx->esp &= uMask;
10350 pMixedCtx->rip += pDis->cbInstr;
10351 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS;
10352 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
10353 break;
10354 }
10355
10356 case OP_PUSHF:
10357 {
10358 uint32_t cbParm = 0;
10359 uint32_t uMask = 0;
10360 if (pDis->fPrefix & DISPREFIX_OPSIZE)
10361 {
10362 cbParm = 4;
10363 uMask = 0xffffffff;
10364 }
10365 else
10366 {
10367 cbParm = 2;
10368 uMask = 0xffff;
10369 }
10370
10371 /* Get the stack pointer & push the contents of eflags onto the stack. */
10372 RTGCPTR GCPtrStack = 0;
10373 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
10374 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
10375 if (RT_FAILURE(rc))
10376 {
10377 rc = VERR_EM_INTERPRETER;
10378 break;
10379 }
10380 X86EFLAGS Eflags = pMixedCtx->eflags;
10381 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
10382 Eflags.Bits.u1RF = 0;
10383 Eflags.Bits.u1VM = 0;
10384
10385 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
10386 if (RT_FAILURE(rc))
10387 {
10388 rc = VERR_EM_INTERPRETER;
10389 break;
10390 }
10391 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
10392 pMixedCtx->esp -= cbParm;
10393 pMixedCtx->esp &= uMask;
10394 pMixedCtx->rip += pDis->cbInstr;
10395 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP;
10396 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
10397 break;
10398 }
10399
10400 case OP_IRET:
10401 {
10402 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
10403 * instruction reference. */
10404 RTGCPTR GCPtrStack = 0;
10405 uint32_t uMask = 0xffff;
10406 uint16_t aIretFrame[3];
10407 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
10408 {
10409 rc = VERR_EM_INTERPRETER;
10410 break;
10411 }
10412 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10413 &GCPtrStack);
10414 if (RT_SUCCESS(rc))
10415 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
10416 if (RT_FAILURE(rc))
10417 {
10418 rc = VERR_EM_INTERPRETER;
10419 break;
10420 }
10421 pMixedCtx->eip = 0;
10422 pMixedCtx->ip = aIretFrame[0];
10423 pMixedCtx->cs.Sel = aIretFrame[1];
10424 pMixedCtx->cs.ValidSel = aIretFrame[1];
10425 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
10426 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10427 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
10428 pMixedCtx->sp += sizeof(aIretFrame);
10429 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_SEGMENT_REGS | HM_CHANGED_GUEST_RSP
10430 | HM_CHANGED_GUEST_RFLAGS;
10431 Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
10432 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
10433 break;
10434 }
10435
10436 case OP_INT:
10437 {
10438 uint16_t uVector = pDis->Param1.uValue & 0xff;
10439 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
10440 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10441 break;
10442 }
10443
10444 case OP_INTO:
10445 {
10446 if (pMixedCtx->eflags.Bits.u1OF)
10447 {
10448 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
10449 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10450 }
10451 break;
10452 }
10453
10454 default:
10455 {
10456 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
10457 EMCODETYPE_SUPERVISOR);
10458 rc = VBOXSTRICTRC_VAL(rc2);
10459 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_ALL_GUEST;
10460 Log4(("#GP rc=%Rrc\n", rc));
10461 break;
10462 }
10463 }
10464 }
10465 else
10466 rc = VERR_EM_INTERPRETER;
10467
10468 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
10469 ("#GP Unexpected rc=%Rrc\n", rc));
10470 return rc;
10471}
10472
10473
10474/**
10475 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
10476 * the exception reported in the VMX transient structure back into the VM.
10477 *
10478 * @remarks Requires uExitIntrInfo in the VMX transient structure to be
10479 * up-to-date.
10480 */
10481static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10482{
10483 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10484
10485 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
10486 hmR0VmxCheckExitDueToEventDelivery(). */
10487 int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10488 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10489 AssertRCReturn(rc, rc);
10490 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
10491
10492 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10493 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10494 return VINF_SUCCESS;
10495}
10496
10497
10498/**
10499 * VM-exit exception handler for #PF (Page-fault exception).
10500 */
10501static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10502{
10503 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10504 PVM pVM = pVCpu->CTX_SUFF(pVM);
10505 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10506 rc |= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10507 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10508 AssertRCReturn(rc, rc);
10509
10510#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
10511 if (pVM->hm.s.fNestedPaging)
10512 {
10513 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
10514 if (RT_LIKELY(!pVmxTransient->fVectoringPF))
10515 {
10516 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10517 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10518 0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
10519 }
10520 else
10521 {
10522 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10523 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10524 Log4(("Pending #DF due to vectoring #PF. NP\n"));
10525 }
10526 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10527 return rc;
10528 }
10529#else
10530 Assert(!pVM->hm.s.fNestedPaging);
10531#endif
10532
10533 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10534 AssertRCReturn(rc, rc);
10535
10536 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
10537 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
10538
10539 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
10540 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
10541 (RTGCPTR)pVmxTransient->uExitQualification);
10542
10543 Log4(("#PF: rc=%Rrc\n", rc));
10544 if (rc == VINF_SUCCESS)
10545 {
10546 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
10547 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
10548 * memory? We don't update the whole state here... */
10549 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
10550 | HM_CHANGED_VMX_GUEST_APIC_STATE;
10551 TRPMResetTrap(pVCpu);
10552 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
10553 return rc;
10554 }
10555 else if (rc == VINF_EM_RAW_GUEST_TRAP)
10556 {
10557 if (!pVmxTransient->fVectoringPF)
10558 {
10559 /* It's a guest page fault and needs to be reflected to the guest. */
10560 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
10561 TRPMResetTrap(pVCpu);
10562 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
10563 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10564 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10565 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
10566 }
10567 else
10568 {
10569 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10570 TRPMResetTrap(pVCpu);
10571 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
10572 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10573 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
10574 }
10575
10576 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10577 return VINF_SUCCESS;
10578 }
10579
10580 TRPMResetTrap(pVCpu);
10581 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
10582 return rc;
10583}
10584
10585/** @} */
10586
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette