VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 50698

Last change on this file since 50698 was 50698, checked in by vboxsync, 11 years ago

VMM/HMVMXR0: More info. on assert failures.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 477.6 KB
Line 
1/* $Id: HMVMXR0.cpp 50698 2014-03-05 11:21:20Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2013 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/x86.h>
23#include <iprt/asm-amd64-x86.h>
24#include <iprt/thread.h>
25#include <iprt/string.h>
26
27#include "HMInternal.h"
28#include <VBox/vmm/vm.h>
29#include "HMVMXR0.h"
30#include <VBox/vmm/pdmapi.h>
31#include <VBox/vmm/dbgf.h>
32#include <VBox/vmm/iem.h>
33#include <VBox/vmm/iom.h>
34#include <VBox/vmm/selm.h>
35#include <VBox/vmm/tm.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#ifdef DEBUG_ramshankar
40# define HMVMX_SAVE_FULL_GUEST_STATE
41# define HMVMX_SYNC_FULL_GUEST_STATE
42# define HMVMX_ALWAYS_CHECK_GUEST_STATE
43# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
44# define HMVMX_ALWAYS_TRAP_PF
45# define HMVMX_ALWAYS_SWAP_FPU_STATE
46# define HMVMX_ALWAYS_FLUSH_TLB
47#endif
48
49
50/*******************************************************************************
51* Defined Constants And Macros *
52*******************************************************************************/
53#if defined(RT_ARCH_AMD64)
54# define HMVMX_IS_64BIT_HOST_MODE() (true)
55typedef RTHCUINTREG HMVMXHCUINTREG;
56#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
57extern "C" uint32_t g_fVMXIs64bitHost;
58# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
59typedef uint64_t HMVMXHCUINTREG;
60#else
61# define HMVMX_IS_64BIT_HOST_MODE() (false)
62typedef RTHCUINTREG HMVMXHCUINTREG;
63#endif
64
65/** Use the function table. */
66#define HMVMX_USE_FUNCTION_TABLE
67
68/** Determine which tagged-TLB flush handler to use. */
69#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
70#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
71#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
72#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
73
74/** @name Updated-guest-state flags.
75 * @{ */
76#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
77#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
78#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
79#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
80#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
81#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
82#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
83#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
84#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
85#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
86#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
87#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
88#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
89#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
90#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
91#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
92#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
93#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
94#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(18)
95#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
96 | HMVMX_UPDATED_GUEST_RSP \
97 | HMVMX_UPDATED_GUEST_RFLAGS \
98 | HMVMX_UPDATED_GUEST_CR0 \
99 | HMVMX_UPDATED_GUEST_CR3 \
100 | HMVMX_UPDATED_GUEST_CR4 \
101 | HMVMX_UPDATED_GUEST_GDTR \
102 | HMVMX_UPDATED_GUEST_IDTR \
103 | HMVMX_UPDATED_GUEST_LDTR \
104 | HMVMX_UPDATED_GUEST_TR \
105 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
106 | HMVMX_UPDATED_GUEST_DEBUG \
107 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
112 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
113 | HMVMX_UPDATED_GUEST_APIC_STATE)
114/** @} */
115
116/** @name
117 * Flags to skip redundant reads of some common VMCS fields that are not part of
118 * the guest-CPU state but are in the transient structure.
119 */
120#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
121#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
123#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
124#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
125#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
126#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
127/** @} */
128
129/** @name
130 * States of the VMCS.
131 *
132 * This does not reflect all possible VMCS states but currently only those
133 * needed for maintaining the VMCS consistently even when thread-context hooks
134 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
135 */
136#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
137#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
138#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
139/** @} */
140
141/**
142 * Exception bitmap mask for real-mode guests (real-on-v86).
143 *
144 * We need to intercept all exceptions manually (except #PF). #NM is also
145 * handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
146 * even in real-mode if we have Nested Paging support.
147 */
148#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
149 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
150 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
151 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
152 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
153 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
154 | RT_BIT(X86_XCPT_XF))
155
156/**
157 * Exception bitmap mask for all contributory exceptions.
158 *
159 * Page fault is deliberately excluded here as it's conditional as to whether
160 * it's contributory or benign. Page faults are handled separately.
161 */
162#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
163 | RT_BIT(X86_XCPT_DE))
164
165/** Maximum VM-instruction error number. */
166#define HMVMX_INSTR_ERROR_MAX 28
167
168/** Profiling macro. */
169#ifdef HM_PROFILE_EXIT_DISPATCH
170# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
171# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
172#else
173# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
174# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
175#endif
176
177/** Assert that preemption is disabled or covered by thread-context hooks. */
178#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
179 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
180
181/** Assert that we haven't migrated CPUs when thread-context hooks are not
182 * used. */
183#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
184 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
185 ("Illegal migration! Entered on CPU %u Current %u\n", \
186 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
187
188/** Helper macro for VM-exit handlers called unexpectedly. */
189#define HMVMX_RETURN_UNEXPECTED_EXIT() \
190 do { \
191 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
192 return VERR_VMX_UNEXPECTED_EXIT; \
193 } while (0)
194
195
196/*******************************************************************************
197* Structures and Typedefs *
198*******************************************************************************/
199/**
200 * VMX transient state.
201 *
202 * A state structure for holding miscellaneous information across
203 * VMX non-root operation and restored after the transition.
204 */
205typedef struct VMXTRANSIENT
206{
207 /** The host's rflags/eflags. */
208 RTCCUINTREG uEflags;
209#if HC_ARCH_BITS == 32
210 uint32_t u32Alignment0;
211#endif
212 /** The guest's TPR value used for TPR shadowing. */
213 uint8_t u8GuestTpr;
214 /** Alignment. */
215 uint8_t abAlignment0[7];
216
217 /** The basic VM-exit reason. */
218 uint16_t uExitReason;
219 /** Alignment. */
220 uint16_t u16Alignment0;
221 /** The VM-exit interruption error code. */
222 uint32_t uExitIntErrorCode;
223 /** The VM-exit exit qualification. */
224 uint64_t uExitQualification;
225
226 /** The VM-exit interruption-information field. */
227 uint32_t uExitIntInfo;
228 /** The VM-exit instruction-length field. */
229 uint32_t cbInstr;
230 /** The VM-exit instruction-information field. */
231 union
232 {
233 /** Plain unsigned int representation. */
234 uint32_t u;
235 /** INS and OUTS information. */
236 struct
237 {
238 uint32_t u6Reserved0 : 7;
239 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
240 uint32_t u3AddrSize : 3;
241 uint32_t u5Reserved1 : 5;
242 /** The segment register (X86_SREG_XXX). */
243 uint32_t iSegReg : 3;
244 uint32_t uReserved2 : 14;
245 } StrIo;
246 } ExitInstrInfo;
247 /** Whether the VM-entry failed or not. */
248 bool fVMEntryFailed;
249 /** Alignment. */
250 uint8_t abAlignment1[3];
251
252 /** The VM-entry interruption-information field. */
253 uint32_t uEntryIntInfo;
254 /** The VM-entry exception error code field. */
255 uint32_t uEntryXcptErrorCode;
256 /** The VM-entry instruction length field. */
257 uint32_t cbEntryInstr;
258
259 /** IDT-vectoring information field. */
260 uint32_t uIdtVectoringInfo;
261 /** IDT-vectoring error code. */
262 uint32_t uIdtVectoringErrorCode;
263
264 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
265 uint32_t fVmcsFieldsRead;
266
267 /** Whether the guest FPU was active at the time of VM-exit. */
268 bool fWasGuestFPUStateActive;
269 /** Whether the guest debug state was active at the time of VM-exit. */
270 bool fWasGuestDebugStateActive;
271 /** Whether the hyper debug state was active at the time of VM-exit. */
272 bool fWasHyperDebugStateActive;
273 /** Whether TSC-offsetting should be setup before VM-entry. */
274 bool fUpdateTscOffsettingAndPreemptTimer;
275 /** Whether the VM-exit was caused by a page-fault during delivery of a
276 * contributory exception or a page-fault. */
277 bool fVectoringPF;
278} VMXTRANSIENT;
279AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
280AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
281AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
282AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
283AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
284/** Pointer to VMX transient state. */
285typedef VMXTRANSIENT *PVMXTRANSIENT;
286
287
288/**
289 * MSR-bitmap read permissions.
290 */
291typedef enum VMXMSREXITREAD
292{
293 /** Reading this MSR causes a VM-exit. */
294 VMXMSREXIT_INTERCEPT_READ = 0xb,
295 /** Reading this MSR does not cause a VM-exit. */
296 VMXMSREXIT_PASSTHRU_READ
297} VMXMSREXITREAD;
298/** Pointer to MSR-bitmap read permissions. */
299typedef VMXMSREXITREAD* PVMXMSREXITREAD;
300
301/**
302 * MSR-bitmap write permissions.
303 */
304typedef enum VMXMSREXITWRITE
305{
306 /** Writing to this MSR causes a VM-exit. */
307 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
308 /** Writing to this MSR does not cause a VM-exit. */
309 VMXMSREXIT_PASSTHRU_WRITE
310} VMXMSREXITWRITE;
311/** Pointer to MSR-bitmap write permissions. */
312typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
313
314
315/**
316 * VMX VM-exit handler.
317 *
318 * @returns VBox status code.
319 * @param pVCpu Pointer to the VMCPU.
320 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
321 * out-of-sync. Make sure to update the required
322 * fields before using them.
323 * @param pVmxTransient Pointer to the VMX-transient structure.
324 */
325#ifndef HMVMX_USE_FUNCTION_TABLE
326typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
327#else
328typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
329/** Pointer to VM-exit handler. */
330typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
331#endif
332
333
334/*******************************************************************************
335* Internal Functions *
336*******************************************************************************/
337static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
338static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
339static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
340 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
341#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
342static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
343#endif
344#ifndef HMVMX_USE_FUNCTION_TABLE
345DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
346# define HMVMX_EXIT_DECL static int
347#else
348# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
349#endif
350
351/** @name VM-exit handlers.
352 * @{
353 */
354static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
355static FNVMXEXITHANDLER hmR0VmxExitExtInt;
356static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
357static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
358static FNVMXEXITHANDLER hmR0VmxExitSipi;
359static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
360static FNVMXEXITHANDLER hmR0VmxExitSmi;
361static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
362static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
363static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
364static FNVMXEXITHANDLER hmR0VmxExitCpuid;
365static FNVMXEXITHANDLER hmR0VmxExitGetsec;
366static FNVMXEXITHANDLER hmR0VmxExitHlt;
367static FNVMXEXITHANDLER hmR0VmxExitInvd;
368static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
369static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
370static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
371static FNVMXEXITHANDLER hmR0VmxExitRsm;
372static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
373static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
374static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
375static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
376static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
377static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
378static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
379static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
380static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
381static FNVMXEXITHANDLER hmR0VmxExitMwait;
382static FNVMXEXITHANDLER hmR0VmxExitMtf;
383static FNVMXEXITHANDLER hmR0VmxExitMonitor;
384static FNVMXEXITHANDLER hmR0VmxExitPause;
385static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
386static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
387static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
388static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
389static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
390static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
391static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
392static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
393static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
394static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
395static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
396static FNVMXEXITHANDLER hmR0VmxExitRdrand;
397static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
398/** @} */
399
400static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
401static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
402static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
403static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
404static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
405static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
406#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
407static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
408#endif
409static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
410
411/*******************************************************************************
412* Global Variables *
413*******************************************************************************/
414#ifdef HMVMX_USE_FUNCTION_TABLE
415
416/**
417 * VMX_EXIT dispatch table.
418 */
419static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
420{
421 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
422 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
423 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
424 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
425 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
426 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
427 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
428 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
429 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
430 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
431 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
432 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
433 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
434 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
435 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
436 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
437 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
438 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
439 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
440 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
441 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
442 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
443 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
444 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
445 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
446 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
447 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
448 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
449 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
450 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
451 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
452 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
453 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
454 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
455 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
456 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
457 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
458 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
459 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
460 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
461 /* 40 UNDEFINED */ hmR0VmxExitPause,
462 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
463 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
464 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
465 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
466 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
467 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
468 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
469 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
470 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
471 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
472 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
473 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
474 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
475 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
476 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
477 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
478 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
479 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
480 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
481};
482#endif /* HMVMX_USE_FUNCTION_TABLE */
483
484#ifdef VBOX_STRICT
485static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
486{
487 /* 0 */ "(Not Used)",
488 /* 1 */ "VMCALL executed in VMX root operation.",
489 /* 2 */ "VMCLEAR with invalid physical address.",
490 /* 3 */ "VMCLEAR with VMXON pointer.",
491 /* 4 */ "VMLAUNCH with non-clear VMCS.",
492 /* 5 */ "VMRESUME with non-launched VMCS.",
493 /* 6 */ "VMRESUME after VMXOFF",
494 /* 7 */ "VM entry with invalid control fields.",
495 /* 8 */ "VM entry with invalid host state fields.",
496 /* 9 */ "VMPTRLD with invalid physical address.",
497 /* 10 */ "VMPTRLD with VMXON pointer.",
498 /* 11 */ "VMPTRLD with incorrect revision identifier.",
499 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
500 /* 13 */ "VMWRITE to read-only VMCS component.",
501 /* 14 */ "(Not Used)",
502 /* 15 */ "VMXON executed in VMX root operation.",
503 /* 16 */ "VM entry with invalid executive-VMCS pointer.",
504 /* 17 */ "VM entry with non-launched executing VMCS.",
505 /* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
506 /* 19 */ "VMCALL with non-clear VMCS.",
507 /* 20 */ "VMCALL with invalid VM-exit control fields.",
508 /* 21 */ "(Not Used)",
509 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
510 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
511 /* 24 */ "VMCALL with invalid SMM-monitor features.",
512 /* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
513 /* 26 */ "VM entry with events blocked by MOV SS.",
514 /* 27 */ "(Not Used)",
515 /* 28 */ "Invalid operand to INVEPT/INVVPID."
516};
517#endif /* VBOX_STRICT */
518
519
520
521/**
522 * Updates the VM's last error record. If there was a VMX instruction error,
523 * reads the error data from the VMCS and updates VCPU's last error record as
524 * well.
525 *
526 * @param pVM Pointer to the VM.
527 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
528 * VERR_VMX_UNABLE_TO_START_VM or
529 * VERR_VMX_INVALID_VMCS_FIELD).
530 * @param rc The error code.
531 */
532static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
533{
534 AssertPtr(pVM);
535 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
536 || rc == VERR_VMX_UNABLE_TO_START_VM)
537 {
538 AssertPtrReturnVoid(pVCpu);
539 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
540 }
541 pVM->hm.s.lLastError = rc;
542}
543
544
545/**
546 * Reads the VM-entry interruption-information field from the VMCS into the VMX
547 * transient structure.
548 *
549 * @returns VBox status code.
550 * @param pVmxTransient Pointer to the VMX transient structure.
551 *
552 * @remarks No-long-jump zone!!!
553 */
554DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
555{
556 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
557 AssertRCReturn(rc, rc);
558 return VINF_SUCCESS;
559}
560
561
562/**
563 * Reads the VM-entry exception error code field from the VMCS into
564 * the VMX transient structure.
565 *
566 * @returns VBox status code.
567 * @param pVmxTransient Pointer to the VMX transient structure.
568 *
569 * @remarks No-long-jump zone!!!
570 */
571DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
572{
573 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
574 AssertRCReturn(rc, rc);
575 return VINF_SUCCESS;
576}
577
578
579/**
580 * Reads the VM-entry exception error code field from the VMCS into
581 * the VMX transient structure.
582 *
583 * @returns VBox status code.
584 * @param pVmxTransient Pointer to the VMX transient structure.
585 *
586 * @remarks No-long-jump zone!!!
587 */
588DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
589{
590 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
591 AssertRCReturn(rc, rc);
592 return VINF_SUCCESS;
593}
594
595
596/**
597 * Reads the VM-exit interruption-information field from the VMCS into the VMX
598 * transient structure.
599 *
600 * @returns VBox status code.
601 * @param pVmxTransient Pointer to the VMX transient structure.
602 */
603DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
604{
605 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
606 {
607 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
608 AssertRCReturn(rc, rc);
609 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
610 }
611 return VINF_SUCCESS;
612}
613
614
615/**
616 * Reads the VM-exit interruption error code from the VMCS into the VMX
617 * transient structure.
618 *
619 * @returns VBox status code.
620 * @param pVmxTransient Pointer to the VMX transient structure.
621 */
622DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
623{
624 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
625 {
626 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
627 AssertRCReturn(rc, rc);
628 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
629 }
630 return VINF_SUCCESS;
631}
632
633
634/**
635 * Reads the VM-exit instruction length field from the VMCS into the VMX
636 * transient structure.
637 *
638 * @returns VBox status code.
639 * @param pVCpu Pointer to the VMCPU.
640 * @param pVmxTransient Pointer to the VMX transient structure.
641 */
642DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
643{
644 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
645 {
646 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
647 AssertRCReturn(rc, rc);
648 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
649 }
650 return VINF_SUCCESS;
651}
652
653
654/**
655 * Reads the VM-exit instruction-information field from the VMCS into
656 * the VMX transient structure.
657 *
658 * @returns VBox status code.
659 * @param pVmxTransient Pointer to the VMX transient structure.
660 */
661DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
662{
663 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
664 {
665 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
666 AssertRCReturn(rc, rc);
667 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
668 }
669 return VINF_SUCCESS;
670}
671
672
673/**
674 * Reads the exit qualification from the VMCS into the VMX transient structure.
675 *
676 * @returns VBox status code.
677 * @param pVCpu Pointer to the VMCPU (required for the VMCS cache
678 * case).
679 * @param pVmxTransient Pointer to the VMX transient structure.
680 */
681DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
682{
683 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
684 {
685 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
686 AssertRCReturn(rc, rc);
687 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
688 }
689 return VINF_SUCCESS;
690}
691
692
693/**
694 * Reads the IDT-vectoring information field from the VMCS into the VMX
695 * transient structure.
696 *
697 * @returns VBox status code.
698 * @param pVmxTransient Pointer to the VMX transient structure.
699 *
700 * @remarks No-long-jump zone!!!
701 */
702DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
703{
704 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
705 {
706 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
707 AssertRCReturn(rc, rc);
708 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
709 }
710 return VINF_SUCCESS;
711}
712
713
714/**
715 * Reads the IDT-vectoring error code from the VMCS into the VMX
716 * transient structure.
717 *
718 * @returns VBox status code.
719 * @param pVmxTransient Pointer to the VMX transient structure.
720 */
721DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
722{
723 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
724 {
725 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
726 AssertRCReturn(rc, rc);
727 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
728 }
729 return VINF_SUCCESS;
730}
731
732
733/**
734 * Enters VMX root mode operation on the current CPU.
735 *
736 * @returns VBox status code.
737 * @param pVM Pointer to the VM (optional, can be NULL, after
738 * a resume).
739 * @param HCPhysCpuPage Physical address of the VMXON region.
740 * @param pvCpuPage Pointer to the VMXON region.
741 */
742static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
743{
744 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
745 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
746 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
747
748 if (pVM)
749 {
750 /* Write the VMCS revision dword to the VMXON region. */
751 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
752 }
753
754 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
755 RTCCUINTREG uEflags = ASMIntDisableFlags();
756
757 /* Enable the VMX bit in CR4 if necessary. */
758 RTCCUINTREG uCr4 = ASMGetCR4();
759 if (!(uCr4 & X86_CR4_VMXE))
760 ASMSetCR4(uCr4 | X86_CR4_VMXE);
761
762 /* Enter VMX root mode. */
763 int rc = VMXEnable(HCPhysCpuPage);
764 if (RT_FAILURE(rc))
765 ASMSetCR4(uCr4);
766
767 /* Restore interrupts. */
768 ASMSetFlags(uEflags);
769 return rc;
770}
771
772
773/**
774 * Exits VMX root mode operation on the current CPU.
775 *
776 * @returns VBox status code.
777 */
778static int hmR0VmxLeaveRootMode(void)
779{
780 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
781
782 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
783 RTCCUINTREG uEflags = ASMIntDisableFlags();
784
785 /* If we're for some reason not in VMX root mode, then don't leave it. */
786 RTCCUINTREG uHostCR4 = ASMGetCR4();
787
788 int rc;
789 if (uHostCR4 & X86_CR4_VMXE)
790 {
791 /* Exit VMX root mode and clear the VMX bit in CR4. */
792 VMXDisable();
793 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
794 rc = VINF_SUCCESS;
795 }
796 else
797 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
798
799 /* Restore interrupts. */
800 ASMSetFlags(uEflags);
801 return rc;
802}
803
804
805/**
806 * Allocates and maps one physically contiguous page. The allocated page is
807 * zero'd out. (Used by various VT-x structures).
808 *
809 * @returns IPRT status code.
810 * @param pMemObj Pointer to the ring-0 memory object.
811 * @param ppVirt Where to store the virtual address of the
812 * allocation.
813 * @param pPhys Where to store the physical address of the
814 * allocation.
815 */
816DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
817{
818 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
819 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
820 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
821
822 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
823 if (RT_FAILURE(rc))
824 return rc;
825 *ppVirt = RTR0MemObjAddress(*pMemObj);
826 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
827 ASMMemZero32(*ppVirt, PAGE_SIZE);
828 return VINF_SUCCESS;
829}
830
831
832/**
833 * Frees and unmaps an allocated physical page.
834 *
835 * @param pMemObj Pointer to the ring-0 memory object.
836 * @param ppVirt Where to re-initialize the virtual address of
837 * allocation as 0.
838 * @param pHCPhys Where to re-initialize the physical address of the
839 * allocation as 0.
840 */
841DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
842{
843 AssertPtr(pMemObj);
844 AssertPtr(ppVirt);
845 AssertPtr(pHCPhys);
846 if (*pMemObj != NIL_RTR0MEMOBJ)
847 {
848 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
849 AssertRC(rc);
850 *pMemObj = NIL_RTR0MEMOBJ;
851 *ppVirt = 0;
852 *pHCPhys = 0;
853 }
854}
855
856
857/**
858 * Worker function to free VT-x related structures.
859 *
860 * @returns IPRT status code.
861 * @param pVM Pointer to the VM.
862 */
863static void hmR0VmxStructsFree(PVM pVM)
864{
865 for (VMCPUID i = 0; i < pVM->cCpus; i++)
866 {
867 PVMCPU pVCpu = &pVM->aCpus[i];
868 AssertPtr(pVCpu);
869
870 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
871 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
872
873 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
874 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
875
876 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
877 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
878 }
879
880 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
881#ifdef VBOX_WITH_CRASHDUMP_MAGIC
882 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
883#endif
884}
885
886
887/**
888 * Worker function to allocate VT-x related VM structures.
889 *
890 * @returns IPRT status code.
891 * @param pVM Pointer to the VM.
892 */
893static int hmR0VmxStructsAlloc(PVM pVM)
894{
895 /*
896 * Initialize members up-front so we can cleanup properly on allocation failure.
897 */
898#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
899 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
900 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
901 pVM->hm.s.vmx.HCPhys##a_Name = 0;
902
903#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
904 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
905 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
906 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
907
908#ifdef VBOX_WITH_CRASHDUMP_MAGIC
909 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
910#endif
911 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
912
913 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
914 for (VMCPUID i = 0; i < pVM->cCpus; i++)
915 {
916 PVMCPU pVCpu = &pVM->aCpus[i];
917 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
918 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
919 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
920 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
921 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
922 }
923#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
924#undef VMXLOCAL_INIT_VM_MEMOBJ
925
926 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
927 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
928 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
929 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
930
931 /*
932 * Allocate all the VT-x structures.
933 */
934 int rc = VINF_SUCCESS;
935#ifdef VBOX_WITH_CRASHDUMP_MAGIC
936 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
937 if (RT_FAILURE(rc))
938 goto cleanup;
939 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
940 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
941#endif
942
943 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
944 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
945 {
946 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
947 &pVM->hm.s.vmx.HCPhysApicAccess);
948 if (RT_FAILURE(rc))
949 goto cleanup;
950 }
951
952 /*
953 * Initialize per-VCPU VT-x structures.
954 */
955 for (VMCPUID i = 0; i < pVM->cCpus; i++)
956 {
957 PVMCPU pVCpu = &pVM->aCpus[i];
958 AssertPtr(pVCpu);
959
960 /* Allocate the VM control structure (VMCS). */
961 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
962 if (RT_FAILURE(rc))
963 goto cleanup;
964
965 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
966 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
967 {
968 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
969 &pVCpu->hm.s.vmx.HCPhysVirtApic);
970 if (RT_FAILURE(rc))
971 goto cleanup;
972 }
973
974 /* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
975 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
976 {
977 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
978 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
979 if (RT_FAILURE(rc))
980 goto cleanup;
981 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
982 }
983
984 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
985 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
986 if (RT_FAILURE(rc))
987 goto cleanup;
988
989 /* Allocate the VM-exit MSR-load page for the host MSRs. */
990 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
991 if (RT_FAILURE(rc))
992 goto cleanup;
993 }
994
995 return VINF_SUCCESS;
996
997cleanup:
998 hmR0VmxStructsFree(pVM);
999 return rc;
1000}
1001
1002
1003/**
1004 * Does global VT-x initialization (called during module initialization).
1005 *
1006 * @returns VBox status code.
1007 */
1008VMMR0DECL(int) VMXR0GlobalInit(void)
1009{
1010#ifdef HMVMX_USE_FUNCTION_TABLE
1011 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1012# ifdef VBOX_STRICT
1013 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1014 Assert(g_apfnVMExitHandlers[i]);
1015# endif
1016#endif
1017 return VINF_SUCCESS;
1018}
1019
1020
1021/**
1022 * Does global VT-x termination (called during module termination).
1023 */
1024VMMR0DECL(void) VMXR0GlobalTerm()
1025{
1026 /* Nothing to do currently. */
1027}
1028
1029
1030/**
1031 * Sets up and activates VT-x on the current CPU.
1032 *
1033 * @returns VBox status code.
1034 * @param pCpu Pointer to the global CPU info struct.
1035 * @param pVM Pointer to the VM (can be NULL after a host resume
1036 * operation).
1037 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1038 * fEnabledByHost is true).
1039 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1040 * @a fEnabledByHost is true).
1041 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1042 * enable VT-x on the host.
1043 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1044 */
1045VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1046 void *pvMsrs)
1047{
1048 AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1049 AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1050 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1051
1052 /* Enable VT-x if it's not already enabled by the host. */
1053 if (!fEnabledByHost)
1054 {
1055 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1056 if (RT_FAILURE(rc))
1057 return rc;
1058 }
1059
1060 /*
1061 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1062 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1063 */
1064 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1065 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1066 {
1067 hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1068 pCpu->fFlushAsidBeforeUse = false;
1069 }
1070 else
1071 pCpu->fFlushAsidBeforeUse = true;
1072
1073 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1074 ++pCpu->cTlbFlushes;
1075
1076 return VINF_SUCCESS;
1077}
1078
1079
1080/**
1081 * Deactivates VT-x on the current CPU.
1082 *
1083 * @returns VBox status code.
1084 * @param pCpu Pointer to the global CPU info struct.
1085 * @param pvCpuPage Pointer to the VMXON region.
1086 * @param HCPhysCpuPage Physical address of the VMXON region.
1087 *
1088 * @remarks This function should never be called when SUPR0EnableVTx() or
1089 * similar was used to enable VT-x on the host.
1090 */
1091VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1092{
1093 NOREF(pCpu);
1094 NOREF(pvCpuPage);
1095 NOREF(HCPhysCpuPage);
1096
1097 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1098 return hmR0VmxLeaveRootMode();
1099}
1100
1101
1102/**
1103 * Sets the permission bits for the specified MSR in the MSR bitmap.
1104 *
1105 * @param pVCpu Pointer to the VMCPU.
1106 * @param uMSR The MSR value.
1107 * @param enmRead Whether reading this MSR causes a VM-exit.
1108 * @param enmWrite Whether writing this MSR causes a VM-exit.
1109 */
1110static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1111{
1112 int32_t iBit;
1113 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1114
1115 /*
1116 * Layout:
1117 * 0x000 - 0x3ff - Low MSR read bits
1118 * 0x400 - 0x7ff - High MSR read bits
1119 * 0x800 - 0xbff - Low MSR write bits
1120 * 0xc00 - 0xfff - High MSR write bits
1121 */
1122 if (uMsr <= 0x00001FFF)
1123 iBit = uMsr;
1124 else if ( uMsr >= 0xC0000000
1125 && uMsr <= 0xC0001FFF)
1126 {
1127 iBit = (uMsr - 0xC0000000);
1128 pbMsrBitmap += 0x400;
1129 }
1130 else
1131 {
1132 AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1133 return;
1134 }
1135
1136 Assert(iBit <= 0x1fff);
1137 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1138 ASMBitSet(pbMsrBitmap, iBit);
1139 else
1140 ASMBitClear(pbMsrBitmap, iBit);
1141
1142 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1143 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1144 else
1145 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1146}
1147
1148
1149#ifdef VBOX_STRICT
1150/**
1151 * Gets the permission bits for the specified MSR in the MSR bitmap.
1152 *
1153 * @returns VBox status code.
1154 * @retval VINF_SUCCESS if the specified MSR is found.
1155 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1156 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1157 *
1158 * @param pVCpu Pointer to the VMCPU.
1159 * @param uMsr The MSR.
1160 * @param penmRead Where to store the read permissions.
1161 * @param penmWrite Where to store the write permissions.
1162 */
1163static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1164{
1165 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1166 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1167 int32_t iBit;
1168 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1169
1170 /* See hmR0VmxSetMsrPermission() for the layout. */
1171 if (uMsr <= 0x00001FFF)
1172 iBit = uMsr;
1173 else if ( uMsr >= 0xC0000000
1174 && uMsr <= 0xC0001FFF)
1175 {
1176 iBit = (uMsr - 0xC0000000);
1177 pbMsrBitmap += 0x400;
1178 }
1179 else
1180 {
1181 AssertMsgFailed(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1182 return VERR_NOT_SUPPORTED;
1183 }
1184
1185 Assert(iBit <= 0x1fff);
1186 if (ASMBitTest(pbMsrBitmap, iBit))
1187 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1188 else
1189 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1190
1191 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1192 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1193 else
1194 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1195 return VINF_SUCCESS;
1196}
1197#endif /* VBOX_STRICT */
1198
1199
1200/**
1201 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1202 * area.
1203 *
1204 * @returns VBox status code.
1205 * @param pVCpu Pointer to the VMCPU.
1206 * @param cMsrs The number of MSRs.
1207 */
1208DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1209{
1210 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1211 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1212 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1213 {
1214 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1215 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1216 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1217 }
1218
1219 /* Update number of guest MSRs to load/store across the world-switch. */
1220 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1221 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1222
1223 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1224 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1225
1226 /* Update the VCPU's copy of the MSR count. */
1227 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1228
1229 return VINF_SUCCESS;
1230}
1231
1232
1233/**
1234 * Adds a new (or updates the value of an existing) guest/host MSR
1235 * pair to be swapped during the world-switch as part of the
1236 * auto-load/store MSR area in the VMCS.
1237 *
1238 * @returns true if the MSR was added -and- its value was updated, false
1239 * otherwise.
1240 * @param pVCpu Pointer to the VMCPU.
1241 * @param uMsr The MSR.
1242 * @param uGuestMsr Value of the guest MSR.
1243 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1244 * necessary.
1245 */
1246static bool hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1247{
1248 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1249 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1250 uint32_t i;
1251 for (i = 0; i < cMsrs; i++)
1252 {
1253 if (pGuestMsr->u32Msr == uMsr)
1254 break;
1255 pGuestMsr++;
1256 }
1257
1258 bool fAdded = false;
1259 if (i == cMsrs)
1260 {
1261 ++cMsrs;
1262 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1263 AssertRC(rc);
1264
1265 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1266 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1267 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1268
1269 fAdded = true;
1270 }
1271
1272 /* Update the MSR values in the auto-load/store MSR area. */
1273 pGuestMsr->u32Msr = uMsr;
1274 pGuestMsr->u64Value = uGuestMsrValue;
1275
1276 /* Create/update the MSR slot in the host MSR area. */
1277 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1278 pHostMsr += i;
1279 pHostMsr->u32Msr = uMsr;
1280
1281 /*
1282 * Update the host MSR only when requested by the caller AND when we're
1283 * adding it to the auto-load/store area. Otherwise, it would have been
1284 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1285 */
1286 bool fUpdatedMsrValue = false;
1287 if ( fAdded
1288 && fUpdateHostMsr)
1289 {
1290 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1291 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1292 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1293 fUpdatedMsrValue = true;
1294 }
1295
1296 return fUpdatedMsrValue;
1297}
1298
1299
1300/**
1301 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1302 * auto-load/store MSR area in the VMCS.
1303 *
1304 * Does not fail if the MSR in @a uMsr is not found in the auto-load/store MSR
1305 * area.
1306 *
1307 * @returns VBox status code.
1308 * @param pVCpu Pointer to the VMCPU.
1309 * @param uMsr The MSR.
1310 */
1311static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1312{
1313 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1314 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1315 for (uint32_t i = 0; i < cMsrs; i++)
1316 {
1317 /* Find the MSR. */
1318 if (pGuestMsr->u32Msr == uMsr)
1319 {
1320 /* If it's the last MSR, simply reduce the count. */
1321 if (i == cMsrs - 1)
1322 {
1323 --cMsrs;
1324 break;
1325 }
1326
1327 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1328 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1329 pLastGuestMsr += cMsrs;
1330 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1331 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1332
1333 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1334 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1335 pLastHostMsr += cMsrs;
1336 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1337 pHostMsr->u64Value = pLastHostMsr->u64Value;
1338 --cMsrs;
1339 break;
1340 }
1341 pGuestMsr++;
1342 }
1343
1344 /* Update the VMCS if the count changed (meaning the MSR was found). */
1345 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1346 {
1347 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1348 AssertRCReturn(rc, rc);
1349
1350 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1351 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1352 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1353 }
1354
1355 return VINF_SUCCESS;
1356}
1357
1358
1359/**
1360 * Checks if the specified guest MSR is part of the auto-load/store area in
1361 * the VMCS.
1362 *
1363 * @returns true if found, false otherwise.
1364 * @param pVCpu Pointer to the VMCPU.
1365 * @param uMsr The MSR to find.
1366 */
1367static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1368{
1369 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1370 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1371
1372 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1373 {
1374 if (pGuestMsr->u32Msr == uMsr)
1375 return true;
1376 }
1377 return false;
1378}
1379
1380
1381/**
1382 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1383 *
1384 * @param pVCpu Pointer to the VMCPU.
1385 *
1386 * @remarks No-long-jump zone!!!
1387 */
1388static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1389{
1390 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1391 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1392 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1393 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1394
1395 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1396 {
1397 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1398 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1399 }
1400
1401 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1402}
1403
1404
1405#if HC_ARCH_BITS == 64
1406/**
1407 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1408 * perform lazy restoration of the host MSRs while leaving VT-x.
1409 *
1410 * @param pVCpu Pointer to the VMCPU.
1411 *
1412 * @remarks No-long-jump zone!!!
1413 */
1414static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1415{
1416 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1417
1418 /*
1419 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1420 */
1421 if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1422 {
1423 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1424 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1425 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1426 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1427 pVCpu->hm.s.vmx.fRestoreHostMsrs |= VMX_RESTORE_HOST_MSR_SAVED_HOST;
1428 }
1429}
1430
1431
1432/**
1433 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1434 * lazily while leaving VT-x.
1435 *
1436 * @returns true if it does, false otherwise.
1437 * @param pVCpu Pointer to the VMCPU.
1438 * @param uMsr The MSR to check.
1439 */
1440static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1441{
1442 NOREF(pVCpu);
1443 switch (uMsr)
1444 {
1445 case MSR_K8_LSTAR:
1446 case MSR_K6_STAR:
1447 case MSR_K8_SF_MASK:
1448 case MSR_K8_KERNEL_GS_BASE:
1449 return true;
1450 }
1451 return false;
1452}
1453
1454
1455/**
1456 * Saves a set of guests MSRs back into the guest-CPU context.
1457 *
1458 * @param pVCpu Pointer to the VMCPU.
1459 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1460 * out-of-sync. Make sure to update the required fields
1461 * before using them.
1462 *
1463 * @remarks No-long-jump zone!!!
1464 */
1465static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1466{
1467 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1468 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1469
1470 if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1471 {
1472 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1473 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1474 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1475 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1476 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1477 }
1478}
1479
1480
1481/**
1482 * Loads a set of guests MSRs to allow read/passthru to the guest.
1483 *
1484 * The name of this function is slightly confusing. This function does NOT
1485 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1486 * common prefix for functions dealing with "lazy restoration" of the shared
1487 * MSRs.
1488 *
1489 * @param pVCpu Pointer to the VMCPU.
1490 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1491 * out-of-sync. Make sure to update the required fields
1492 * before using them.
1493 *
1494 * @remarks No-long-jump zone!!!
1495 */
1496static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1497{
1498 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1499 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1500
1501 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1502 if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1503 {
1504#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1505 do { \
1506 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1507 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1508 else \
1509 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1510 } while (0)
1511
1512 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1513 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1514 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1515 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1516#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1517 }
1518 else
1519 {
1520 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1521 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1522 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1523 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1524 }
1525 pVCpu->hm.s.vmx.fRestoreHostMsrs |= VMX_RESTORE_HOST_MSR_LOADED_GUEST;
1526}
1527
1528
1529/**
1530 * Performs lazy restoration of the set of host MSRs if they were previously
1531 * loaded with guest MSR values.
1532 *
1533 * @param pVCpu Pointer to the VMCPU.
1534 *
1535 * @remarks No-long-jump zone!!!
1536 * @remarks The guest MSRs should have been saved back into the guest-CPU
1537 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1538 */
1539static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1540{
1541 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1542 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1543
1544 if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1545 {
1546 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1547 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1548 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1549 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1550 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1551 }
1552 pVCpu->hm.s.vmx.fRestoreHostMsrs &= ~(VMX_RESTORE_HOST_MSR_LOADED_GUEST | VMX_RESTORE_HOST_MSR_SAVED_HOST);
1553}
1554#endif /* HC_ARCH_BITS == 64 */
1555
1556
1557#ifdef VBOX_STRICT
1558/**
1559 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1560 * VMCS are correct.
1561 *
1562 * @param pVCpu Pointer to the VMCPU.
1563 */
1564static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1565{
1566 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1567
1568 /* Verify MSR counts in the VMCS are what we think it should be. */
1569 uint32_t cMsrs;
1570 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1571 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1572
1573 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1574 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1575
1576 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1577 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1578
1579 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1580 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1581 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1582 {
1583 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1584 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32\n", pHostMsr->u32Msr,
1585 pGuestMsr->u32Msr));
1586
1587 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1588 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64\n", pHostMsr->u32Msr,
1589 pHostMsr->u64Value, u64Msr));
1590
1591 /* Verify that the permissions are as expected in the MSR bitmap. */
1592 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1593 {
1594 VMXMSREXITREAD enmRead;
1595 VMXMSREXITWRITE enmWrite;
1596 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1597 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1598 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 No passthru read permission!\n",
1599 pGuestMsr->u32Msr));
1600 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 No passthru write permission!\n",
1601 pGuestMsr->u32Msr));
1602 }
1603 }
1604}
1605# endif /* VBOX_STRICT */
1606
1607
1608/**
1609 * Flushes the TLB using EPT.
1610 *
1611 * @returns VBox status code.
1612 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1613 * enmFlush).
1614 * @param enmFlush Type of flush.
1615 *
1616 * @remarks Caller is responsible for making sure this function is called only
1617 * when NestedPaging is supported and providing @a enmFlush that is
1618 * supported by the CPU.
1619 * @remarks Can be called with interrupts disabled.
1620 */
1621static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1622{
1623 uint64_t au64Descriptor[2];
1624 if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1625 au64Descriptor[0] = 0;
1626 else
1627 {
1628 Assert(pVCpu);
1629 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1630 }
1631 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1632
1633 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1634 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1635 rc));
1636 if ( RT_SUCCESS(rc)
1637 && pVCpu)
1638 {
1639 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1640 }
1641}
1642
1643
1644/**
1645 * Flushes the TLB using VPID.
1646 *
1647 * @returns VBox status code.
1648 * @param pVM Pointer to the VM.
1649 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1650 * enmFlush).
1651 * @param enmFlush Type of flush.
1652 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1653 * on @a enmFlush).
1654 *
1655 * @remarks Can be called with interrupts disabled.
1656 */
1657static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1658{
1659 NOREF(pVM);
1660 AssertPtr(pVM);
1661 Assert(pVM->hm.s.vmx.fVpid);
1662
1663 uint64_t au64Descriptor[2];
1664 if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1665 {
1666 au64Descriptor[0] = 0;
1667 au64Descriptor[1] = 0;
1668 }
1669 else
1670 {
1671 AssertPtr(pVCpu);
1672 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1673 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1674 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1675 au64Descriptor[1] = GCPtr;
1676 }
1677
1678 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1679 AssertMsg(rc == VINF_SUCCESS,
1680 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1681 if ( RT_SUCCESS(rc)
1682 && pVCpu)
1683 {
1684 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1685 }
1686}
1687
1688
1689/**
1690 * Invalidates a guest page by guest virtual address. Only relevant for
1691 * EPT/VPID, otherwise there is nothing really to invalidate.
1692 *
1693 * @returns VBox status code.
1694 * @param pVM Pointer to the VM.
1695 * @param pVCpu Pointer to the VMCPU.
1696 * @param GCVirt Guest virtual address of the page to invalidate.
1697 */
1698VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1699{
1700 AssertPtr(pVM);
1701 AssertPtr(pVCpu);
1702 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1703
1704 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1705 if (!fFlushPending)
1706 {
1707 /*
1708 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1709 * See @bugref{6043} and @bugref{6177}.
1710 *
1711 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1712 * function maybe called in a loop with individual addresses.
1713 */
1714 if (pVM->hm.s.vmx.fVpid)
1715 {
1716 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1717 {
1718 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1719 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1720 }
1721 else
1722 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1723 }
1724 else if (pVM->hm.s.fNestedPaging)
1725 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1726 }
1727
1728 return VINF_SUCCESS;
1729}
1730
1731
1732/**
1733 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1734 * otherwise there is nothing really to invalidate.
1735 *
1736 * @returns VBox status code.
1737 * @param pVM Pointer to the VM.
1738 * @param pVCpu Pointer to the VMCPU.
1739 * @param GCPhys Guest physical address of the page to invalidate.
1740 */
1741VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1742{
1743 NOREF(pVM); NOREF(GCPhys);
1744 LogFlowFunc(("%RGp\n", GCPhys));
1745
1746 /*
1747 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1748 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1749 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1750 */
1751 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1752 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1753 return VINF_SUCCESS;
1754}
1755
1756
1757/**
1758 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1759 * case where neither EPT nor VPID is supported by the CPU.
1760 *
1761 * @param pVM Pointer to the VM.
1762 * @param pVCpu Pointer to the VMCPU.
1763 * @param pCpu Pointer to the global HM struct.
1764 *
1765 * @remarks Called with interrupts disabled.
1766 */
1767static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1768{
1769 AssertPtr(pVCpu);
1770 AssertPtr(pCpu);
1771 NOREF(pVM);
1772
1773 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1774
1775 /** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1776#if 0
1777 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1778 pVCpu->hm.s.TlbShootdown.cPages = 0;
1779#endif
1780
1781 Assert(pCpu->idCpu != NIL_RTCPUID);
1782 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1783 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1784 pVCpu->hm.s.fForceTLBFlush = false;
1785 return;
1786}
1787
1788
1789/**
1790 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1791 *
1792 * @param pVM Pointer to the VM.
1793 * @param pVCpu Pointer to the VMCPU.
1794 * @param pCpu Pointer to the global HM CPU struct.
1795 * @remarks All references to "ASID" in this function pertains to "VPID" in
1796 * Intel's nomenclature. The reason is, to avoid confusion in compare
1797 * statements since the host-CPU copies are named "ASID".
1798 *
1799 * @remarks Called with interrupts disabled.
1800 */
1801static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1802{
1803#ifdef VBOX_WITH_STATISTICS
1804 bool fTlbFlushed = false;
1805# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1806# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1807 if (!fTlbFlushed) \
1808 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1809 } while (0)
1810#else
1811# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1812# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1813#endif
1814
1815 AssertPtr(pVM);
1816 AssertPtr(pCpu);
1817 AssertPtr(pVCpu);
1818 Assert(pCpu->idCpu != NIL_RTCPUID);
1819 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1820 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1821 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1822
1823
1824 /*
1825 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1826 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1827 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1828 */
1829 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1830 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1831 {
1832 ++pCpu->uCurrentAsid;
1833 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1834 {
1835 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1836 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1837 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1838 }
1839
1840 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1841 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1842 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1843
1844 /*
1845 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1846 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1847 */
1848 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1849 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1850 HMVMX_SET_TAGGED_TLB_FLUSHED();
1851 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1852 }
1853
1854 /* Check for explicit TLB shootdowns. */
1855 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1856 {
1857 /*
1858 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1859 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1860 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1861 * but not guest-physical mappings.
1862 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1863 */
1864 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1865 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1866 HMVMX_SET_TAGGED_TLB_FLUSHED();
1867 }
1868
1869 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1870 * where it is commented out. Support individual entry flushing
1871 * someday. */
1872#if 0
1873 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1874 {
1875 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1876
1877 /*
1878 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1879 * as supported by the CPU.
1880 */
1881 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1882 {
1883 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1884 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1885 }
1886 else
1887 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1888
1889 HMVMX_SET_TAGGED_TLB_FLUSHED();
1890 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1891 pVCpu->hm.s.TlbShootdown.cPages = 0;
1892 }
1893#endif
1894
1895 pVCpu->hm.s.fForceTLBFlush = false;
1896
1897 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1898
1899 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1900 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1901 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1902 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1903 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1904 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
1905 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
1906 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1907 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1908
1909 /* Update VMCS with the VPID. */
1910 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1911 AssertRC(rc);
1912
1913#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1914}
1915
1916
1917/**
1918 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
1919 *
1920 * @returns VBox status code.
1921 * @param pVM Pointer to the VM.
1922 * @param pVCpu Pointer to the VMCPU.
1923 * @param pCpu Pointer to the global HM CPU struct.
1924 *
1925 * @remarks Called with interrupts disabled.
1926 */
1927static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1928{
1929 AssertPtr(pVM);
1930 AssertPtr(pVCpu);
1931 AssertPtr(pCpu);
1932 Assert(pCpu->idCpu != NIL_RTCPUID);
1933 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1934 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1935
1936 /*
1937 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1938 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1939 */
1940 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1941 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1942 {
1943 pVCpu->hm.s.fForceTLBFlush = true;
1944 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1945 }
1946
1947 /* Check for explicit TLB shootdown flushes. */
1948 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1949 {
1950 pVCpu->hm.s.fForceTLBFlush = true;
1951 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1952 }
1953
1954 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1955 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1956
1957 if (pVCpu->hm.s.fForceTLBFlush)
1958 {
1959 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1960 pVCpu->hm.s.fForceTLBFlush = false;
1961 }
1962 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1963 * where it is commented out. Support individual entry flushing
1964 * someday. */
1965#if 0
1966 else
1967 {
1968 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1969 {
1970 /* We cannot flush individual entries without VPID support. Flush using EPT. */
1971 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1972 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1973 }
1974 else
1975 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1976
1977 pVCpu->hm.s.TlbShootdown.cPages = 0;
1978 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1979 }
1980#endif
1981}
1982
1983
1984/**
1985 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
1986 *
1987 * @returns VBox status code.
1988 * @param pVM Pointer to the VM.
1989 * @param pVCpu Pointer to the VMCPU.
1990 * @param pCpu Pointer to the global HM CPU struct.
1991 *
1992 * @remarks Called with interrupts disabled.
1993 */
1994static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1995{
1996 AssertPtr(pVM);
1997 AssertPtr(pVCpu);
1998 AssertPtr(pCpu);
1999 Assert(pCpu->idCpu != NIL_RTCPUID);
2000 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2001 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2002
2003 /*
2004 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2005 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2006 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2007 */
2008 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2009 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2010 {
2011 pVCpu->hm.s.fForceTLBFlush = true;
2012 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2013 }
2014
2015 /* Check for explicit TLB shootdown flushes. */
2016 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2017 {
2018 /*
2019 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2020 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2021 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2022 */
2023 pVCpu->hm.s.fForceTLBFlush = true;
2024 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2025 }
2026
2027 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2028 if (pVCpu->hm.s.fForceTLBFlush)
2029 {
2030 ++pCpu->uCurrentAsid;
2031 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2032 {
2033 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2034 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2035 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2036 }
2037
2038 pVCpu->hm.s.fForceTLBFlush = false;
2039 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2040 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2041 if (pCpu->fFlushAsidBeforeUse)
2042 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2043 }
2044 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2045 * where it is commented out. Support individual entry flushing
2046 * someday. */
2047#if 0
2048 else
2049 {
2050 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2051 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2052 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2053 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2054
2055 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2056 {
2057 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2058 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2059 {
2060 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2061 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2062 }
2063 else
2064 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2065
2066 pVCpu->hm.s.TlbShootdown.cPages = 0;
2067 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2068 }
2069 else
2070 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2071 }
2072#endif
2073
2074 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2075 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2076 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2077 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2078 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2079 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2080 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2081
2082 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2083 AssertRC(rc);
2084}
2085
2086
2087/**
2088 * Flushes the guest TLB entry based on CPU capabilities.
2089 *
2090 * @param pVCpu Pointer to the VMCPU.
2091 * @param pCpu Pointer to the global HM CPU struct.
2092 */
2093DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2094{
2095#ifdef HMVMX_ALWAYS_FLUSH_TLB
2096 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2097#endif
2098 PVM pVM = pVCpu->CTX_SUFF(pVM);
2099 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2100 {
2101 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2102 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2103 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2104 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2105 default:
2106 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2107 break;
2108 }
2109
2110 /* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2111 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2112
2113 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2114}
2115
2116
2117/**
2118 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2119 * TLB entries from the host TLB before VM-entry.
2120 *
2121 * @returns VBox status code.
2122 * @param pVM Pointer to the VM.
2123 */
2124static int hmR0VmxSetupTaggedTlb(PVM pVM)
2125{
2126 /*
2127 * Determine optimal flush type for Nested Paging.
2128 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2129 * guest execution (see hmR3InitFinalizeR0()).
2130 */
2131 if (pVM->hm.s.fNestedPaging)
2132 {
2133 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2134 {
2135 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2136 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
2137 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2138 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
2139 else
2140 {
2141 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2142 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2143 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2144 }
2145
2146 /* Make sure the write-back cacheable memory type for EPT is supported. */
2147 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2148 {
2149 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2150 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2151 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2152 }
2153 }
2154 else
2155 {
2156 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2157 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2158 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2159 }
2160 }
2161
2162 /*
2163 * Determine optimal flush type for VPID.
2164 */
2165 if (pVM->hm.s.vmx.fVpid)
2166 {
2167 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2168 {
2169 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2170 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
2171 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2172 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
2173 else
2174 {
2175 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2176 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2177 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2178 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2179 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2180 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2181 pVM->hm.s.vmx.fVpid = false;
2182 }
2183 }
2184 else
2185 {
2186 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2187 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2188 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2189 pVM->hm.s.vmx.fVpid = false;
2190 }
2191 }
2192
2193 /*
2194 * Setup the handler for flushing tagged-TLBs.
2195 */
2196 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2197 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2198 else if (pVM->hm.s.fNestedPaging)
2199 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2200 else if (pVM->hm.s.vmx.fVpid)
2201 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2202 else
2203 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2204 return VINF_SUCCESS;
2205}
2206
2207
2208/**
2209 * Sets up pin-based VM-execution controls in the VMCS.
2210 *
2211 * @returns VBox status code.
2212 * @param pVM Pointer to the VM.
2213 * @param pVCpu Pointer to the VMCPU.
2214 */
2215static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2216{
2217 AssertPtr(pVM);
2218 AssertPtr(pVCpu);
2219
2220 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2221 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2222
2223 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
2224 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
2225 Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
2226
2227 /* Enable the VMX preemption timer. */
2228 if (pVM->hm.s.vmx.fUsePreemptTimer)
2229 {
2230 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2231 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2232 }
2233
2234 if ((val & zap) != val)
2235 {
2236 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2237 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2238 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2239 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2240 }
2241
2242 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2243 AssertRCReturn(rc, rc);
2244
2245 /* Update VCPU with the currently set pin-based VM-execution controls. */
2246 pVCpu->hm.s.vmx.u32PinCtls = val;
2247 return rc;
2248}
2249
2250
2251/**
2252 * Sets up processor-based VM-execution controls in the VMCS.
2253 *
2254 * @returns VBox status code.
2255 * @param pVM Pointer to the VM.
2256 * @param pVMCPU Pointer to the VMCPU.
2257 */
2258static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2259{
2260 AssertPtr(pVM);
2261 AssertPtr(pVCpu);
2262
2263 int rc = VERR_INTERNAL_ERROR_5;
2264 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2265 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2266
2267 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2268 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2269 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2270 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2271 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2272 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2273 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2274
2275 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2276 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2277 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2278 {
2279 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2280 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2281 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2282 }
2283
2284 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2285 if (!pVM->hm.s.fNestedPaging)
2286 {
2287 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2288 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2289 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2290 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2291 }
2292
2293 /* Use TPR shadowing if supported by the CPU. */
2294 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2295 {
2296 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2297 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2298 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2299 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2300 AssertRCReturn(rc, rc);
2301
2302 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2303 /* CR8 writes causes a VM-exit based on TPR threshold. */
2304 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2305 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2306 }
2307 else
2308 {
2309 /*
2310 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2311 * Set this control only for 64-bit guests.
2312 */
2313 if (pVM->hm.s.fAllow64BitGuests)
2314 {
2315 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
2316 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
2317 }
2318 }
2319
2320 /* Use MSR-bitmaps if supported by the CPU. */
2321 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2322 {
2323 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2324
2325 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2326 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2327 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2328 AssertRCReturn(rc, rc);
2329
2330 /*
2331 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2332 * automatically as dedicated fields in the VMCS.
2333 */
2334 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2335 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2336 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2337 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2338 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2339
2340#if HC_ARCH_BITS == 64
2341 /*
2342 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2343 */
2344 if (pVM->hm.s.fAllow64BitGuests)
2345 {
2346 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2347 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2348 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2349 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2350 }
2351#endif
2352 }
2353
2354 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2355 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2356 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2357
2358 if ((val & zap) != val)
2359 {
2360 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2361 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2362 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2363 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2364 }
2365
2366 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2367 AssertRCReturn(rc, rc);
2368
2369 /* Update VCPU with the currently set processor-based VM-execution controls. */
2370 pVCpu->hm.s.vmx.u32ProcCtls = val;
2371
2372 /*
2373 * Secondary processor-based VM-execution controls.
2374 */
2375 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2376 {
2377 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2378 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2379
2380 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2381 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2382
2383 if (pVM->hm.s.fNestedPaging)
2384 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2385 else
2386 {
2387 /*
2388 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2389 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2390 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2391 */
2392 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2393 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2394 }
2395
2396 if (pVM->hm.s.vmx.fVpid)
2397 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2398
2399 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2400 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2401
2402 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2403 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2404 * done dynamically. */
2405 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2406 {
2407 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2408 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2409 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2410 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2411 AssertRCReturn(rc, rc);
2412 }
2413
2414 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2415 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2416
2417 if ((val & zap) != val)
2418 {
2419 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2420 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2421 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2422 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2423 }
2424
2425 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2426 AssertRCReturn(rc, rc);
2427
2428 /* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2429 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2430 }
2431 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2432 {
2433 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2434 "available\n"));
2435 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2436 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2437 }
2438
2439 return VINF_SUCCESS;
2440}
2441
2442
2443/**
2444 * Sets up miscellaneous (everything other than Pin & Processor-based
2445 * VM-execution) control fields in the VMCS.
2446 *
2447 * @returns VBox status code.
2448 * @param pVM Pointer to the VM.
2449 * @param pVCpu Pointer to the VMCPU.
2450 */
2451static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2452{
2453 NOREF(pVM);
2454 AssertPtr(pVM);
2455 AssertPtr(pVCpu);
2456
2457 int rc = VERR_GENERAL_FAILURE;
2458
2459 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2460#if 0
2461 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2462 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2463 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2464
2465 /*
2466 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2467 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2468 * We thus use the exception bitmap to control it rather than use both.
2469 */
2470 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2471 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2472
2473 /** @todo Explore possibility of using IO-bitmaps. */
2474 /* All IO & IOIO instructions cause VM-exits. */
2475 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2476 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2477
2478 /* Initialize the MSR-bitmap area. */
2479 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2480 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2481 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2482#endif
2483
2484 /* Setup MSR auto-load/store area. */
2485 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2486 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2487 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2488 AssertRCReturn(rc, rc);
2489 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2490 AssertRCReturn(rc, rc);
2491
2492 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2493 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2494 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2495 AssertRCReturn(rc, rc);
2496
2497 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2498 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2499 AssertRCReturn(rc, rc);
2500
2501 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2502#if 0
2503 /* Setup debug controls */
2504 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2505 AssertRCReturn(rc, rc);
2506 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2507 AssertRCReturn(rc, rc);
2508#endif
2509
2510 return rc;
2511}
2512
2513
2514/**
2515 * Sets up the initial exception bitmap in the VMCS based on static conditions
2516 * (i.e. conditions that cannot ever change after starting the VM).
2517 *
2518 * @returns VBox status code.
2519 * @param pVM Pointer to the VM.
2520 * @param pVCpu Pointer to the VMCPU.
2521 */
2522static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2523{
2524 AssertPtr(pVM);
2525 AssertPtr(pVCpu);
2526
2527 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2528
2529 uint32_t u32XcptBitmap = 0;
2530
2531 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2532 if (!pVM->hm.s.fNestedPaging)
2533 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2534
2535 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2536 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2537 AssertRCReturn(rc, rc);
2538 return rc;
2539}
2540
2541
2542/**
2543 * Sets up the initial guest-state mask. The guest-state mask is consulted
2544 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2545 * for the nested virtualization case (as it would cause a VM-exit).
2546 *
2547 * @param pVCpu Pointer to the VMCPU.
2548 */
2549static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2550{
2551 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2552 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2553 return VINF_SUCCESS;
2554}
2555
2556
2557/**
2558 * Does per-VM VT-x initialization.
2559 *
2560 * @returns VBox status code.
2561 * @param pVM Pointer to the VM.
2562 */
2563VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2564{
2565 LogFlowFunc(("pVM=%p\n", pVM));
2566
2567 int rc = hmR0VmxStructsAlloc(pVM);
2568 if (RT_FAILURE(rc))
2569 {
2570 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2571 return rc;
2572 }
2573
2574 return VINF_SUCCESS;
2575}
2576
2577
2578/**
2579 * Does per-VM VT-x termination.
2580 *
2581 * @returns VBox status code.
2582 * @param pVM Pointer to the VM.
2583 */
2584VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2585{
2586 LogFlowFunc(("pVM=%p\n", pVM));
2587
2588#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2589 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2590 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2591#endif
2592 hmR0VmxStructsFree(pVM);
2593 return VINF_SUCCESS;
2594}
2595
2596
2597/**
2598 * Sets up the VM for execution under VT-x.
2599 * This function is only called once per-VM during initialization.
2600 *
2601 * @returns VBox status code.
2602 * @param pVM Pointer to the VM.
2603 */
2604VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2605{
2606 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2607 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2608
2609 LogFlowFunc(("pVM=%p\n", pVM));
2610
2611 /*
2612 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2613 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2614 */
2615 /* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2616 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2617 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2618 || !pVM->hm.s.vmx.pRealModeTSS))
2619 {
2620 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2621 return VERR_INTERNAL_ERROR;
2622 }
2623
2624#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2625 /*
2626 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2627 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2628 */
2629 if ( pVM->hm.s.fAllow64BitGuests
2630 && !HMVMX_IS_64BIT_HOST_MODE())
2631 {
2632 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2633 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2634 }
2635#endif
2636
2637 /* Initialize these always, see hmR3InitFinalizeR0().*/
2638 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2639 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2640
2641 /* Setup the tagged-TLB flush handlers. */
2642 int rc = hmR0VmxSetupTaggedTlb(pVM);
2643 if (RT_FAILURE(rc))
2644 {
2645 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2646 return rc;
2647 }
2648
2649 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2650 {
2651 PVMCPU pVCpu = &pVM->aCpus[i];
2652 AssertPtr(pVCpu);
2653 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2654
2655 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2656 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2657
2658 /* Set revision dword at the beginning of the VMCS structure. */
2659 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2660
2661 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2662 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2663 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2664 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2665
2666 /* Load this VMCS as the current VMCS. */
2667 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2668 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2669 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2670
2671 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2672 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2673 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2674
2675 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2676 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2677 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2678
2679 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2680 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2681 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2682
2683 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2684 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2685 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2686
2687 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2688 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2689 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2690
2691#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2692 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2693 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2694 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2695#endif
2696
2697 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2698 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2699 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2700 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2701
2702 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2703
2704 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2705 }
2706
2707 return VINF_SUCCESS;
2708}
2709
2710
2711/**
2712 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2713 * the VMCS.
2714 *
2715 * @returns VBox status code.
2716 * @param pVM Pointer to the VM.
2717 * @param pVCpu Pointer to the VMCPU.
2718 */
2719DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2720{
2721 NOREF(pVM); NOREF(pVCpu);
2722
2723 RTCCUINTREG uReg = ASMGetCR0();
2724 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2725 AssertRCReturn(rc, rc);
2726
2727#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2728 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2729 if (HMVMX_IS_64BIT_HOST_MODE())
2730 {
2731 uint64_t uRegCR3 = HMR0Get64bitCR3();
2732 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2733 }
2734 else
2735#endif
2736 {
2737 uReg = ASMGetCR3();
2738 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2739 }
2740 AssertRCReturn(rc, rc);
2741
2742 uReg = ASMGetCR4();
2743 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2744 AssertRCReturn(rc, rc);
2745 return rc;
2746}
2747
2748
2749#if HC_ARCH_BITS == 64
2750/**
2751 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2752 * requirements. See hmR0VmxSaveHostSegmentRegs().
2753 */
2754# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2755 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2756 { \
2757 bool fValidSelector = true; \
2758 if ((selValue) & X86_SEL_LDT) \
2759 { \
2760 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2761 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2762 } \
2763 if (fValidSelector) \
2764 { \
2765 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2766 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2767 } \
2768 (selValue) = 0; \
2769 }
2770#endif
2771
2772
2773/**
2774 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2775 * the host-state area in the VMCS.
2776 *
2777 * @returns VBox status code.
2778 * @param pVM Pointer to the VM.
2779 * @param pVCpu Pointer to the VMCPU.
2780 */
2781DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2782{
2783 NOREF(pVM);
2784 int rc = VERR_INTERNAL_ERROR_5;
2785
2786#if HC_ARCH_BITS == 64
2787 /*
2788 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2789 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2790 */
2791 AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2792 ("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2793#endif
2794
2795 /*
2796 * Host DS, ES, FS and GS segment registers.
2797 */
2798#if HC_ARCH_BITS == 64
2799 RTSEL uSelDS = ASMGetDS();
2800 RTSEL uSelES = ASMGetES();
2801 RTSEL uSelFS = ASMGetFS();
2802 RTSEL uSelGS = ASMGetGS();
2803#else
2804 RTSEL uSelDS = 0;
2805 RTSEL uSelES = 0;
2806 RTSEL uSelFS = 0;
2807 RTSEL uSelGS = 0;
2808#endif
2809
2810 /* Recalculate which host-state bits need to be manually restored. */
2811 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2812
2813 /*
2814 * Host CS and SS segment registers.
2815 */
2816#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2817 RTSEL uSelCS;
2818 RTSEL uSelSS;
2819 if (HMVMX_IS_64BIT_HOST_MODE())
2820 {
2821 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2822 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2823 }
2824 else
2825 {
2826 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2827 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2828 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2829 }
2830#else
2831 RTSEL uSelCS = ASMGetCS();
2832 RTSEL uSelSS = ASMGetSS();
2833#endif
2834
2835 /*
2836 * Host TR segment register.
2837 */
2838 RTSEL uSelTR = ASMGetTR();
2839
2840#if HC_ARCH_BITS == 64
2841 /*
2842 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2843 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2844 */
2845 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2846 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2847 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2848 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2849# undef VMXLOCAL_ADJUST_HOST_SEG
2850#endif
2851
2852 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2853 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2854 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2855 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2856 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2857 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2858 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2859 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2860 Assert(uSelCS);
2861 Assert(uSelTR);
2862
2863 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2864#if 0
2865 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2866 Assert(uSelSS != 0);
2867#endif
2868
2869 /* Write these host selector fields into the host-state area in the VMCS. */
2870 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2871 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2872#if HC_ARCH_BITS == 64
2873 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2874 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2875 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2876 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2877#endif
2878 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2879
2880 /*
2881 * Host GDTR and IDTR.
2882 */
2883 RTGDTR Gdtr;
2884 RT_ZERO(Gdtr);
2885#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2886 if (HMVMX_IS_64BIT_HOST_MODE())
2887 {
2888 X86XDTR64 Gdtr64;
2889 X86XDTR64 Idtr64;
2890 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2891 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2892 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2893
2894 Gdtr.cbGdt = Gdtr64.cb;
2895 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2896 }
2897 else
2898#endif
2899 {
2900 RTIDTR Idtr;
2901 ASMGetGDTR(&Gdtr);
2902 ASMGetIDTR(&Idtr);
2903 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2904 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2905
2906#if HC_ARCH_BITS == 64
2907 /*
2908 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2909 * maximum limit (0xffff) on every VM-exit.
2910 */
2911 if (Gdtr.cbGdt != 0xffff)
2912 {
2913 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2914 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2915 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2916 }
2917
2918 /*
2919 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
2920 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
2921 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
2922 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
2923 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
2924 * hosts where we are pretty sure it won't cause trouble.
2925 */
2926# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
2927 if (Idtr.cbIdt < 0x0fff)
2928# else
2929 if (Idtr.cbIdt != 0xffff)
2930# endif
2931 {
2932 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
2933 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2934 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2935 }
2936#endif
2937 }
2938
2939 /*
2940 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2941 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2942 */
2943 if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2944 {
2945 AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2946 return VERR_VMX_INVALID_HOST_STATE;
2947 }
2948
2949 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2950#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2951 if (HMVMX_IS_64BIT_HOST_MODE())
2952 {
2953 /* We need the 64-bit TR base for hybrid darwin. */
2954 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2955 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2956 }
2957 else
2958#endif
2959 {
2960 uintptr_t uTRBase;
2961#if HC_ARCH_BITS == 64
2962 uTRBase = X86DESC64_BASE(pDesc);
2963
2964 /*
2965 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2966 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2967 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2968 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2969 *
2970 * [1] See Intel spec. 3.5 "System Descriptor Types".
2971 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2972 */
2973 Assert(pDesc->System.u4Type == 11);
2974 if ( pDesc->System.u16LimitLow != 0x67
2975 || pDesc->System.u4LimitHigh)
2976 {
2977 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
2978 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2979
2980 /* Store the GDTR here as we need it while restoring TR. */
2981 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2982 }
2983#else
2984 uTRBase = X86DESC_BASE(pDesc);
2985#endif
2986 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2987 }
2988 AssertRCReturn(rc, rc);
2989
2990 /*
2991 * Host FS base and GS base.
2992 */
2993#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2994 if (HMVMX_IS_64BIT_HOST_MODE())
2995 {
2996 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2997 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2998 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2999 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
3000
3001# if HC_ARCH_BITS == 64
3002 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3003 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3004 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3005 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3006 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3007# endif
3008 }
3009#endif
3010 return rc;
3011}
3012
3013
3014/**
3015 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3016 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3017 * the host after every successful VM-exit.
3018 *
3019 * @returns VBox status code.
3020 * @param pVM Pointer to the VM.
3021 * @param pVCpu Pointer to the VMCPU.
3022 *
3023 * @remarks No-long-jump zone!!!
3024 */
3025DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3026{
3027 NOREF(pVM);
3028
3029 AssertPtr(pVCpu);
3030 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3031
3032 int rc = VINF_SUCCESS;
3033#if HC_ARCH_BITS == 64
3034 if (pVM->hm.s.fAllow64BitGuests)
3035 hmR0VmxLazySaveHostMsrs(pVCpu);
3036#endif
3037
3038 /*
3039 * Host Sysenter MSRs.
3040 */
3041 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3042 AssertRCReturn(rc, rc);
3043#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3044 if (HMVMX_IS_64BIT_HOST_MODE())
3045 {
3046 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3047 AssertRCReturn(rc, rc);
3048 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3049 }
3050 else
3051 {
3052 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3053 AssertRCReturn(rc, rc);
3054 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3055 }
3056#elif HC_ARCH_BITS == 32
3057 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3058 AssertRCReturn(rc, rc);
3059 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3060#else
3061 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3062 AssertRCReturn(rc, rc);
3063 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3064#endif
3065 AssertRCReturn(rc, rc);
3066
3067 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
3068 * hmR0VmxSetupExitCtls() !! */
3069 return rc;
3070}
3071
3072
3073/**
3074 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3075 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3076 * controls".
3077 *
3078 * @returns VBox status code.
3079 * @param pVCpu Pointer to the VMCPU.
3080 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3081 * out-of-sync. Make sure to update the required fields
3082 * before using them.
3083 *
3084 * @remarks No-long-jump zone!!!
3085 */
3086DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3087{
3088 int rc = VINF_SUCCESS;
3089 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3090 {
3091 PVM pVM = pVCpu->CTX_SUFF(pVM);
3092 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3093 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3094
3095 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3096 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3097
3098 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3099 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3100 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3101 else
3102 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3103
3104 /*
3105 * The following should -not- be set (since we're not in SMM mode):
3106 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3107 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3108 */
3109
3110 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3111 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
3112 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
3113
3114 if ((val & zap) != val)
3115 {
3116 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3117 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3118 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3119 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3120 }
3121
3122 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3123 AssertRCReturn(rc, rc);
3124
3125 /* Update VCPU with the currently set VM-exit controls. */
3126 pVCpu->hm.s.vmx.u32EntryCtls = val;
3127 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3128 }
3129 return rc;
3130}
3131
3132
3133/**
3134 * Sets up the VM-exit controls in the VMCS.
3135 *
3136 * @returns VBox status code.
3137 * @param pVM Pointer to the VM.
3138 * @param pVCpu Pointer to the VMCPU.
3139 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3140 * out-of-sync. Make sure to update the required fields
3141 * before using them.
3142 *
3143 * @remarks requires EFER.
3144 */
3145DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3146{
3147 NOREF(pMixedCtx);
3148
3149 int rc = VINF_SUCCESS;
3150 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3151 {
3152 PVM pVM = pVCpu->CTX_SUFF(pVM);
3153 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3154 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3155
3156 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3157 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3158
3159 /*
3160 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3161 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3162 */
3163#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3164 if (HMVMX_IS_64BIT_HOST_MODE())
3165 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3166 else
3167 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3168#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3169 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3170 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
3171 else
3172 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3173#endif
3174
3175 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3176 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3177
3178 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3179 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3180 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
3181 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
3182 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
3183
3184 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3185 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3186
3187 if ((val & zap) != val)
3188 {
3189 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3190 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3191 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3192 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3193 }
3194
3195 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3196 AssertRCReturn(rc, rc);
3197
3198 /* Update VCPU with the currently set VM-exit controls. */
3199 pVCpu->hm.s.vmx.u32ExitCtls = val;
3200 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3201 }
3202 return rc;
3203}
3204
3205
3206/**
3207 * Loads the guest APIC and related state.
3208 *
3209 * @returns VBox status code.
3210 * @param pVM Pointer to the VM.
3211 * @param pVCpu Pointer to the VMCPU.
3212 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3213 * out-of-sync. Make sure to update the required fields
3214 * before using them.
3215 */
3216DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3217{
3218 NOREF(pMixedCtx);
3219
3220 int rc = VINF_SUCCESS;
3221 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3222 {
3223 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3224 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3225 {
3226 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3227
3228 bool fPendingIntr = false;
3229 uint8_t u8Tpr = 0;
3230 uint8_t u8PendingIntr = 0;
3231 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3232 AssertRCReturn(rc, rc);
3233
3234 /*
3235 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3236 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3237 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3238 * the interrupt when we VM-exit for other reasons.
3239 */
3240 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3241 uint32_t u32TprThreshold = 0;
3242 if (fPendingIntr)
3243 {
3244 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3245 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3246 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3247 if (u8PendingPriority <= u8TprPriority)
3248 u32TprThreshold = u8PendingPriority;
3249 else
3250 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3251 }
3252 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3253
3254 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3255 AssertRCReturn(rc, rc);
3256 }
3257
3258 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3259 }
3260 return rc;
3261}
3262
3263
3264/**
3265 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3266 *
3267 * @returns Guest's interruptibility-state.
3268 * @param pVCpu Pointer to the VMCPU.
3269 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3270 * out-of-sync. Make sure to update the required fields
3271 * before using them.
3272 *
3273 * @remarks No-long-jump zone!!!
3274 * @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3275 */
3276DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3277{
3278 /*
3279 * Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3280 * inhibit interrupts or clear any existing interrupt-inhibition.
3281 */
3282 uint32_t uIntrState = 0;
3283 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3284 {
3285 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3286 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3287 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3288 if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3289 {
3290 /*
3291 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3292 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3293 */
3294 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3295 }
3296 else if (pMixedCtx->eflags.Bits.u1IF)
3297 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3298 else
3299 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3300 }
3301 return uIntrState;
3302}
3303
3304
3305/**
3306 * Loads the guest's interruptibility-state into the guest-state area in the
3307 * VMCS.
3308 *
3309 * @returns VBox status code.
3310 * @param pVCpu Pointer to the VMCPU.
3311 * @param uIntrState The interruptibility-state to set.
3312 */
3313static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3314{
3315 NOREF(pVCpu);
3316 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3317 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3318 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3319 AssertRCReturn(rc, rc);
3320 return rc;
3321}
3322
3323
3324/**
3325 * Loads the guest's RIP into the guest-state area in the VMCS.
3326 *
3327 * @returns VBox status code.
3328 * @param pVCpu Pointer to the VMCPU.
3329 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3330 * out-of-sync. Make sure to update the required fields
3331 * before using them.
3332 *
3333 * @remarks No-long-jump zone!!!
3334 */
3335static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3336{
3337 int rc = VINF_SUCCESS;
3338 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3339 {
3340 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3341 AssertRCReturn(rc, rc);
3342
3343 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3344 Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, HMCPU_CF_VALUE(pVCpu)));
3345 }
3346 return rc;
3347}
3348
3349
3350/**
3351 * Loads the guest's RSP into the guest-state area in the VMCS.
3352 *
3353 * @returns VBox status code.
3354 * @param pVCpu Pointer to the VMCPU.
3355 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3356 * out-of-sync. Make sure to update the required fields
3357 * before using them.
3358 *
3359 * @remarks No-long-jump zone!!!
3360 */
3361static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3362{
3363 int rc = VINF_SUCCESS;
3364 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3365 {
3366 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3367 AssertRCReturn(rc, rc);
3368
3369 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3370 Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
3371 }
3372 return rc;
3373}
3374
3375
3376/**
3377 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3378 *
3379 * @returns VBox status code.
3380 * @param pVCpu Pointer to the VMCPU.
3381 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3382 * out-of-sync. Make sure to update the required fields
3383 * before using them.
3384 *
3385 * @remarks No-long-jump zone!!!
3386 */
3387static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3388{
3389 int rc = VINF_SUCCESS;
3390 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3391 {
3392 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3393 Let us assert it as such and use 32-bit VMWRITE. */
3394 Assert(!(pMixedCtx->rflags.u64 >> 32));
3395 X86EFLAGS Eflags = pMixedCtx->eflags;
3396 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3397 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3398
3399 /*
3400 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3401 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3402 */
3403 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3404 {
3405 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3406 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3407 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3408 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3409 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3410 }
3411
3412 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3413 AssertRCReturn(rc, rc);
3414
3415 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3416 Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
3417 }
3418 return rc;
3419}
3420
3421
3422/**
3423 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3424 *
3425 * @returns VBox status code.
3426 * @param pVCpu Pointer to the VMCPU.
3427 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3428 * out-of-sync. Make sure to update the required fields
3429 * before using them.
3430 *
3431 * @remarks No-long-jump zone!!!
3432 */
3433DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3434{
3435 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3436 AssertRCReturn(rc, rc);
3437 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3438 AssertRCReturn(rc, rc);
3439 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3440 AssertRCReturn(rc, rc);
3441 return rc;
3442}
3443
3444
3445/**
3446 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3447 * CR0 is partially shared with the host and we have to consider the FPU bits.
3448 *
3449 * @returns VBox status code.
3450 * @param pVM Pointer to the VM.
3451 * @param pVCpu Pointer to the VMCPU.
3452 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3453 * out-of-sync. Make sure to update the required fields
3454 * before using them.
3455 *
3456 * @remarks No-long-jump zone!!!
3457 */
3458static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3459{
3460 /*
3461 * Guest CR0.
3462 * Guest FPU.
3463 */
3464 int rc = VINF_SUCCESS;
3465 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3466 {
3467 Assert(!(pMixedCtx->cr0 >> 32));
3468 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3469 PVM pVM = pVCpu->CTX_SUFF(pVM);
3470
3471 /* The guest's view (read access) of its CR0 is unblemished. */
3472 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3473 AssertRCReturn(rc, rc);
3474 Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3475
3476 /* Setup VT-x's view of the guest CR0. */
3477 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3478 if (pVM->hm.s.fNestedPaging)
3479 {
3480 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3481 {
3482 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3483 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3484 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3485 }
3486 else
3487 {
3488 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3489 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3490 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3491 }
3492
3493 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3494 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3495 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3496
3497 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3498 AssertRCReturn(rc, rc);
3499 }
3500 else
3501 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3502
3503 /*
3504 * Guest FPU bits.
3505 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3506 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3507 */
3508 u32GuestCR0 |= X86_CR0_NE;
3509 bool fInterceptNM = false;
3510 if (CPUMIsGuestFPUStateActive(pVCpu))
3511 {
3512 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3513 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3514 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3515 }
3516 else
3517 {
3518 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3519 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3520 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3521 }
3522
3523 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3524 bool fInterceptMF = false;
3525 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3526 fInterceptMF = true;
3527
3528 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3529 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3530 {
3531 Assert(PDMVmmDevHeapIsEnabled(pVM));
3532 Assert(pVM->hm.s.vmx.pRealModeTSS);
3533 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3534 fInterceptNM = true;
3535 fInterceptMF = true;
3536 }
3537 else
3538 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3539
3540 if (fInterceptNM)
3541 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3542 else
3543 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3544
3545 if (fInterceptMF)
3546 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3547 else
3548 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3549
3550 /* Additional intercepts for debugging, define these yourself explicitly. */
3551#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3552 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3553 | RT_BIT(X86_XCPT_BP)
3554 | RT_BIT(X86_XCPT_DB)
3555 | RT_BIT(X86_XCPT_DE)
3556 | RT_BIT(X86_XCPT_NM)
3557 | RT_BIT(X86_XCPT_UD)
3558 | RT_BIT(X86_XCPT_NP)
3559 | RT_BIT(X86_XCPT_SS)
3560 | RT_BIT(X86_XCPT_GP)
3561 | RT_BIT(X86_XCPT_PF)
3562 | RT_BIT(X86_XCPT_MF)
3563 ;
3564#elif defined(HMVMX_ALWAYS_TRAP_PF)
3565 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3566#endif
3567
3568 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3569
3570 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3571 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3572 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3573 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3574 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3575 else
3576 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3577
3578 u32GuestCR0 |= uSetCR0;
3579 u32GuestCR0 &= uZapCR0;
3580 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3581
3582 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3583 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3584 AssertRCReturn(rc, rc);
3585 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3586 AssertRCReturn(rc, rc);
3587 Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3588
3589 /*
3590 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3591 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3592 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3593 */
3594 uint32_t u32CR0Mask = 0;
3595 u32CR0Mask = X86_CR0_PE
3596 | X86_CR0_NE
3597 | X86_CR0_WP
3598 | X86_CR0_PG
3599 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3600 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3601 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3602
3603 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3604 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3605 * and @bugref{6944}. */
3606#if 0
3607 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3608 u32CR0Mask &= ~X86_CR0_PE;
3609#endif
3610 if (pVM->hm.s.fNestedPaging)
3611 u32CR0Mask &= ~X86_CR0_WP;
3612
3613 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3614 if (fInterceptNM)
3615 {
3616 u32CR0Mask |= X86_CR0_TS
3617 | X86_CR0_MP;
3618 }
3619
3620 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3621 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3622 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3623 AssertRCReturn(rc, rc);
3624 Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3625
3626 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3627 }
3628 return rc;
3629}
3630
3631
3632/**
3633 * Loads the guest control registers (CR3, CR4) into the guest-state area
3634 * in the VMCS.
3635 *
3636 * @returns VBox status code.
3637 * @param pVM Pointer to the VM.
3638 * @param pVCpu Pointer to the VMCPU.
3639 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3640 * out-of-sync. Make sure to update the required fields
3641 * before using them.
3642 *
3643 * @remarks No-long-jump zone!!!
3644 */
3645static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3646{
3647 int rc = VINF_SUCCESS;
3648 PVM pVM = pVCpu->CTX_SUFF(pVM);
3649
3650 /*
3651 * Guest CR2.
3652 * It's always loaded in the assembler code. Nothing to do here.
3653 */
3654
3655 /*
3656 * Guest CR3.
3657 */
3658 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3659 {
3660 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3661 if (pVM->hm.s.fNestedPaging)
3662 {
3663 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3664
3665 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3666 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3667 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3668 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3669
3670 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3671 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3672 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3673
3674 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3675 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3676 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3677 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3678
3679 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3680 AssertRCReturn(rc, rc);
3681 Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3682
3683 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3684 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3685 {
3686 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3687 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3688 {
3689 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3690 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3691 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3692 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3693 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3694 }
3695
3696 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3697 have Unrestricted Execution to handle the guest when it's not using paging. */
3698 GCPhysGuestCR3 = pMixedCtx->cr3;
3699 }
3700 else
3701 {
3702 /*
3703 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3704 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3705 * EPT takes care of translating it to host-physical addresses.
3706 */
3707 RTGCPHYS GCPhys;
3708 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3709 Assert(PDMVmmDevHeapIsEnabled(pVM));
3710
3711 /* We obtain it here every time as the guest could have relocated this PCI region. */
3712 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3713 AssertRCReturn(rc, rc);
3714
3715 GCPhysGuestCR3 = GCPhys;
3716 }
3717
3718 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3719 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3720 }
3721 else
3722 {
3723 /* Non-nested paging case, just use the hypervisor's CR3. */
3724 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3725
3726 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3727 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3728 }
3729 AssertRCReturn(rc, rc);
3730
3731 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3732 }
3733
3734 /*
3735 * Guest CR4.
3736 */
3737 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3738 {
3739 Assert(!(pMixedCtx->cr4 >> 32));
3740 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3741
3742 /* The guest's view of its CR4 is unblemished. */
3743 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3744 AssertRCReturn(rc, rc);
3745 Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3746
3747 /* Setup VT-x's view of the guest CR4. */
3748 /*
3749 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3750 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3751 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3752 */
3753 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3754 {
3755 Assert(pVM->hm.s.vmx.pRealModeTSS);
3756 Assert(PDMVmmDevHeapIsEnabled(pVM));
3757 u32GuestCR4 &= ~X86_CR4_VME;
3758 }
3759
3760 if (pVM->hm.s.fNestedPaging)
3761 {
3762 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3763 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3764 {
3765 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3766 u32GuestCR4 |= X86_CR4_PSE;
3767 /* Our identity mapping is a 32-bit page directory. */
3768 u32GuestCR4 &= ~X86_CR4_PAE;
3769 }
3770 /* else use guest CR4.*/
3771 }
3772 else
3773 {
3774 /*
3775 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3776 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3777 */
3778 switch (pVCpu->hm.s.enmShadowMode)
3779 {
3780 case PGMMODE_REAL: /* Real-mode. */
3781 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3782 case PGMMODE_32_BIT: /* 32-bit paging. */
3783 {
3784 u32GuestCR4 &= ~X86_CR4_PAE;
3785 break;
3786 }
3787
3788 case PGMMODE_PAE: /* PAE paging. */
3789 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3790 {
3791 u32GuestCR4 |= X86_CR4_PAE;
3792 break;
3793 }
3794
3795 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3796 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3797#ifdef VBOX_ENABLE_64_BITS_GUESTS
3798 break;
3799#endif
3800 default:
3801 AssertFailed();
3802 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3803 }
3804 }
3805
3806 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3807 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3808 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3809 u32GuestCR4 |= uSetCR4;
3810 u32GuestCR4 &= uZapCR4;
3811
3812 /* Write VT-x's view of the guest CR4 into the VMCS. */
3813 Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3814 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3815 AssertRCReturn(rc, rc);
3816
3817 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
3818 uint32_t u32CR4Mask = 0;
3819 u32CR4Mask = X86_CR4_VME
3820 | X86_CR4_PAE
3821 | X86_CR4_PGE
3822 | X86_CR4_PSE
3823 | X86_CR4_VMXE;
3824 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3825 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3826 AssertRCReturn(rc, rc);
3827
3828 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
3829 }
3830 return rc;
3831}
3832
3833
3834/**
3835 * Loads the guest debug registers into the guest-state area in the VMCS.
3836 * This also sets up whether #DB and MOV DRx accesses cause VM-exits.
3837 *
3838 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3839 *
3840 * @returns VBox status code.
3841 * @param pVCpu Pointer to the VMCPU.
3842 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3843 * out-of-sync. Make sure to update the required fields
3844 * before using them.
3845 *
3846 * @remarks No-long-jump zone!!!
3847 */
3848static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3849{
3850 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
3851 return VINF_SUCCESS;
3852
3853#ifdef VBOX_STRICT
3854 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3855 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3856 {
3857 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3858 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3859 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3860 }
3861#endif
3862
3863 int rc;
3864 PVM pVM = pVCpu->CTX_SUFF(pVM);
3865 bool fInterceptDB = false;
3866 bool fInterceptMovDRx = false;
3867 if ( pVCpu->hm.s.fSingleInstruction
3868 || DBGFIsStepping(pVCpu))
3869 {
3870 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3871 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3872 {
3873 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3874 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3875 AssertRCReturn(rc, rc);
3876 Assert(fInterceptDB == false);
3877 }
3878 else
3879 {
3880 pMixedCtx->eflags.u32 |= X86_EFL_TF;
3881 pVCpu->hm.s.fClearTrapFlag = true;
3882 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3883 fInterceptDB = true;
3884 }
3885 }
3886
3887 if ( fInterceptDB
3888 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3889 {
3890 /*
3891 * Use the combined guest and host DRx values found in the hypervisor
3892 * register set because the debugger has breakpoints active or someone
3893 * is single stepping on the host side without a monitor trap flag.
3894 *
3895 * Note! DBGF expects a clean DR6 state before executing guest code.
3896 */
3897#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3898 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3899 && !CPUMIsHyperDebugStateActivePending(pVCpu))
3900 {
3901 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3902 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
3903 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
3904 }
3905 else
3906#endif
3907 if (!CPUMIsHyperDebugStateActive(pVCpu))
3908 {
3909 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3910 Assert(CPUMIsHyperDebugStateActive(pVCpu));
3911 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3912 }
3913
3914 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3915 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3916 AssertRCReturn(rc, rc);
3917
3918 pVCpu->hm.s.fUsingHyperDR7 = true;
3919 fInterceptDB = true;
3920 fInterceptMovDRx = true;
3921 }
3922 else
3923 {
3924 /*
3925 * If the guest has enabled debug registers, we need to load them prior to
3926 * executing guest code so they'll trigger at the right time.
3927 */
3928 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
3929 {
3930#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3931 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3932 && !CPUMIsGuestDebugStateActivePending(pVCpu))
3933 {
3934 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3935 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
3936 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
3937 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3938 }
3939 else
3940#endif
3941 if (!CPUMIsGuestDebugStateActive(pVCpu))
3942 {
3943 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3944 Assert(CPUMIsGuestDebugStateActive(pVCpu));
3945 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3946 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3947 }
3948 Assert(!fInterceptDB);
3949 Assert(!fInterceptMovDRx);
3950 }
3951 /*
3952 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3953 * must intercept #DB in order to maintain a correct DR6 guest value.
3954 */
3955#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3956 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
3957 && !CPUMIsGuestDebugStateActive(pVCpu))
3958#else
3959 else if (!CPUMIsGuestDebugStateActive(pVCpu))
3960#endif
3961 {
3962 fInterceptMovDRx = true;
3963 fInterceptDB = true;
3964 }
3965
3966 /* Update guest DR7. */
3967 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3968 AssertRCReturn(rc, rc);
3969
3970 pVCpu->hm.s.fUsingHyperDR7 = false;
3971 }
3972
3973 /*
3974 * Update the exception bitmap regarding intercepting #DB generated by the guest.
3975 */
3976 if (fInterceptDB)
3977 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
3978 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3979 {
3980#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3981 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3982#endif
3983 }
3984 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3985 AssertRCReturn(rc, rc);
3986
3987 /*
3988 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3989 */
3990 if (fInterceptMovDRx)
3991 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3992 else
3993 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3994 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3995 AssertRCReturn(rc, rc);
3996
3997 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
3998 return VINF_SUCCESS;
3999}
4000
4001
4002#ifdef VBOX_STRICT
4003/**
4004 * Strict function to validate segment registers.
4005 *
4006 * @remarks ASSUMES CR0 is up to date.
4007 */
4008static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4009{
4010 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4011 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4012 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4013 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4014 && ( !CPUMIsGuestInRealModeEx(pCtx)
4015 && !CPUMIsGuestInV86ModeEx(pCtx)))
4016 {
4017 /* Protected mode checks */
4018 /* CS */
4019 Assert(pCtx->cs.Attr.n.u1Present);
4020 Assert(!(pCtx->cs.Attr.u & 0xf00));
4021 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4022 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4023 || !(pCtx->cs.Attr.n.u1Granularity));
4024 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4025 || (pCtx->cs.Attr.n.u1Granularity));
4026 /* CS cannot be loaded with NULL in protected mode. */
4027 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4028 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4029 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4030 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4031 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4032 else
4033 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4034 /* SS */
4035 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4036 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4037 if ( !(pCtx->cr0 & X86_CR0_PE)
4038 || pCtx->cs.Attr.n.u4Type == 3)
4039 {
4040 Assert(!pCtx->ss.Attr.n.u2Dpl);
4041 }
4042 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4043 {
4044 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4045 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4046 Assert(pCtx->ss.Attr.n.u1Present);
4047 Assert(!(pCtx->ss.Attr.u & 0xf00));
4048 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4049 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4050 || !(pCtx->ss.Attr.n.u1Granularity));
4051 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4052 || (pCtx->ss.Attr.n.u1Granularity));
4053 }
4054 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4055 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4056 {
4057 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4058 Assert(pCtx->ds.Attr.n.u1Present);
4059 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4060 Assert(!(pCtx->ds.Attr.u & 0xf00));
4061 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4062 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4063 || !(pCtx->ds.Attr.n.u1Granularity));
4064 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4065 || (pCtx->ds.Attr.n.u1Granularity));
4066 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4067 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4068 }
4069 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4070 {
4071 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4072 Assert(pCtx->es.Attr.n.u1Present);
4073 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4074 Assert(!(pCtx->es.Attr.u & 0xf00));
4075 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4076 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4077 || !(pCtx->es.Attr.n.u1Granularity));
4078 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4079 || (pCtx->es.Attr.n.u1Granularity));
4080 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4081 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4082 }
4083 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4084 {
4085 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4086 Assert(pCtx->fs.Attr.n.u1Present);
4087 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4088 Assert(!(pCtx->fs.Attr.u & 0xf00));
4089 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4090 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4091 || !(pCtx->fs.Attr.n.u1Granularity));
4092 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4093 || (pCtx->fs.Attr.n.u1Granularity));
4094 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4095 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4096 }
4097 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4098 {
4099 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4100 Assert(pCtx->gs.Attr.n.u1Present);
4101 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4102 Assert(!(pCtx->gs.Attr.u & 0xf00));
4103 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4104 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4105 || !(pCtx->gs.Attr.n.u1Granularity));
4106 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4107 || (pCtx->gs.Attr.n.u1Granularity));
4108 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4109 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4110 }
4111 /* 64-bit capable CPUs. */
4112# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4113 if (HMVMX_IS_64BIT_HOST_MODE())
4114 {
4115 Assert(!(pCtx->cs.u64Base >> 32));
4116 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4117 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4118 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4119 }
4120# endif
4121 }
4122 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4123 || ( CPUMIsGuestInRealModeEx(pCtx)
4124 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4125 {
4126 /* Real and v86 mode checks. */
4127 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4128 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4129 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4130 {
4131 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4132 }
4133 else
4134 {
4135 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4136 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4137 }
4138
4139 /* CS */
4140 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4141 Assert(pCtx->cs.u32Limit == 0xffff);
4142 Assert(u32CSAttr == 0xf3);
4143 /* SS */
4144 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4145 Assert(pCtx->ss.u32Limit == 0xffff);
4146 Assert(u32SSAttr == 0xf3);
4147 /* DS */
4148 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4149 Assert(pCtx->ds.u32Limit == 0xffff);
4150 Assert(u32DSAttr == 0xf3);
4151 /* ES */
4152 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4153 Assert(pCtx->es.u32Limit == 0xffff);
4154 Assert(u32ESAttr == 0xf3);
4155 /* FS */
4156 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4157 Assert(pCtx->fs.u32Limit == 0xffff);
4158 Assert(u32FSAttr == 0xf3);
4159 /* GS */
4160 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4161 Assert(pCtx->gs.u32Limit == 0xffff);
4162 Assert(u32GSAttr == 0xf3);
4163 /* 64-bit capable CPUs. */
4164# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4165 if (HMVMX_IS_64BIT_HOST_MODE())
4166 {
4167 Assert(!(pCtx->cs.u64Base >> 32));
4168 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4169 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4170 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4171 }
4172# endif
4173 }
4174}
4175#endif /* VBOX_STRICT */
4176
4177
4178/**
4179 * Writes a guest segment register into the guest-state area in the VMCS.
4180 *
4181 * @returns VBox status code.
4182 * @param pVCpu Pointer to the VMCPU.
4183 * @param idxSel Index of the selector in the VMCS.
4184 * @param idxLimit Index of the segment limit in the VMCS.
4185 * @param idxBase Index of the segment base in the VMCS.
4186 * @param idxAccess Index of the access rights of the segment in the VMCS.
4187 * @param pSelReg Pointer to the segment selector.
4188 *
4189 * @remarks No-long-jump zone!!!
4190 */
4191static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4192 uint32_t idxAccess, PCPUMSELREG pSelReg)
4193{
4194 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4195 AssertRCReturn(rc, rc);
4196 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4197 AssertRCReturn(rc, rc);
4198 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4199 AssertRCReturn(rc, rc);
4200
4201 uint32_t u32Access = pSelReg->Attr.u;
4202 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4203 {
4204 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4205 u32Access = 0xf3;
4206 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4207 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4208 }
4209 else
4210 {
4211 /*
4212 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4213 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4214 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4215 * loaded in protected-mode have their attribute as 0.
4216 */
4217 if (!u32Access)
4218 u32Access = X86DESCATTR_UNUSABLE;
4219 }
4220
4221 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4222 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4223 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4224
4225 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4226 AssertRCReturn(rc, rc);
4227 return rc;
4228}
4229
4230
4231/**
4232 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4233 * into the guest-state area in the VMCS.
4234 *
4235 * @returns VBox status code.
4236 * @param pVM Pointer to the VM.
4237 * @param pVCPU Pointer to the VMCPU.
4238 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4239 * out-of-sync. Make sure to update the required fields
4240 * before using them.
4241 *
4242 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4243 * @remarks No-long-jump zone!!!
4244 */
4245static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4246{
4247 int rc = VERR_INTERNAL_ERROR_5;
4248 PVM pVM = pVCpu->CTX_SUFF(pVM);
4249
4250 /*
4251 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4252 */
4253 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4254 {
4255 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4256 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4257 {
4258 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4259 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4260 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4261 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4262 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4263 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4264 }
4265
4266#ifdef VBOX_WITH_REM
4267 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4268 {
4269 Assert(pVM->hm.s.vmx.pRealModeTSS);
4270 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4271 if ( pVCpu->hm.s.vmx.fWasInRealMode
4272 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4273 {
4274 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4275 in real-mode (e.g. OpenBSD 4.0) */
4276 REMFlushTBs(pVM);
4277 Log4(("Load: Switch to protected mode detected!\n"));
4278 pVCpu->hm.s.vmx.fWasInRealMode = false;
4279 }
4280 }
4281#endif
4282 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4283 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4284 AssertRCReturn(rc, rc);
4285 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4286 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4287 AssertRCReturn(rc, rc);
4288 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4289 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4290 AssertRCReturn(rc, rc);
4291 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4292 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4293 AssertRCReturn(rc, rc);
4294 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4295 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4296 AssertRCReturn(rc, rc);
4297 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4298 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4299 AssertRCReturn(rc, rc);
4300
4301#ifdef VBOX_STRICT
4302 /* Validate. */
4303 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4304#endif
4305
4306 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4307 Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4308 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4309 }
4310
4311 /*
4312 * Guest TR.
4313 */
4314 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4315 {
4316 /*
4317 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4318 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4319 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4320 */
4321 uint16_t u16Sel = 0;
4322 uint32_t u32Limit = 0;
4323 uint64_t u64Base = 0;
4324 uint32_t u32AccessRights = 0;
4325
4326 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4327 {
4328 u16Sel = pMixedCtx->tr.Sel;
4329 u32Limit = pMixedCtx->tr.u32Limit;
4330 u64Base = pMixedCtx->tr.u64Base;
4331 u32AccessRights = pMixedCtx->tr.Attr.u;
4332 }
4333 else
4334 {
4335 Assert(pVM->hm.s.vmx.pRealModeTSS);
4336 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4337
4338 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4339 RTGCPHYS GCPhys;
4340 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4341 AssertRCReturn(rc, rc);
4342
4343 X86DESCATTR DescAttr;
4344 DescAttr.u = 0;
4345 DescAttr.n.u1Present = 1;
4346 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4347
4348 u16Sel = 0;
4349 u32Limit = HM_VTX_TSS_SIZE;
4350 u64Base = GCPhys; /* in real-mode phys = virt. */
4351 u32AccessRights = DescAttr.u;
4352 }
4353
4354 /* Validate. */
4355 Assert(!(u16Sel & RT_BIT(2)));
4356 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4357 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4358 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4359 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4360 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4361 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4362 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4363 Assert( (u32Limit & 0xfff) == 0xfff
4364 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4365 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4366 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4367
4368 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4369 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4370 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4371 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4372
4373 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4374 Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
4375 }
4376
4377 /*
4378 * Guest GDTR.
4379 */
4380 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4381 {
4382 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4383 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4384
4385 /* Validate. */
4386 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4387
4388 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4389 Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
4390 }
4391
4392 /*
4393 * Guest LDTR.
4394 */
4395 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4396 {
4397 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4398 uint32_t u32Access = 0;
4399 if (!pMixedCtx->ldtr.Attr.u)
4400 u32Access = X86DESCATTR_UNUSABLE;
4401 else
4402 u32Access = pMixedCtx->ldtr.Attr.u;
4403
4404 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4405 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4406 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4407 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4408
4409 /* Validate. */
4410 if (!(u32Access & X86DESCATTR_UNUSABLE))
4411 {
4412 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4413 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4414 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4415 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4416 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4417 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4418 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4419 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4420 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4421 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4422 }
4423
4424 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4425 Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
4426 }
4427
4428 /*
4429 * Guest IDTR.
4430 */
4431 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4432 {
4433 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4434 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4435
4436 /* Validate. */
4437 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4438
4439 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4440 Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
4441 }
4442
4443 return VINF_SUCCESS;
4444}
4445
4446
4447/**
4448 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4449 * areas. These MSRs will automatically be loaded to the host CPU on every
4450 * successful VM entry and stored from the host CPU on every successful VM-exit.
4451 *
4452 * This also creates/updates MSR slots for the host MSRs. The actual host
4453 * MSR values are -not- updated here for performance reasons. See
4454 * hmR0VmxSaveHostMsrs().
4455 *
4456 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4457 *
4458 * @returns VBox status code.
4459 * @param pVCpu Pointer to the VMCPU.
4460 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4461 * out-of-sync. Make sure to update the required fields
4462 * before using them.
4463 *
4464 * @remarks No-long-jump zone!!!
4465 */
4466static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4467{
4468 AssertPtr(pVCpu);
4469 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4470
4471 /*
4472 * MSRs that we use the auto-load/store MSR area in the VMCS.
4473 */
4474 PVM pVM = pVCpu->CTX_SUFF(pVM);
4475 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4476 {
4477#if HC_ARCH_BITS == 32 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4478 if (pVM->hm.s.fAllow64BitGuests)
4479 {
4480 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4481 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4482 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4483 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4484# ifdef DEBUG
4485 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4486 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4487 Log4(("Load: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4488# endif
4489 }
4490#endif
4491 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4492 }
4493
4494 /*
4495 * Guest Sysenter MSRs.
4496 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4497 * VM-exits on WRMSRs for these MSRs.
4498 */
4499 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4500 {
4501 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4502 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4503 }
4504
4505 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4506 {
4507 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4508 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4509 }
4510
4511 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4512 {
4513 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4514 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4515 }
4516
4517 return VINF_SUCCESS;
4518}
4519
4520
4521/**
4522 * Loads the guest activity state into the guest-state area in the VMCS.
4523 *
4524 * @returns VBox status code.
4525 * @param pVCpu Pointer to the VMCPU.
4526 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4527 * out-of-sync. Make sure to update the required fields
4528 * before using them.
4529 *
4530 * @remarks No-long-jump zone!!!
4531 */
4532static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4533{
4534 NOREF(pCtx);
4535 /** @todo See if we can make use of other states, e.g.
4536 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4537 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4538 {
4539 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4540 AssertRCReturn(rc, rc);
4541
4542 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4543 }
4544 return VINF_SUCCESS;
4545}
4546
4547
4548/**
4549 * Sets up the appropriate function to run guest code.
4550 *
4551 * @returns VBox status code.
4552 * @param pVCpu Pointer to the VMCPU.
4553 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4554 * out-of-sync. Make sure to update the required fields
4555 * before using them.
4556 *
4557 * @remarks No-long-jump zone!!!
4558 */
4559static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4560{
4561 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4562 {
4563#ifndef VBOX_ENABLE_64_BITS_GUESTS
4564 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4565#endif
4566 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4567#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4568 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4569 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4570 {
4571 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4572 {
4573 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4574 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_EXIT_CTLS | HM_CHANGED_VMX_ENTRY_CTLS),
4575 ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4576 }
4577 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4578 }
4579#else
4580 /* 64-bit host or hybrid host. */
4581 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4582#endif
4583 }
4584 else
4585 {
4586 /* Guest is not in long mode, use the 32-bit handler. */
4587#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4588 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4589 {
4590 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4591 {
4592 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4593 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_EXIT_CTLS | HM_CHANGED_VMX_ENTRY_CTLS),
4594 ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4595 }
4596 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4597 }
4598#else
4599 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4600#endif
4601 }
4602 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4603 return VINF_SUCCESS;
4604}
4605
4606
4607/**
4608 * Wrapper for running the guest code in VT-x.
4609 *
4610 * @returns VBox strict status code.
4611 * @param pVM Pointer to the VM.
4612 * @param pVCpu Pointer to the VMCPU.
4613 * @param pCtx Pointer to the guest-CPU context.
4614 *
4615 * @remarks No-long-jump zone!!!
4616 */
4617DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4618{
4619 /*
4620 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4621 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4622 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4623 */
4624 const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4625 /** @todo Add stats for resume vs launch. */
4626#ifdef VBOX_WITH_KERNEL_USING_XMM
4627 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4628#else
4629 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4630#endif
4631}
4632
4633
4634/**
4635 * Reports world-switch error and dumps some useful debug info.
4636 *
4637 * @param pVM Pointer to the VM.
4638 * @param pVCpu Pointer to the VMCPU.
4639 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4640 * @param pCtx Pointer to the guest-CPU context.
4641 * @param pVmxTransient Pointer to the VMX transient structure (only
4642 * exitReason updated).
4643 */
4644static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4645{
4646 Assert(pVM);
4647 Assert(pVCpu);
4648 Assert(pCtx);
4649 Assert(pVmxTransient);
4650 HMVMX_ASSERT_PREEMPT_SAFE();
4651
4652 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4653 switch (rcVMRun)
4654 {
4655 case VERR_VMX_INVALID_VMXON_PTR:
4656 AssertFailed();
4657 break;
4658 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4659 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4660 {
4661 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4662 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4663 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4664 AssertRC(rc);
4665
4666 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4667 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4668 Cannot do it here as we may have been long preempted. */
4669
4670#ifdef VBOX_STRICT
4671 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4672 pVmxTransient->uExitReason));
4673 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4674 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4675 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4676 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4677 else
4678 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4679 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4680 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4681
4682 /* VMX control bits. */
4683 uint32_t u32Val;
4684 uint64_t u64Val;
4685 HMVMXHCUINTREG uHCReg;
4686 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4687 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4688 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4689 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4690 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4691 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4692 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4693 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4694 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4695 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4696 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4697 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4698 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4699 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4700 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4701 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4702 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4703 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4704 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4705 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4706 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4707 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4708 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4709 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4710 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4711 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4712 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4713 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4714 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4715 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4716 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4717 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4718 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4719 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4720 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4721 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4722 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4723 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4724 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4725 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4726 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4727 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4728
4729 /* Guest bits. */
4730 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4731 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4732 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4733 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4734 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4735 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4736 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4737 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4738
4739 /* Host bits. */
4740 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4741 Log4(("Host CR0 %#RHr\n", uHCReg));
4742 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4743 Log4(("Host CR3 %#RHr\n", uHCReg));
4744 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4745 Log4(("Host CR4 %#RHr\n", uHCReg));
4746
4747 RTGDTR HostGdtr;
4748 PCX86DESCHC pDesc;
4749 ASMGetGDTR(&HostGdtr);
4750 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4751 Log4(("Host CS %#08x\n", u32Val));
4752 if (u32Val < HostGdtr.cbGdt)
4753 {
4754 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4755 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4756 }
4757
4758 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4759 Log4(("Host DS %#08x\n", u32Val));
4760 if (u32Val < HostGdtr.cbGdt)
4761 {
4762 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4763 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4764 }
4765
4766 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4767 Log4(("Host ES %#08x\n", u32Val));
4768 if (u32Val < HostGdtr.cbGdt)
4769 {
4770 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4771 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4772 }
4773
4774 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4775 Log4(("Host FS %#08x\n", u32Val));
4776 if (u32Val < HostGdtr.cbGdt)
4777 {
4778 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4779 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4780 }
4781
4782 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4783 Log4(("Host GS %#08x\n", u32Val));
4784 if (u32Val < HostGdtr.cbGdt)
4785 {
4786 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4787 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4788 }
4789
4790 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4791 Log4(("Host SS %#08x\n", u32Val));
4792 if (u32Val < HostGdtr.cbGdt)
4793 {
4794 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4795 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4796 }
4797
4798 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4799 Log4(("Host TR %#08x\n", u32Val));
4800 if (u32Val < HostGdtr.cbGdt)
4801 {
4802 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4803 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4804 }
4805
4806 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4807 Log4(("Host TR Base %#RHv\n", uHCReg));
4808 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4809 Log4(("Host GDTR Base %#RHv\n", uHCReg));
4810 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4811 Log4(("Host IDTR Base %#RHv\n", uHCReg));
4812 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4813 Log4(("Host SYSENTER CS %#08x\n", u32Val));
4814 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4815 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4816 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4817 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4818 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4819 Log4(("Host RSP %#RHv\n", uHCReg));
4820 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4821 Log4(("Host RIP %#RHv\n", uHCReg));
4822# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4823 if (HMVMX_IS_64BIT_HOST_MODE())
4824 {
4825 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4826 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4827 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4828 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4829 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4830 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4831 }
4832# endif
4833#endif /* VBOX_STRICT */
4834 break;
4835 }
4836
4837 default:
4838 /* Impossible */
4839 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4840 break;
4841 }
4842 NOREF(pVM); NOREF(pCtx);
4843}
4844
4845
4846#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4847#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4848# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4849#endif
4850#ifdef VBOX_STRICT
4851static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4852{
4853 switch (idxField)
4854 {
4855 case VMX_VMCS_GUEST_RIP:
4856 case VMX_VMCS_GUEST_RSP:
4857 case VMX_VMCS_GUEST_SYSENTER_EIP:
4858 case VMX_VMCS_GUEST_SYSENTER_ESP:
4859 case VMX_VMCS_GUEST_GDTR_BASE:
4860 case VMX_VMCS_GUEST_IDTR_BASE:
4861 case VMX_VMCS_GUEST_CS_BASE:
4862 case VMX_VMCS_GUEST_DS_BASE:
4863 case VMX_VMCS_GUEST_ES_BASE:
4864 case VMX_VMCS_GUEST_FS_BASE:
4865 case VMX_VMCS_GUEST_GS_BASE:
4866 case VMX_VMCS_GUEST_SS_BASE:
4867 case VMX_VMCS_GUEST_LDTR_BASE:
4868 case VMX_VMCS_GUEST_TR_BASE:
4869 case VMX_VMCS_GUEST_CR3:
4870 return true;
4871 }
4872 return false;
4873}
4874
4875static bool hmR0VmxIsValidReadField(uint32_t idxField)
4876{
4877 switch (idxField)
4878 {
4879 /* Read-only fields. */
4880 case VMX_VMCS_RO_EXIT_QUALIFICATION:
4881 return true;
4882 }
4883 /* Remaining readable fields should also be writable. */
4884 return hmR0VmxIsValidWriteField(idxField);
4885}
4886#endif /* VBOX_STRICT */
4887
4888
4889/**
4890 * Executes the specified handler in 64-bit mode.
4891 *
4892 * @returns VBox status code.
4893 * @param pVM Pointer to the VM.
4894 * @param pVCpu Pointer to the VMCPU.
4895 * @param pCtx Pointer to the guest CPU context.
4896 * @param enmOp The operation to perform.
4897 * @param cbParam Number of parameters.
4898 * @param paParam Array of 32-bit parameters.
4899 */
4900VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4901 uint32_t *paParam)
4902{
4903 int rc, rc2;
4904 PHMGLOBALCPUINFO pCpu;
4905 RTHCPHYS HCPhysCpuPage;
4906 RTCCUINTREG uOldEflags;
4907
4908 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4909 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4910 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4911 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4912
4913#ifdef VBOX_STRICT
4914 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4915 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4916
4917 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4918 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4919#endif
4920
4921 /* Disable interrupts. */
4922 uOldEflags = ASMIntDisableFlags();
4923
4924#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4925 RTCPUID idHostCpu = RTMpCpuId();
4926 CPUMR0SetLApic(pVCpu, idHostCpu);
4927#endif
4928
4929 pCpu = HMR0GetCurrentCpu();
4930 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4931
4932 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4933 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4934
4935 /* Leave VMX Root Mode. */
4936 VMXDisable();
4937
4938 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4939
4940 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4941 CPUMSetHyperEIP(pVCpu, enmOp);
4942 for (int i = (int)cbParam - 1; i >= 0; i--)
4943 CPUMPushHyper(pVCpu, paParam[i]);
4944
4945 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4946
4947 /* Call the switcher. */
4948 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4949 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4950
4951 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4952 /* Make sure the VMX instructions don't cause #UD faults. */
4953 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
4954
4955 /* Re-enter VMX Root Mode */
4956 rc2 = VMXEnable(HCPhysCpuPage);
4957 if (RT_FAILURE(rc2))
4958 {
4959 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4960 ASMSetFlags(uOldEflags);
4961 return rc2;
4962 }
4963
4964 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4965 AssertRC(rc2);
4966 Assert(!(ASMGetFlags() & X86_EFL_IF));
4967 ASMSetFlags(uOldEflags);
4968 return rc;
4969}
4970
4971
4972/**
4973 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
4974 * supporting 64-bit guests.
4975 *
4976 * @returns VBox status code.
4977 * @param fResume Whether to VMLAUNCH or VMRESUME.
4978 * @param pCtx Pointer to the guest-CPU context.
4979 * @param pCache Pointer to the VMCS cache.
4980 * @param pVM Pointer to the VM.
4981 * @param pVCpu Pointer to the VMCPU.
4982 */
4983DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4984{
4985 uint32_t aParam[6];
4986 PHMGLOBALCPUINFO pCpu = NULL;
4987 RTHCPHYS HCPhysCpuPage = 0;
4988 int rc = VERR_INTERNAL_ERROR_5;
4989
4990 pCpu = HMR0GetCurrentCpu();
4991 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4992
4993#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4994 pCache->uPos = 1;
4995 pCache->interPD = PGMGetInterPaeCR3(pVM);
4996 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4997#endif
4998
4999#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5000 pCache->TestIn.HCPhysCpuPage = 0;
5001 pCache->TestIn.HCPhysVmcs = 0;
5002 pCache->TestIn.pCache = 0;
5003 pCache->TestOut.HCPhysVmcs = 0;
5004 pCache->TestOut.pCache = 0;
5005 pCache->TestOut.pCtx = 0;
5006 pCache->TestOut.eflags = 0;
5007#endif
5008
5009 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5010 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5011 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5012 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5013 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5014 aParam[5] = 0;
5015
5016#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5017 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5018 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5019#endif
5020 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
5021
5022#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5023 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5024 Assert(pCtx->dr[4] == 10);
5025 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5026#endif
5027
5028#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5029 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5030 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5031 pVCpu->hm.s.vmx.HCPhysVmcs));
5032 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5033 pCache->TestOut.HCPhysVmcs));
5034 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5035 pCache->TestOut.pCache));
5036 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5037 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5038 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5039 pCache->TestOut.pCtx));
5040 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5041#endif
5042 return rc;
5043}
5044
5045
5046/**
5047 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5048 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5049 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5050 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5051 *
5052 * @returns VBox status code.
5053 * @param pVM Pointer to the VM.
5054 * @param pVCpu Pointer to the VMCPU.
5055 */
5056static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5057{
5058#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5059{ \
5060 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5061 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5062 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5063 ++cReadFields; \
5064}
5065
5066 AssertPtr(pVM);
5067 AssertPtr(pVCpu);
5068 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5069 uint32_t cReadFields = 0;
5070
5071 /*
5072 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5073 * and serve to indicate exceptions to the rules.
5074 */
5075
5076 /* Guest-natural selector base fields. */
5077#if 0
5078 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5079 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5080 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5081#endif
5082 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5083 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5084 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5085 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5086 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5087 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5088 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5089 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5090 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5091 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5092 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5093 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5094#if 0
5095 /* Unused natural width guest-state fields. */
5096 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5097 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5098#endif
5099 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5100 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5101
5102 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5103#if 0
5104 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5105 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5106 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5107 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5108 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5109 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5110 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5111 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5112 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5113#endif
5114
5115 /* Natural width guest-state fields. */
5116 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5117#if 0
5118 /* Currently unused field. */
5119 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5120#endif
5121
5122 if (pVM->hm.s.fNestedPaging)
5123 {
5124 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5125 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5126 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5127 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5128 }
5129 else
5130 {
5131 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5132 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5133 }
5134
5135#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5136 return VINF_SUCCESS;
5137}
5138
5139
5140/**
5141 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5142 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5143 * darwin, running 64-bit guests).
5144 *
5145 * @returns VBox status code.
5146 * @param pVCpu Pointer to the VMCPU.
5147 * @param idxField The VMCS field encoding.
5148 * @param u64Val 16, 32 or 64-bit value.
5149 */
5150VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5151{
5152 int rc;
5153 switch (idxField)
5154 {
5155 /*
5156 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5157 */
5158 /* 64-bit Control fields. */
5159 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5160 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5161 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5162 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5163 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5164 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5165 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5166 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5167 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5168 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5169 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5170 case VMX_VMCS64_CTRL_EPTP_FULL:
5171 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5172 /* 64-bit Guest-state fields. */
5173 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5174 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5175 case VMX_VMCS64_GUEST_PAT_FULL:
5176 case VMX_VMCS64_GUEST_EFER_FULL:
5177 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5178 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5179 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5180 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5181 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5182 /* 64-bit Host-state fields. */
5183 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5184 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5185 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5186 {
5187 rc = VMXWriteVmcs32(idxField, u64Val);
5188 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5189 break;
5190 }
5191
5192 /*
5193 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5194 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5195 */
5196 /* Natural-width Guest-state fields. */
5197 case VMX_VMCS_GUEST_CR3:
5198 case VMX_VMCS_GUEST_ES_BASE:
5199 case VMX_VMCS_GUEST_CS_BASE:
5200 case VMX_VMCS_GUEST_SS_BASE:
5201 case VMX_VMCS_GUEST_DS_BASE:
5202 case VMX_VMCS_GUEST_FS_BASE:
5203 case VMX_VMCS_GUEST_GS_BASE:
5204 case VMX_VMCS_GUEST_LDTR_BASE:
5205 case VMX_VMCS_GUEST_TR_BASE:
5206 case VMX_VMCS_GUEST_GDTR_BASE:
5207 case VMX_VMCS_GUEST_IDTR_BASE:
5208 case VMX_VMCS_GUEST_RSP:
5209 case VMX_VMCS_GUEST_RIP:
5210 case VMX_VMCS_GUEST_SYSENTER_ESP:
5211 case VMX_VMCS_GUEST_SYSENTER_EIP:
5212 {
5213 if (!(u64Val >> 32))
5214 {
5215 /* If this field is 64-bit, VT-x will zero out the top bits. */
5216 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5217 }
5218 else
5219 {
5220 /* Assert that only the 32->64 switcher case should ever come here. */
5221 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5222 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5223 }
5224 break;
5225 }
5226
5227 default:
5228 {
5229 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5230 rc = VERR_INVALID_PARAMETER;
5231 break;
5232 }
5233 }
5234 AssertRCReturn(rc, rc);
5235 return rc;
5236}
5237
5238
5239/**
5240 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5241 * hosts (except darwin) for 64-bit guests.
5242 *
5243 * @param pVCpu Pointer to the VMCPU.
5244 * @param idxField The VMCS field encoding.
5245 * @param u64Val 16, 32 or 64-bit value.
5246 */
5247VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5248{
5249 AssertPtr(pVCpu);
5250 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5251
5252 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5253 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5254
5255 /* Make sure there are no duplicates. */
5256 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5257 {
5258 if (pCache->Write.aField[i] == idxField)
5259 {
5260 pCache->Write.aFieldVal[i] = u64Val;
5261 return VINF_SUCCESS;
5262 }
5263 }
5264
5265 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5266 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5267 pCache->Write.cValidEntries++;
5268 return VINF_SUCCESS;
5269}
5270
5271/* Enable later when the assembly code uses these as callbacks. */
5272#if 0
5273/*
5274 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5275 *
5276 * @param pVCpu Pointer to the VMCPU.
5277 * @param pCache Pointer to the VMCS cache.
5278 *
5279 * @remarks No-long-jump zone!!!
5280 */
5281VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5282{
5283 AssertPtr(pCache);
5284 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5285 {
5286 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5287 AssertRC(rc);
5288 }
5289 pCache->Write.cValidEntries = 0;
5290}
5291
5292
5293/**
5294 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
5295 *
5296 * @param pVCpu Pointer to the VMCPU.
5297 * @param pCache Pointer to the VMCS cache.
5298 *
5299 * @remarks No-long-jump zone!!!
5300 */
5301VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5302{
5303 AssertPtr(pCache);
5304 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5305 {
5306 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5307 AssertRC(rc);
5308 }
5309}
5310#endif
5311#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5312
5313
5314/**
5315 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5316 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5317 * timer.
5318 *
5319 * @returns VBox status code.
5320 * @param pVCpu Pointer to the VMCPU.
5321 *
5322 * @remarks No-long-jump zone!!!
5323 */
5324static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5325{
5326 int rc = VERR_INTERNAL_ERROR_5;
5327 bool fOffsettedTsc = false;
5328 PVM pVM = pVCpu->CTX_SUFF(pVM);
5329 if (pVM->hm.s.vmx.fUsePreemptTimer)
5330 {
5331 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
5332
5333 /* Make sure the returned values have sane upper and lower boundaries. */
5334 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5335 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5336 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5337 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5338
5339 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5340 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5341 }
5342 else
5343 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
5344
5345 if (fOffsettedTsc)
5346 {
5347 uint64_t u64CurTSC = ASMReadTSC();
5348 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5349 {
5350 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5351 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5352
5353 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5354 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5355 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5356 }
5357 else
5358 {
5359 /* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5360 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5361 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5362 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5363 }
5364 }
5365 else
5366 {
5367 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5368 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5369 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5370 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5371 }
5372}
5373
5374
5375/**
5376 * Determines if an exception is a contributory exception. Contributory
5377 * exceptions are ones which can cause double-faults. Page-fault is
5378 * intentionally not included here as it's a conditional contributory exception.
5379 *
5380 * @returns true if the exception is contributory, false otherwise.
5381 * @param uVector The exception vector.
5382 */
5383DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5384{
5385 switch (uVector)
5386 {
5387 case X86_XCPT_GP:
5388 case X86_XCPT_SS:
5389 case X86_XCPT_NP:
5390 case X86_XCPT_TS:
5391 case X86_XCPT_DE:
5392 return true;
5393 default:
5394 break;
5395 }
5396 return false;
5397}
5398
5399
5400/**
5401 * Sets an event as a pending event to be injected into the guest.
5402 *
5403 * @param pVCpu Pointer to the VMCPU.
5404 * @param u32IntInfo The VM-entry interruption-information field.
5405 * @param cbInstr The VM-entry instruction length in bytes (for software
5406 * interrupts, exceptions and privileged software
5407 * exceptions).
5408 * @param u32ErrCode The VM-entry exception error code.
5409 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5410 * page-fault.
5411 *
5412 * @remarks Statistics counter assumes this is a guest event being injected or
5413 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5414 * always incremented.
5415 */
5416DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5417 RTGCUINTPTR GCPtrFaultAddress)
5418{
5419 Assert(!pVCpu->hm.s.Event.fPending);
5420 pVCpu->hm.s.Event.fPending = true;
5421 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5422 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5423 pVCpu->hm.s.Event.cbInstr = cbInstr;
5424 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5425
5426 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5427}
5428
5429
5430/**
5431 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5432 *
5433 * @param pVCpu Pointer to the VMCPU.
5434 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5435 * out-of-sync. Make sure to update the required fields
5436 * before using them.
5437 */
5438DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5439{
5440 NOREF(pMixedCtx);
5441 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5442 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5443 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5444 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5445}
5446
5447
5448/**
5449 * Handle a condition that occurred while delivering an event through the guest
5450 * IDT.
5451 *
5452 * @returns VBox status code (informational error codes included).
5453 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5454 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5455 * continue execution of the guest which will delivery the #DF.
5456 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5457 *
5458 * @param pVCpu Pointer to the VMCPU.
5459 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5460 * out-of-sync. Make sure to update the required fields
5461 * before using them.
5462 * @param pVmxTransient Pointer to the VMX transient structure.
5463 *
5464 * @remarks No-long-jump zone!!!
5465 */
5466static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5467{
5468 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5469 AssertRCReturn(rc, rc);
5470 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5471 {
5472 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5473 AssertRCReturn(rc, rc);
5474
5475 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5476 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5477 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5478
5479 typedef enum
5480 {
5481 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5482 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5483 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5484 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5485 } VMXREFLECTXCPT;
5486
5487 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5488 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5489 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5490 {
5491 if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5492 {
5493 enmReflect = VMXREFLECTXCPT_XCPT;
5494#ifdef VBOX_STRICT
5495 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5496 && uExitVector == X86_XCPT_PF)
5497 {
5498 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5499 }
5500#endif
5501 if ( uExitVector == X86_XCPT_PF
5502 && uIdtVector == X86_XCPT_PF)
5503 {
5504 pVmxTransient->fVectoringPF = true;
5505 Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5506 }
5507 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5508 && hmR0VmxIsContributoryXcpt(uExitVector)
5509 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5510 || uIdtVector == X86_XCPT_PF))
5511 {
5512 enmReflect = VMXREFLECTXCPT_DF;
5513 }
5514 else if (uIdtVector == X86_XCPT_DF)
5515 enmReflect = VMXREFLECTXCPT_TF;
5516 }
5517 else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5518 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5519 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5520 {
5521 /*
5522 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5523 * (whatever they are) as they reoccur when restarting the instruction.
5524 */
5525 enmReflect = VMXREFLECTXCPT_XCPT;
5526 }
5527 }
5528 else
5529 {
5530 /*
5531 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5532 * interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5533 * original exception to the guest after handling the VM-exit.
5534 */
5535 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5536 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5537 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5538 {
5539 enmReflect = VMXREFLECTXCPT_XCPT;
5540 }
5541 }
5542
5543 switch (enmReflect)
5544 {
5545 case VMXREFLECTXCPT_XCPT:
5546 {
5547 Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5548 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5549 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5550
5551 uint32_t u32ErrCode = 0;
5552 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5553 {
5554 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5555 AssertRCReturn(rc, rc);
5556 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5557 }
5558
5559 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5560 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5561 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5562 rc = VINF_SUCCESS;
5563 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5564 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5565
5566 break;
5567 }
5568
5569 case VMXREFLECTXCPT_DF:
5570 {
5571 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5572 rc = VINF_HM_DOUBLE_FAULT;
5573 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5574 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5575
5576 break;
5577 }
5578
5579 case VMXREFLECTXCPT_TF:
5580 {
5581 rc = VINF_EM_RESET;
5582 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5583 uExitVector));
5584 break;
5585 }
5586
5587 default:
5588 Assert(rc == VINF_SUCCESS);
5589 break;
5590 }
5591 }
5592 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5593 return rc;
5594}
5595
5596
5597/**
5598 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5599 *
5600 * @returns VBox status code.
5601 * @param pVCpu Pointer to the VMCPU.
5602 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5603 * out-of-sync. Make sure to update the required fields
5604 * before using them.
5605 *
5606 * @remarks No-long-jump zone!!!
5607 */
5608static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5609{
5610 NOREF(pMixedCtx);
5611
5612 /*
5613 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5614 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5615 */
5616 VMMRZCallRing3Disable(pVCpu);
5617 HM_DISABLE_PREEMPT_IF_NEEDED();
5618
5619 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5620 {
5621 uint32_t uVal = 0;
5622 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5623 AssertRCReturn(rc, rc);
5624
5625 uint32_t uShadow = 0;
5626 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5627 AssertRCReturn(rc, rc);
5628
5629 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5630 CPUMSetGuestCR0(pVCpu, uVal);
5631 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5632 }
5633
5634 HM_RESTORE_PREEMPT_IF_NEEDED();
5635 VMMRZCallRing3Enable(pVCpu);
5636 return VINF_SUCCESS;
5637}
5638
5639
5640/**
5641 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5642 *
5643 * @returns VBox status code.
5644 * @param pVCpu Pointer to the VMCPU.
5645 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5646 * out-of-sync. Make sure to update the required fields
5647 * before using them.
5648 *
5649 * @remarks No-long-jump zone!!!
5650 */
5651static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5652{
5653 NOREF(pMixedCtx);
5654
5655 int rc = VINF_SUCCESS;
5656 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5657 {
5658 uint32_t uVal = 0;
5659 uint32_t uShadow = 0;
5660 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5661 AssertRCReturn(rc, rc);
5662 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5663 AssertRCReturn(rc, rc);
5664
5665 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5666 CPUMSetGuestCR4(pVCpu, uVal);
5667 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5668 }
5669 return rc;
5670}
5671
5672
5673/**
5674 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5675 *
5676 * @returns VBox status code.
5677 * @param pVCpu Pointer to the VMCPU.
5678 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5679 * out-of-sync. Make sure to update the required fields
5680 * before using them.
5681 *
5682 * @remarks No-long-jump zone!!!
5683 */
5684static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5685{
5686 int rc = VINF_SUCCESS;
5687 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
5688 {
5689 uint64_t u64Val = 0;
5690 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5691 AssertRCReturn(rc, rc);
5692
5693 pMixedCtx->rip = u64Val;
5694 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
5695 }
5696 return rc;
5697}
5698
5699
5700/**
5701 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
5702 *
5703 * @returns VBox status code.
5704 * @param pVCpu Pointer to the VMCPU.
5705 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5706 * out-of-sync. Make sure to update the required fields
5707 * before using them.
5708 *
5709 * @remarks No-long-jump zone!!!
5710 */
5711static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5712{
5713 int rc = VINF_SUCCESS;
5714 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
5715 {
5716 uint64_t u64Val = 0;
5717 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5718 AssertRCReturn(rc, rc);
5719
5720 pMixedCtx->rsp = u64Val;
5721 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
5722 }
5723 return rc;
5724}
5725
5726
5727/**
5728 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5729 *
5730 * @returns VBox status code.
5731 * @param pVCpu Pointer to the VMCPU.
5732 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5733 * out-of-sync. Make sure to update the required fields
5734 * before using them.
5735 *
5736 * @remarks No-long-jump zone!!!
5737 */
5738static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5739{
5740 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
5741 {
5742 uint32_t uVal = 0;
5743 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5744 AssertRCReturn(rc, rc);
5745
5746 pMixedCtx->eflags.u32 = uVal;
5747 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5748 {
5749 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5750 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5751
5752 pMixedCtx->eflags.Bits.u1VM = 0;
5753 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5754 }
5755
5756 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
5757 }
5758 return VINF_SUCCESS;
5759}
5760
5761
5762/**
5763 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5764 * guest-CPU context.
5765 */
5766DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5767{
5768 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5769 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5770 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5771 return rc;
5772}
5773
5774
5775/**
5776 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5777 * from the guest-state area in the VMCS.
5778 *
5779 * @param pVCpu Pointer to the VMCPU.
5780 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5781 * out-of-sync. Make sure to update the required fields
5782 * before using them.
5783 *
5784 * @remarks No-long-jump zone!!!
5785 */
5786static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5787{
5788 uint32_t uIntrState = 0;
5789 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5790 AssertRC(rc);
5791
5792 if (!uIntrState)
5793 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5794 else
5795 {
5796 Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5797 || uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5798 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5799 AssertRC(rc);
5800 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5801 AssertRC(rc);
5802
5803 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5804 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5805 }
5806}
5807
5808
5809/**
5810 * Saves the guest's activity state.
5811 *
5812 * @returns VBox status code.
5813 * @param pVCpu Pointer to the VMCPU.
5814 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5815 * out-of-sync. Make sure to update the required fields
5816 * before using them.
5817 *
5818 * @remarks No-long-jump zone!!!
5819 */
5820static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5821{
5822 NOREF(pMixedCtx);
5823 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5824 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
5825 return VINF_SUCCESS;
5826}
5827
5828
5829/**
5830 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5831 * the current VMCS into the guest-CPU context.
5832 *
5833 * @returns VBox status code.
5834 * @param pVCpu Pointer to the VMCPU.
5835 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5836 * out-of-sync. Make sure to update the required fields
5837 * before using them.
5838 *
5839 * @remarks No-long-jump zone!!!
5840 */
5841static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5842{
5843 int rc = VINF_SUCCESS;
5844 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5845 {
5846 uint32_t u32Val = 0;
5847 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5848 pMixedCtx->SysEnter.cs = u32Val;
5849 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
5850 }
5851
5852 uint64_t u64Val = 0;
5853 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5854 {
5855 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5856 pMixedCtx->SysEnter.eip = u64Val;
5857 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
5858 }
5859 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5860 {
5861 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5862 pMixedCtx->SysEnter.esp = u64Val;
5863 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
5864 }
5865 return rc;
5866}
5867
5868
5869/**
5870 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
5871 * the CPU back into the guest-CPU context.
5872 *
5873 * @returns VBox status code.
5874 * @param pVCpu Pointer to the VMCPU.
5875 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5876 * out-of-sync. Make sure to update the required fields
5877 * before using them.
5878 *
5879 * @remarks No-long-jump zone!!!
5880 */
5881static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5882{
5883#if HC_ARCH_BITS == 64
5884 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
5885 {
5886 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
5887 VMMRZCallRing3Disable(pVCpu);
5888 HM_DISABLE_PREEMPT_IF_NEEDED();
5889
5890 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
5891 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
5892 {
5893 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
5894 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
5895 }
5896
5897 HM_RESTORE_PREEMPT_IF_NEEDED();
5898 VMMRZCallRing3Enable(pVCpu);
5899 }
5900 else
5901 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
5902#else
5903 NOREF(pMixedCtx);
5904 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
5905#endif
5906
5907 return VINF_SUCCESS;
5908}
5909
5910
5911/**
5912 * Saves the auto load/store'd guest MSRs from the current VMCS into
5913 * the guest-CPU context.
5914 *
5915 * @returns VBox status code.
5916 * @param pVCpu Pointer to the VMCPU.
5917 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5918 * out-of-sync. Make sure to update the required fields
5919 * before using them.
5920 *
5921 * @remarks No-long-jump zone!!!
5922 */
5923static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5924{
5925 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
5926 return VINF_SUCCESS;
5927
5928 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5929 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
5930 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
5931 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
5932 {
5933 switch (pMsr->u32Msr)
5934 {
5935 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
5936 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5937 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5938 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5939 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5940 default:
5941 {
5942 AssertFailed();
5943 return VERR_HM_UNEXPECTED_LD_ST_MSR;
5944 }
5945 }
5946 }
5947
5948 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
5949 return VINF_SUCCESS;
5950}
5951
5952
5953/**
5954 * Saves the guest control registers from the current VMCS into the guest-CPU
5955 * context.
5956 *
5957 * @returns VBox status code.
5958 * @param pVCpu Pointer to the VMCPU.
5959 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5960 * out-of-sync. Make sure to update the required fields
5961 * before using them.
5962 *
5963 * @remarks No-long-jump zone!!!
5964 */
5965static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5966{
5967 /* Guest CR0. Guest FPU. */
5968 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5969 AssertRCReturn(rc, rc);
5970
5971 /* Guest CR4. */
5972 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5973 AssertRCReturn(rc, rc);
5974
5975 /* Guest CR2 - updated always during the world-switch or in #PF. */
5976 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5977 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
5978 {
5979 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
5980 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
5981
5982 PVM pVM = pVCpu->CTX_SUFF(pVM);
5983 if ( pVM->hm.s.vmx.fUnrestrictedGuest
5984 || ( pVM->hm.s.fNestedPaging
5985 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5986 {
5987 uint64_t u64Val = 0;
5988 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5989 if (pMixedCtx->cr3 != u64Val)
5990 {
5991 CPUMSetGuestCR3(pVCpu, u64Val);
5992 if (VMMRZCallRing3IsEnabled(pVCpu))
5993 {
5994 PGMUpdateCR3(pVCpu, u64Val);
5995 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5996 }
5997 else
5998 {
5999 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6000 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6001 }
6002 }
6003
6004 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6005 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6006 {
6007 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
6008 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
6009 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
6010 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
6011
6012 if (VMMRZCallRing3IsEnabled(pVCpu))
6013 {
6014 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6015 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6016 }
6017 else
6018 {
6019 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6020 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6021 }
6022 }
6023 }
6024
6025 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6026 }
6027
6028 /*
6029 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6030 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6031 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6032 *
6033 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6034 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6035 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6036 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6037 *
6038 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6039 */
6040 if (VMMRZCallRing3IsEnabled(pVCpu))
6041 {
6042 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6043 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6044
6045 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6046 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6047
6048 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6049 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6050 }
6051
6052 return rc;
6053}
6054
6055
6056/**
6057 * Reads a guest segment register from the current VMCS into the guest-CPU
6058 * context.
6059 *
6060 * @returns VBox status code.
6061 * @param pVCpu Pointer to the VMCPU.
6062 * @param idxSel Index of the selector in the VMCS.
6063 * @param idxLimit Index of the segment limit in the VMCS.
6064 * @param idxBase Index of the segment base in the VMCS.
6065 * @param idxAccess Index of the access rights of the segment in the VMCS.
6066 * @param pSelReg Pointer to the segment selector.
6067 *
6068 * @remarks No-long-jump zone!!!
6069 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6070 * macro as that takes care of whether to read from the VMCS cache or
6071 * not.
6072 */
6073DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6074 PCPUMSELREG pSelReg)
6075{
6076 NOREF(pVCpu);
6077
6078 uint32_t u32Val = 0;
6079 int rc = VMXReadVmcs32(idxSel, &u32Val);
6080 AssertRCReturn(rc, rc);
6081 pSelReg->Sel = (uint16_t)u32Val;
6082 pSelReg->ValidSel = (uint16_t)u32Val;
6083 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6084
6085 rc = VMXReadVmcs32(idxLimit, &u32Val);
6086 AssertRCReturn(rc, rc);
6087 pSelReg->u32Limit = u32Val;
6088
6089 uint64_t u64Val = 0;
6090 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6091 AssertRCReturn(rc, rc);
6092 pSelReg->u64Base = u64Val;
6093
6094 rc = VMXReadVmcs32(idxAccess, &u32Val);
6095 AssertRCReturn(rc, rc);
6096 pSelReg->Attr.u = u32Val;
6097
6098 /*
6099 * If VT-x marks the segment as unusable, most other bits remain undefined:
6100 * - For CS the L, D and G bits have meaning.
6101 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6102 * - For the remaining data segments no bits are defined.
6103 *
6104 * The present bit and the unusable bit has been observed to be set at the
6105 * same time (the selector was supposed to invalid as we started executing
6106 * a V8086 interrupt in ring-0).
6107 *
6108 * What should be important for the rest of the VBox code, is that the P bit is
6109 * cleared. Some of the other VBox code recognizes the unusable bit, but
6110 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6111 * safe side here, we'll strip off P and other bits we don't care about. If
6112 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6113 *
6114 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6115 */
6116 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6117 {
6118 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6119
6120 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6121 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6122 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6123
6124 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6125#ifdef DEBUG_bird
6126 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6127 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6128 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6129#endif
6130 }
6131 return VINF_SUCCESS;
6132}
6133
6134
6135#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6136# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6137 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6138 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6139#else
6140# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6141 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6142 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6143#endif
6144
6145
6146/**
6147 * Saves the guest segment registers from the current VMCS into the guest-CPU
6148 * context.
6149 *
6150 * @returns VBox status code.
6151 * @param pVCpu Pointer to the VMCPU.
6152 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6153 * out-of-sync. Make sure to update the required fields
6154 * before using them.
6155 *
6156 * @remarks No-long-jump zone!!!
6157 */
6158static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6159{
6160 /* Guest segment registers. */
6161 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6162 {
6163 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6164 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6165 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6166 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6167 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6168 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6169 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6170
6171 /* Restore segment attributes for real-on-v86 mode hack. */
6172 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6173 {
6174 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6175 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6176 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6177 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6178 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6179 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6180 }
6181 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6182 }
6183
6184 return VINF_SUCCESS;
6185}
6186
6187
6188/**
6189 * Saves the guest descriptor table registers and task register from the current
6190 * VMCS into the guest-CPU context.
6191 *
6192 * @returns VBox status code.
6193 * @param pVCpu Pointer to the VMCPU.
6194 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6195 * out-of-sync. Make sure to update the required fields
6196 * before using them.
6197 *
6198 * @remarks No-long-jump zone!!!
6199 */
6200static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6201{
6202 int rc = VINF_SUCCESS;
6203
6204 /* Guest LDTR. */
6205 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6206 {
6207 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6208 AssertRCReturn(rc, rc);
6209 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6210 }
6211
6212 /* Guest GDTR. */
6213 uint64_t u64Val = 0;
6214 uint32_t u32Val = 0;
6215 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6216 {
6217 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6218 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6219 pMixedCtx->gdtr.pGdt = u64Val;
6220 pMixedCtx->gdtr.cbGdt = u32Val;
6221 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6222 }
6223
6224 /* Guest IDTR. */
6225 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6226 {
6227 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6228 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6229 pMixedCtx->idtr.pIdt = u64Val;
6230 pMixedCtx->idtr.cbIdt = u32Val;
6231 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6232 }
6233
6234 /* Guest TR. */
6235 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6236 {
6237 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6238 AssertRCReturn(rc, rc);
6239
6240 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6241 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6242 {
6243 rc = VMXLOCAL_READ_SEG(TR, tr);
6244 AssertRCReturn(rc, rc);
6245 }
6246 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6247 }
6248 return rc;
6249}
6250
6251#undef VMXLOCAL_READ_SEG
6252
6253
6254/**
6255 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6256 * context.
6257 *
6258 * @returns VBox status code.
6259 * @param pVCpu Pointer to the VMCPU.
6260 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6261 * out-of-sync. Make sure to update the required fields
6262 * before using them.
6263 *
6264 * @remarks No-long-jump zone!!!
6265 */
6266static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6267{
6268 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6269 {
6270 if (!pVCpu->hm.s.fUsingHyperDR7)
6271 {
6272 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6273 uint32_t u32Val;
6274 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6275 pMixedCtx->dr[7] = u32Val;
6276 }
6277
6278 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6279 }
6280 return VINF_SUCCESS;
6281}
6282
6283
6284/**
6285 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6286 *
6287 * @returns VBox status code.
6288 * @param pVCpu Pointer to the VMCPU.
6289 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6290 * out-of-sync. Make sure to update the required fields
6291 * before using them.
6292 *
6293 * @remarks No-long-jump zone!!!
6294 */
6295static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6296{
6297 NOREF(pMixedCtx);
6298
6299 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6300 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6301 return VINF_SUCCESS;
6302}
6303
6304
6305/**
6306 * Saves the entire guest state from the currently active VMCS into the
6307 * guest-CPU context. This essentially VMREADs all guest-data.
6308 *
6309 * @returns VBox status code.
6310 * @param pVCpu Pointer to the VMCPU.
6311 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6312 * out-of-sync. Make sure to update the required fields
6313 * before using them.
6314 */
6315static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6316{
6317 Assert(pVCpu);
6318 Assert(pMixedCtx);
6319
6320 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6321 return VINF_SUCCESS;
6322
6323 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6324 again on the ring-3 callback path, there is no real need to. */
6325 if (VMMRZCallRing3IsEnabled(pVCpu))
6326 VMMR0LogFlushDisable(pVCpu);
6327 else
6328 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6329 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6330
6331 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6332 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6333
6334 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6335 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6336
6337 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6338 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6339
6340 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6341 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6342
6343 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6344 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6345
6346 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6347 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6348
6349 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6350 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6351
6352 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6353 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6354
6355 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6356 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6357
6358 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6359 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6360
6361 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6362 ("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6363
6364 if (VMMRZCallRing3IsEnabled(pVCpu))
6365 VMMR0LogFlushEnable(pVCpu);
6366
6367 return rc;
6368}
6369
6370
6371/**
6372 * Check per-VM and per-VCPU force flag actions that require us to go back to
6373 * ring-3 for one reason or another.
6374 *
6375 * @returns VBox status code (information status code included).
6376 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6377 * ring-3.
6378 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6379 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6380 * interrupts)
6381 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6382 * all EMTs to be in ring-3.
6383 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6384 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6385 * to the EM loop.
6386 *
6387 * @param pVM Pointer to the VM.
6388 * @param pVCpu Pointer to the VMCPU.
6389 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6390 * out-of-sync. Make sure to update the required fields
6391 * before using them.
6392 */
6393static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6394{
6395 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6396
6397 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6398 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6399 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6400 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6401 {
6402 /* We need the control registers now, make sure the guest-CPU context is updated. */
6403 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6404 AssertRCReturn(rc3, rc3);
6405
6406 /* Pending HM CR3 sync. */
6407 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6408 {
6409 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6410 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6411 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6412 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6413 }
6414
6415 /* Pending HM PAE PDPEs. */
6416 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6417 {
6418 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6419 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6420 }
6421
6422 /* Pending PGM C3 sync. */
6423 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6424 {
6425 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6426 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6427 if (rc2 != VINF_SUCCESS)
6428 {
6429 AssertRC(rc2);
6430 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6431 return rc2;
6432 }
6433 }
6434
6435 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6436 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6437 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6438 {
6439 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6440 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6441 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6442 return rc2;
6443 }
6444
6445 /* Pending VM request packets, such as hardware interrupts. */
6446 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6447 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6448 {
6449 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6450 return VINF_EM_PENDING_REQUEST;
6451 }
6452
6453 /* Pending PGM pool flushes. */
6454 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6455 {
6456 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6457 return VINF_PGM_POOL_FLUSH_PENDING;
6458 }
6459
6460 /* Pending DMA requests. */
6461 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6462 {
6463 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6464 return VINF_EM_RAW_TO_R3;
6465 }
6466 }
6467
6468 return VINF_SUCCESS;
6469}
6470
6471
6472/**
6473 * Converts any TRPM trap into a pending HM event. This is typically used when
6474 * entering from ring-3 (not longjmp returns).
6475 *
6476 * @param pVCpu Pointer to the VMCPU.
6477 */
6478static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6479{
6480 Assert(TRPMHasTrap(pVCpu));
6481 Assert(!pVCpu->hm.s.Event.fPending);
6482
6483 uint8_t uVector;
6484 TRPMEVENT enmTrpmEvent;
6485 RTGCUINT uErrCode;
6486 RTGCUINTPTR GCPtrFaultAddress;
6487 uint8_t cbInstr;
6488
6489 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6490 AssertRC(rc);
6491
6492 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6493 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6494 if (enmTrpmEvent == TRPM_TRAP)
6495 {
6496 switch (uVector)
6497 {
6498 case X86_XCPT_BP:
6499 case X86_XCPT_OF:
6500 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6501 break;
6502
6503 case X86_XCPT_PF:
6504 case X86_XCPT_DF:
6505 case X86_XCPT_TS:
6506 case X86_XCPT_NP:
6507 case X86_XCPT_SS:
6508 case X86_XCPT_GP:
6509 case X86_XCPT_AC:
6510 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6511 /* no break! */
6512 default:
6513 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6514 break;
6515 }
6516 }
6517 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6518 {
6519 if (uVector == X86_XCPT_NMI)
6520 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6521 else
6522 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6523 }
6524 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6525 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6526 else
6527 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6528
6529 rc = TRPMResetTrap(pVCpu);
6530 AssertRC(rc);
6531 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6532 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6533
6534 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6535 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6536}
6537
6538
6539/**
6540 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6541 * VT-x to execute any instruction.
6542 *
6543 * @param pvCpu Pointer to the VMCPU.
6544 */
6545static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6546{
6547 Assert(pVCpu->hm.s.Event.fPending);
6548
6549 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6550 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6551 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6552 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6553
6554 /* If a trap was already pending, we did something wrong! */
6555 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6556
6557 TRPMEVENT enmTrapType;
6558 switch (uVectorType)
6559 {
6560 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6561 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6562 enmTrapType = TRPM_HARDWARE_INT;
6563 break;
6564
6565 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6566 enmTrapType = TRPM_SOFTWARE_INT;
6567 break;
6568
6569 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6570 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6571 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6572 enmTrapType = TRPM_TRAP;
6573 break;
6574
6575 default:
6576 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6577 enmTrapType = TRPM_32BIT_HACK;
6578 break;
6579 }
6580
6581 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6582
6583 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6584 AssertRC(rc);
6585
6586 if (fErrorCodeValid)
6587 TRPMSetErrorCode(pVCpu, uErrorCode);
6588
6589 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6590 && uVector == X86_XCPT_PF)
6591 {
6592 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6593 }
6594 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6595 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6596 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6597 {
6598 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6599 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6600 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6601 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6602 }
6603 pVCpu->hm.s.Event.fPending = false;
6604}
6605
6606
6607/**
6608 * Does the necessary state syncing before returning to ring-3 for any reason
6609 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6610 *
6611 * @returns VBox status code.
6612 * @param pVM Pointer to the VM.
6613 * @param pVCpu Pointer to the VMCPU.
6614 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6615 * be out-of-sync. Make sure to update the required
6616 * fields before using them.
6617 * @param fSaveGuestState Whether to save the guest state or not.
6618 *
6619 * @remarks No-long-jmp zone!!!
6620 */
6621static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6622{
6623 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6624 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6625
6626 RTCPUID idCpu = RTMpCpuId();
6627 Log4Func(("HostCpuId=%u\n", idCpu));
6628
6629 /*
6630 * !!! IMPORTANT !!!
6631 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6632 */
6633
6634 /* Save the guest state if necessary. */
6635 if ( fSaveGuestState
6636 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6637 {
6638 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6639 AssertRCReturn(rc, rc);
6640 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6641 }
6642
6643 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6644 if (CPUMIsGuestFPUStateActive(pVCpu))
6645 {
6646 /* We shouldn't reload CR0 without saving it first. */
6647 if (!fSaveGuestState)
6648 {
6649 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6650 AssertRCReturn(rc, rc);
6651 }
6652 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6653 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6654 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6655 }
6656
6657 /* Restore host debug registers if necessary and resync on next R0 reentry. */
6658#ifdef VBOX_STRICT
6659 if (CPUMIsHyperDebugStateActive(pVCpu))
6660 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6661#endif
6662 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6663 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6664 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6665 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6666
6667#if HC_ARCH_BITS == 64
6668 /* Restore host-state bits that VT-x only restores partially. */
6669 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6670 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6671 {
6672 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6673 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6674 }
6675 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6676#endif
6677
6678#if HC_ARCH_BITS == 64
6679 /* Restore the host MSRs as we're leaving VT-x context. */
6680 if ( pVM->hm.s.fAllow64BitGuests
6681 && pVCpu->hm.s.vmx.fRestoreHostMsrs)
6682 {
6683 /* We shouldn't reload the guest MSRs without saving it first. */
6684 if (!fSaveGuestState)
6685 {
6686 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6687 AssertRCReturn(rc, rc);
6688 }
6689 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
6690 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6691 Assert(!pVCpu->hm.s.vmx.fRestoreHostMsrs);
6692 }
6693#endif
6694
6695 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6696 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6697
6698 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6699 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6700 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6701 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6702 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6703 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6704 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6705 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6706
6707 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6708
6709 /** @todo This partially defeats the purpose of having preemption hooks.
6710 * The problem is, deregistering the hooks should be moved to a place that
6711 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6712 * context.
6713 */
6714 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6715 {
6716 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6717 AssertRCReturn(rc, rc);
6718
6719 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6720 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6721 }
6722 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6723 NOREF(idCpu);
6724
6725 return VINF_SUCCESS;
6726}
6727
6728
6729/**
6730 * Leaves the VT-x session.
6731 *
6732 * @returns VBox status code.
6733 * @param pVM Pointer to the VM.
6734 * @param pVCpu Pointer to the VMCPU.
6735 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6736 * out-of-sync. Make sure to update the required fields
6737 * before using them.
6738 *
6739 * @remarks No-long-jmp zone!!!
6740 */
6741DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6742{
6743 HM_DISABLE_PREEMPT_IF_NEEDED();
6744 HMVMX_ASSERT_CPU_SAFE();
6745 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6746 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6747
6748 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6749 and done this from the VMXR0ThreadCtxCallback(). */
6750 if (!pVCpu->hm.s.fLeaveDone)
6751 {
6752 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6753 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6754 pVCpu->hm.s.fLeaveDone = true;
6755 }
6756
6757 /*
6758 * !!! IMPORTANT !!!
6759 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6760 */
6761
6762 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6763 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
6764 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6765 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6766 VMMR0ThreadCtxHooksDeregister(pVCpu);
6767
6768 /* Leave HM context. This takes care of local init (term). */
6769 int rc = HMR0LeaveCpu(pVCpu);
6770
6771 HM_RESTORE_PREEMPT_IF_NEEDED();
6772
6773 return rc;
6774}
6775
6776
6777/**
6778 * Does the necessary state syncing before doing a longjmp to ring-3.
6779 *
6780 * @returns VBox status code.
6781 * @param pVM Pointer to the VM.
6782 * @param pVCpu Pointer to the VMCPU.
6783 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6784 * out-of-sync. Make sure to update the required fields
6785 * before using them.
6786 *
6787 * @remarks No-long-jmp zone!!!
6788 */
6789DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6790{
6791 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6792}
6793
6794
6795/**
6796 * Take necessary actions before going back to ring-3.
6797 *
6798 * An action requires us to go back to ring-3. This function does the necessary
6799 * steps before we can safely return to ring-3. This is not the same as longjmps
6800 * to ring-3, this is voluntary and prepares the guest so it may continue
6801 * executing outside HM (recompiler/IEM).
6802 *
6803 * @returns VBox status code.
6804 * @param pVM Pointer to the VM.
6805 * @param pVCpu Pointer to the VMCPU.
6806 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6807 * out-of-sync. Make sure to update the required fields
6808 * before using them.
6809 * @param rcExit The reason for exiting to ring-3. Can be
6810 * VINF_VMM_UNKNOWN_RING3_CALL.
6811 */
6812static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6813{
6814 Assert(pVM);
6815 Assert(pVCpu);
6816 Assert(pMixedCtx);
6817 HMVMX_ASSERT_PREEMPT_SAFE();
6818
6819 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6820 {
6821 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6822 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6823 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6824 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6825 }
6826
6827 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6828 VMMRZCallRing3Disable(pVCpu);
6829 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6830
6831 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6832 if (pVCpu->hm.s.Event.fPending)
6833 {
6834 hmR0VmxPendingEventToTrpmTrap(pVCpu);
6835 Assert(!pVCpu->hm.s.Event.fPending);
6836 }
6837
6838 /* Save guest state and restore host state bits. */
6839 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6840 AssertRCReturn(rc, rc);
6841 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6842
6843 /* Sync recompiler state. */
6844 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6845 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6846 | CPUM_CHANGED_LDTR
6847 | CPUM_CHANGED_GDTR
6848 | CPUM_CHANGED_IDTR
6849 | CPUM_CHANGED_TR
6850 | CPUM_CHANGED_HIDDEN_SEL_REGS);
6851 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6852 if ( pVM->hm.s.fNestedPaging
6853 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
6854 {
6855 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6856 }
6857
6858 Assert(!pVCpu->hm.s.fClearTrapFlag);
6859
6860 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6861 if (rcExit != VINF_EM_RAW_INTERRUPT)
6862 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6863
6864 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6865
6866 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6867 VMMRZCallRing3RemoveNotification(pVCpu);
6868 VMMRZCallRing3Enable(pVCpu);
6869
6870 return rc;
6871}
6872
6873
6874/**
6875 * VMMRZCallRing3() callback wrapper which saves the guest state before we
6876 * longjump to ring-3 and possibly get preempted.
6877 *
6878 * @returns VBox status code.
6879 * @param pVCpu Pointer to the VMCPU.
6880 * @param enmOperation The operation causing the ring-3 longjump.
6881 * @param pvUser Opaque pointer to the guest-CPU context. The data
6882 * may be out-of-sync. Make sure to update the required
6883 * fields before using them.
6884 */
6885DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6886{
6887 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
6888 {
6889 /*
6890 * !!! IMPORTANT !!!
6891 * If you modify code here, make sure to check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs
6892 * to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
6893 */
6894 VMMRZCallRing3RemoveNotification(pVCpu);
6895 VMMRZCallRing3Disable(pVCpu);
6896 HM_DISABLE_PREEMPT_IF_NEEDED();
6897
6898 PVM pVM = pVCpu->CTX_SUFF(pVM);
6899 if (CPUMIsGuestFPUStateActive(pVCpu))
6900 CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
6901
6902 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6903
6904#if HC_ARCH_BITS == 64
6905 /* Restore host-state bits that VT-x only restores partially. */
6906 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6907 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6908 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6909 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6910
6911 /* Restore the host MSRs as we're leaving VT-x context. */
6912 if ( pVM->hm.s.fAllow64BitGuests
6913 && pVCpu->hm.s.vmx.fRestoreHostMsrs)
6914 {
6915 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6916 }
6917#endif
6918 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6919 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6920 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6921 {
6922 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6923 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6924 }
6925
6926 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6927 VMMR0ThreadCtxHooksDeregister(pVCpu);
6928
6929 HMR0LeaveCpu(pVCpu);
6930 HM_RESTORE_PREEMPT_IF_NEEDED();
6931 return VINF_SUCCESS;
6932 }
6933
6934 Assert(pVCpu);
6935 Assert(pvUser);
6936 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6937 HMVMX_ASSERT_PREEMPT_SAFE();
6938
6939 VMMRZCallRing3Disable(pVCpu);
6940 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6941
6942 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
6943 enmOperation));
6944
6945 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6946 AssertRCReturn(rc, rc);
6947
6948 VMMRZCallRing3Enable(pVCpu);
6949 return VINF_SUCCESS;
6950}
6951
6952
6953/**
6954 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6955 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
6956 *
6957 * @param pVCpu Pointer to the VMCPU.
6958 */
6959DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6960{
6961 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6962 {
6963 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6964 {
6965 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6966 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6967 AssertRC(rc);
6968 Log4(("Setup interrupt-window exiting\n"));
6969 }
6970 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6971}
6972
6973
6974/**
6975 * Clears the interrupt-window exiting control in the VMCS.
6976 *
6977 * @param pVCpu Pointer to the VMCPU.
6978 */
6979DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
6980{
6981 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
6982 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6983 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6984 AssertRC(rc);
6985 Log4(("Cleared interrupt-window exiting\n"));
6986}
6987
6988
6989/**
6990 * Evaluates the event to be delivered to the guest and sets it as the pending
6991 * event.
6992 *
6993 * @param pVCpu Pointer to the VMCPU.
6994 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6995 * out-of-sync. Make sure to update the required fields
6996 * before using them.
6997 */
6998static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6999{
7000 Assert(!pVCpu->hm.s.Event.fPending);
7001
7002 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7003 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7004 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7005 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7006
7007 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7008 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
7009 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
7010 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7011 Assert(!TRPMHasTrap(pVCpu));
7012
7013 /** @todo SMI. SMIs take priority over NMIs. */
7014 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
7015 {
7016 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7017 if ( !fBlockMovSS
7018 && !fBlockSti)
7019 {
7020 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7021 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7022 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7023 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7024
7025 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddres */);
7026 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7027 }
7028 else
7029 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7030 }
7031 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7032 && !pVCpu->hm.s.fSingleInstruction)
7033 {
7034 /*
7035 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7036 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
7037 * evaluated here and not set as pending, solely based on the force-flags.
7038 */
7039 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7040 AssertRC(rc);
7041 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7042 if ( !fBlockInt
7043 && !fBlockSti
7044 && !fBlockMovSS)
7045 {
7046 uint8_t u8Interrupt;
7047 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7048 if (RT_SUCCESS(rc))
7049 {
7050 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7051 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7052 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7053
7054 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7055 }
7056 else
7057 {
7058 /** @todo Does this actually happen? If not turn it into an assertion. */
7059 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
7060 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7061 }
7062 }
7063 else
7064 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7065 }
7066}
7067
7068
7069/**
7070 * Sets a pending-debug exception to be delivered to the guest if the guest is
7071 * single-stepping.
7072 *
7073 * @param pVCpu Pointer to the VMCPU.
7074 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7075 * out-of-sync. Make sure to update the required fields
7076 * before using them.
7077 */
7078DECLINLINE(void) hmR0VmxSetPendingDebugXcpt(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7079{
7080 HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
7081 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7082 {
7083 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7084 AssertRC(rc);
7085 }
7086}
7087
7088
7089/**
7090 * Injects any pending events into the guest if the guest is in a state to
7091 * receive them.
7092 *
7093 * @returns VBox status code (informational status codes included).
7094 * @param pVCpu Pointer to the VMCPU.
7095 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7096 * out-of-sync. Make sure to update the required fields
7097 * before using them.
7098 */
7099static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7100{
7101 HMVMX_ASSERT_PREEMPT_SAFE();
7102 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7103
7104 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7105 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7106 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7107 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7108
7109 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7110 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
7111 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
7112 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7113 Assert(!TRPMHasTrap(pVCpu));
7114
7115 int rc = VINF_SUCCESS;
7116 if (pVCpu->hm.s.Event.fPending)
7117 {
7118 /*
7119 * Clear any interrupt-window exiting control if we're going to inject an interrupt. Saves one extra
7120 * VM-exit in situations where we previously setup interrupt-window exiting but got other VM-exits and
7121 * ended up enabling interrupts outside VT-x.
7122 */
7123 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7124 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7125 && ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT
7126 || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI))
7127 {
7128 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7129 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7130 }
7131#if defined(VBOX_STRICT) || defined(VBOX_WITH_STATISTICS)
7132 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7133 {
7134 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7135 AssertRCReturn(rc, rc);
7136 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7137 Assert(!fBlockInt);
7138 Assert(!fBlockSti);
7139 Assert(!fBlockMovSS);
7140 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT));
7141 }
7142 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7143 {
7144 Assert(!fBlockSti);
7145 Assert(!fBlockMovSS);
7146 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT));
7147 }
7148#endif
7149 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7150 (uint8_t)uIntType));
7151 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7152 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
7153 AssertRCReturn(rc, rc);
7154
7155 /* Update the interruptibility-state as it could have been changed by
7156 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7157 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7158 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7159
7160#ifdef VBOX_WITH_STATISTICS
7161 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7162 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7163 else
7164 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7165#endif
7166 }
7167
7168 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7169 if ( fBlockSti
7170 || fBlockMovSS)
7171 {
7172 if ( !pVCpu->hm.s.fSingleInstruction
7173 && !DBGFIsStepping(pVCpu))
7174 {
7175 /*
7176 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7177 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7178 * See Intel spec. 27.3.4 "Saving Non-Register State".
7179 */
7180 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7181 AssertRCReturn(rc2, rc2);
7182 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
7183 }
7184 else if (pMixedCtx->eflags.Bits.u1TF)
7185 {
7186 /*
7187 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7188 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7189 */
7190 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7191 uIntrState = 0;
7192 }
7193 }
7194
7195 /*
7196 * There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
7197 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7198 */
7199 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7200 AssertRC(rc2);
7201
7202 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7203 NOREF(fBlockMovSS); NOREF(fBlockSti);
7204 return rc;
7205}
7206
7207
7208/**
7209 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7210 *
7211 * @param pVCpu Pointer to the VMCPU.
7212 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7213 * out-of-sync. Make sure to update the required fields
7214 * before using them.
7215 */
7216DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7217{
7218 NOREF(pMixedCtx);
7219 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7220 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7221}
7222
7223
7224/**
7225 * Injects a double-fault (#DF) exception into the VM.
7226 *
7227 * @returns VBox status code (informational status code included).
7228 * @param pVCpu Pointer to the VMCPU.
7229 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7230 * out-of-sync. Make sure to update the required fields
7231 * before using them.
7232 */
7233DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
7234{
7235 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7236 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7237 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7238 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7239 puIntrState);
7240}
7241
7242
7243/**
7244 * Sets a debug (#DB) exception as pending-for-injection into the VM.
7245 *
7246 * @param pVCpu Pointer to the VMCPU.
7247 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7248 * out-of-sync. Make sure to update the required fields
7249 * before using them.
7250 */
7251DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7252{
7253 NOREF(pMixedCtx);
7254 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7255 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7256 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7257}
7258
7259
7260/**
7261 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
7262 *
7263 * @param pVCpu Pointer to the VMCPU.
7264 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7265 * out-of-sync. Make sure to update the required fields
7266 * before using them.
7267 * @param cbInstr The value of RIP that is to be pushed on the guest
7268 * stack.
7269 */
7270DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7271{
7272 NOREF(pMixedCtx);
7273 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7274 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7275 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7276}
7277
7278
7279/**
7280 * Injects a general-protection (#GP) fault into the VM.
7281 *
7282 * @returns VBox status code (informational status code included).
7283 * @param pVCpu Pointer to the VMCPU.
7284 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7285 * out-of-sync. Make sure to update the required fields
7286 * before using them.
7287 * @param u32ErrorCode The error code associated with the #GP.
7288 */
7289DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7290 uint32_t *puIntrState)
7291{
7292 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7293 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7294 if (fErrorCodeValid)
7295 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7296 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7297 puIntrState);
7298}
7299
7300
7301/**
7302 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7303 *
7304 * @param pVCpu Pointer to the VMCPU.
7305 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7306 * out-of-sync. Make sure to update the required fields
7307 * before using them.
7308 * @param uVector The software interrupt vector number.
7309 * @param cbInstr The value of RIP that is to be pushed on the guest
7310 * stack.
7311 */
7312DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7313{
7314 NOREF(pMixedCtx);
7315 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7316 if ( uVector == X86_XCPT_BP
7317 || uVector == X86_XCPT_OF)
7318 {
7319 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7320 }
7321 else
7322 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7323 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7324}
7325
7326
7327/**
7328 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7329 * stack.
7330 *
7331 * @returns VBox status code (information status code included).
7332 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7333 * @param pVM Pointer to the VM.
7334 * @param pMixedCtx Pointer to the guest-CPU context.
7335 * @param uValue The value to push to the guest stack.
7336 */
7337DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7338{
7339 /*
7340 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7341 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7342 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7343 */
7344 if (pMixedCtx->sp == 1)
7345 return VINF_EM_RESET;
7346 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7347 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7348 AssertRCReturn(rc, rc);
7349 return rc;
7350}
7351
7352
7353/**
7354 * Injects an event into the guest upon VM-entry by updating the relevant fields
7355 * in the VM-entry area in the VMCS.
7356 *
7357 * @returns VBox status code (informational error codes included).
7358 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7359 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7360 *
7361 * @param pVCpu Pointer to the VMCPU.
7362 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7363 * be out-of-sync. Make sure to update the required
7364 * fields before using them.
7365 * @param u64IntInfo The VM-entry interruption-information field.
7366 * @param cbInstr The VM-entry instruction length in bytes (for
7367 * software interrupts, exceptions and privileged
7368 * software exceptions).
7369 * @param u32ErrCode The VM-entry exception error code.
7370 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7371 * @param puIntrState Pointer to the current guest interruptibility-state.
7372 * This interruptibility-state will be updated if
7373 * necessary. This cannot not be NULL.
7374 *
7375 * @remarks Requires CR0!
7376 * @remarks No-long-jump zone!!!
7377 */
7378static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7379 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7380{
7381 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7382 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7383 Assert(puIntrState);
7384 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7385
7386 const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7387 const uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7388
7389#ifdef VBOX_STRICT
7390 /* Validate the error-code-valid bit for hardware exceptions. */
7391 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7392 {
7393 switch (uVector)
7394 {
7395 case X86_XCPT_PF:
7396 case X86_XCPT_DF:
7397 case X86_XCPT_TS:
7398 case X86_XCPT_NP:
7399 case X86_XCPT_SS:
7400 case X86_XCPT_GP:
7401 case X86_XCPT_AC:
7402 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7403 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7404 /* fallthru */
7405 default:
7406 break;
7407 }
7408 }
7409#endif
7410
7411 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7412 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7413 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7414
7415 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7416
7417 /* We require CR0 to check if the guest is in real-mode. */
7418 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7419 AssertRCReturn(rc, rc);
7420
7421 /*
7422 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7423 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7424 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7425 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7426 */
7427 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7428 {
7429 PVM pVM = pVCpu->CTX_SUFF(pVM);
7430 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7431 {
7432 Assert(PDMVmmDevHeapIsEnabled(pVM));
7433 Assert(pVM->hm.s.vmx.pRealModeTSS);
7434
7435 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7436 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7437 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7438 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7439 AssertRCReturn(rc, rc);
7440 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7441
7442 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7443 const size_t cbIdtEntry = sizeof(X86IDTR16);
7444 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7445 {
7446 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7447 if (uVector == X86_XCPT_DF)
7448 return VINF_EM_RESET;
7449 else if (uVector == X86_XCPT_GP)
7450 {
7451 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7452 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7453 }
7454
7455 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7456 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7457 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, puIntrState);
7458 }
7459
7460 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7461 uint16_t uGuestIp = pMixedCtx->ip;
7462 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7463 {
7464 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7465 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7466 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7467 }
7468 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7469 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7470
7471 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7472 X86IDTR16 IdtEntry;
7473 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7474 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7475 AssertRCReturn(rc, rc);
7476
7477 /* Construct the stack frame for the interrupt/exception handler. */
7478 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7479 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7480 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7481 AssertRCReturn(rc, rc);
7482
7483 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7484 if (rc == VINF_SUCCESS)
7485 {
7486 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7487 pMixedCtx->rip = IdtEntry.offSel;
7488 pMixedCtx->cs.Sel = IdtEntry.uSel;
7489 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7490 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7491 && uVector == X86_XCPT_PF)
7492 {
7493 pMixedCtx->cr2 = GCPtrFaultAddress;
7494 }
7495
7496 /* If any other guest-state bits are changed here, make sure to update
7497 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7498 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7499 | HM_CHANGED_GUEST_RIP
7500 | HM_CHANGED_GUEST_RFLAGS
7501 | HM_CHANGED_GUEST_RSP);
7502
7503 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7504 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7505 {
7506 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7507 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7508 Log4(("Clearing inhibition due to STI.\n"));
7509 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7510 }
7511 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7512
7513 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7514 it, if we are returning to ring-3 before executing guest code. */
7515 pVCpu->hm.s.Event.fPending = false;
7516 }
7517 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7518 return rc;
7519 }
7520 else
7521 {
7522 /*
7523 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7524 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7525 */
7526 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7527 }
7528 }
7529
7530 /* Validate. */
7531 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7532 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntInfo)); /* Bit 12 MBZ. */
7533 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7534
7535 /* Inject. */
7536 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7537 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7538 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7539 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7540
7541 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7542 && uVector == X86_XCPT_PF)
7543 {
7544 pMixedCtx->cr2 = GCPtrFaultAddress;
7545 }
7546
7547 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7548 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7549
7550 AssertRCReturn(rc, rc);
7551 return rc;
7552}
7553
7554
7555/**
7556 * Clears the interrupt-window exiting control in the VMCS and if necessary
7557 * clears the current event in the VMCS as well.
7558 *
7559 * @returns VBox status code.
7560 * @param pVCpu Pointer to the VMCPU.
7561 *
7562 * @remarks Use this function only to clear events that have not yet been
7563 * delivered to the guest but are injected in the VMCS!
7564 * @remarks No-long-jump zone!!!
7565 */
7566static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7567{
7568 int rc;
7569 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7570
7571 /* Clear interrupt-window exiting control. */
7572 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7573 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7574
7575 if (!pVCpu->hm.s.Event.fPending)
7576 return;
7577
7578#ifdef VBOX_STRICT
7579 uint32_t u32EntryInfo;
7580 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7581 AssertRC(rc);
7582 Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
7583#endif
7584
7585 /* Clear the entry-interruption field (including the valid bit). */
7586 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7587 AssertRC(rc);
7588
7589 /* Clear the pending debug exception field. */
7590 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7591 AssertRC(rc);
7592}
7593
7594
7595/**
7596 * Enters the VT-x session.
7597 *
7598 * @returns VBox status code.
7599 * @param pVM Pointer to the VM.
7600 * @param pVCpu Pointer to the VMCPU.
7601 * @param pCpu Pointer to the CPU info struct.
7602 */
7603VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7604{
7605 AssertPtr(pVM);
7606 AssertPtr(pVCpu);
7607 Assert(pVM->hm.s.vmx.fSupported);
7608 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7609 NOREF(pCpu); NOREF(pVM);
7610
7611 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7612 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7613
7614#ifdef VBOX_STRICT
7615 /* Make sure we're in VMX root mode. */
7616 RTCCUINTREG u32HostCR4 = ASMGetCR4();
7617 if (!(u32HostCR4 & X86_CR4_VMXE))
7618 {
7619 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7620 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7621 }
7622#endif
7623
7624 /*
7625 * Load the VCPU's VMCS as the current (and active) one.
7626 */
7627 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7628 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7629 if (RT_FAILURE(rc))
7630 return rc;
7631
7632 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7633 pVCpu->hm.s.fLeaveDone = false;
7634 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7635
7636 return VINF_SUCCESS;
7637}
7638
7639
7640/**
7641 * The thread-context callback (only on platforms which support it).
7642 *
7643 * @param enmEvent The thread-context event.
7644 * @param pVCpu Pointer to the VMCPU.
7645 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7646 * @thread EMT(pVCpu)
7647 */
7648VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7649{
7650 NOREF(fGlobalInit);
7651
7652 switch (enmEvent)
7653 {
7654 case RTTHREADCTXEVENT_PREEMPTING:
7655 {
7656 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7657 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7658 VMCPU_ASSERT_EMT(pVCpu);
7659
7660 PVM pVM = pVCpu->CTX_SUFF(pVM);
7661 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7662
7663 /* No longjmps (logger flushes, locks) in this fragile context. */
7664 VMMRZCallRing3Disable(pVCpu);
7665 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7666
7667 /*
7668 * Restore host-state (FPU, debug etc.)
7669 */
7670 if (!pVCpu->hm.s.fLeaveDone)
7671 {
7672 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7673 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7674 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7675 pVCpu->hm.s.fLeaveDone = true;
7676 }
7677
7678 /* Leave HM context, takes care of local init (term). */
7679 int rc = HMR0LeaveCpu(pVCpu);
7680 AssertRC(rc); NOREF(rc);
7681
7682 /* Restore longjmp state. */
7683 VMMRZCallRing3Enable(pVCpu);
7684 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7685 break;
7686 }
7687
7688 case RTTHREADCTXEVENT_RESUMED:
7689 {
7690 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7691 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7692 VMCPU_ASSERT_EMT(pVCpu);
7693
7694 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7695 VMMRZCallRing3Disable(pVCpu);
7696 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7697
7698 /* Initialize the bare minimum state required for HM. This takes care of
7699 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7700 int rc = HMR0EnterCpu(pVCpu);
7701 AssertRC(rc);
7702 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7703
7704 /* Load the active VMCS as the current one. */
7705 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7706 {
7707 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7708 AssertRC(rc); NOREF(rc);
7709 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7710 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7711 }
7712 pVCpu->hm.s.fLeaveDone = false;
7713
7714 /* Restore longjmp state. */
7715 VMMRZCallRing3Enable(pVCpu);
7716 break;
7717 }
7718
7719 default:
7720 break;
7721 }
7722}
7723
7724
7725/**
7726 * Saves the host state in the VMCS host-state.
7727 * Sets up the VM-exit MSR-load area.
7728 *
7729 * The CPU state will be loaded from these fields on every successful VM-exit.
7730 *
7731 * @returns VBox status code.
7732 * @param pVM Pointer to the VM.
7733 * @param pVCpu Pointer to the VMCPU.
7734 *
7735 * @remarks No-long-jump zone!!!
7736 */
7737static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7738{
7739 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7740
7741 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7742 return VINF_SUCCESS;
7743
7744 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7745 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7746
7747 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7748 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7749
7750 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7751 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7752
7753 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
7754 return rc;
7755}
7756
7757
7758/**
7759 * Saves the host state in the VMCS host-state.
7760 *
7761 * @returns VBox status code.
7762 * @param pVM Pointer to the VM.
7763 * @param pVCpu Pointer to the VMCPU.
7764 *
7765 * @remarks No-long-jump zone!!!
7766 */
7767VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7768{
7769 AssertPtr(pVM);
7770 AssertPtr(pVCpu);
7771
7772 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7773
7774 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7775 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7776 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7777 return hmR0VmxSaveHostState(pVM, pVCpu);
7778}
7779
7780
7781/**
7782 * Loads the guest state into the VMCS guest-state area. The CPU state will be
7783 * loaded from these fields on every successful VM-entry.
7784 *
7785 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7786 * Sets up the VM-entry controls.
7787 * Sets up the appropriate VMX non-root function to execute guest code based on
7788 * the guest CPU mode.
7789 *
7790 * @returns VBox status code.
7791 * @param pVM Pointer to the VM.
7792 * @param pVCpu Pointer to the VMCPU.
7793 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7794 * out-of-sync. Make sure to update the required fields
7795 * before using them.
7796 *
7797 * @remarks No-long-jump zone!!!
7798 */
7799static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7800{
7801 AssertPtr(pVM);
7802 AssertPtr(pVCpu);
7803 AssertPtr(pMixedCtx);
7804 HMVMX_ASSERT_PREEMPT_SAFE();
7805
7806#ifdef LOG_ENABLED
7807 /** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7808 * probably not initialized yet? Anyway this will do for now.
7809 *
7810 * Update: Should be possible once VMXR0LoadGuestState() is removed as an
7811 * interface and disable ring-3 calls when thread-context hooks are not
7812 * available. */
7813 bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7814 VMMR0LogFlushDisable(pVCpu);
7815#endif
7816
7817 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7818
7819 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7820
7821 /* Determine real-on-v86 mode. */
7822 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7823 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7824 && CPUMIsGuestInRealModeEx(pMixedCtx))
7825 {
7826 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7827 }
7828
7829 /*
7830 * Load the guest-state into the VMCS.
7831 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7832 * Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7833 */
7834 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7835 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7836
7837 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7838 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7839 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7840
7841 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7842 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7843 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7844
7845 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7846 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7847
7848 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7849 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7850
7851 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7852 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7853 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7854
7855 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7856 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7857
7858 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7859 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7860
7861 /*
7862 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7863 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7864 */
7865 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7866 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7867
7868 /* Clear any unused and reserved bits. */
7869 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
7870
7871#ifdef LOG_ENABLED
7872 /* Only reenable log-flushing if the caller has it enabled. */
7873 if (!fCallerDisabledLogFlush)
7874 VMMR0LogFlushEnable(pVCpu);
7875#endif
7876
7877 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7878 return rc;
7879}
7880
7881
7882/**
7883 * Loads the state shared between the host and guest into the VMCS.
7884 *
7885 * @param pVM Pointer to the VM.
7886 * @param pVCpu Pointer to the VMCPU.
7887 * @param pCtx Pointer to the guest-CPU context.
7888 *
7889 * @remarks No-long-jump zone!!!
7890 */
7891static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7892{
7893 NOREF(pVM);
7894
7895 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7896 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7897
7898 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
7899 {
7900 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7901 AssertRC(rc);
7902 }
7903
7904 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
7905 {
7906 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7907 AssertRC(rc);
7908
7909 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7910 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
7911 {
7912 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7913 AssertRC(rc);
7914 }
7915 }
7916
7917 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
7918 {
7919#if HC_ARCH_BITS == 64
7920 if (pVM->hm.s.fAllow64BitGuests)
7921 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
7922#endif
7923 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
7924 }
7925
7926 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
7927 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7928}
7929
7930
7931/**
7932 * Worker for loading the guest-state bits in the inner VT-x execution loop.
7933 *
7934 * @param pVM Pointer to the VM.
7935 * @param pVCpu Pointer to the VMCPU.
7936 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7937 * out-of-sync. Make sure to update the required fields
7938 * before using them.
7939 */
7940DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7941{
7942 HMVMX_ASSERT_PREEMPT_SAFE();
7943
7944 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7945#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7946 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7947#endif
7948
7949 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
7950 {
7951 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7952 AssertRC(rc);
7953 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7954 }
7955 else if (HMCPU_CF_VALUE(pVCpu))
7956 {
7957 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7958 AssertRC(rc);
7959 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7960 }
7961
7962 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
7963 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
7964 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
7965 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7966
7967#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7968 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7969 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7970 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7971#endif
7972}
7973
7974
7975/**
7976 * Does the preparations before executing guest code in VT-x.
7977 *
7978 * This may cause longjmps to ring-3 and may even result in rescheduling to the
7979 * recompiler. We must be cautious what we do here regarding committing
7980 * guest-state information into the VMCS assuming we assuredly execute the
7981 * guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7982 * and clearing the common-state (TRPM/forceflags), we must undo those changes
7983 * so that the recompiler can (and should) use them when it resumes guest
7984 * execution. Otherwise such operations must be done when we can no longer
7985 * exit to ring-3.
7986 *
7987 * @returns Strict VBox status code.
7988 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7989 * have been disabled.
7990 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7991 * double-fault into the guest.
7992 * @retval VINF_* scheduling changes, we have to go back to ring-3.
7993 *
7994 * @param pVM Pointer to the VM.
7995 * @param pVCpu Pointer to the VMCPU.
7996 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7997 * out-of-sync. Make sure to update the required fields
7998 * before using them.
7999 * @param pVmxTransient Pointer to the VMX transient structure.
8000 */
8001static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8002{
8003 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8004
8005#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8006 PGMRZDynMapFlushAutoSet(pVCpu);
8007#endif
8008
8009 /* Check force flag actions that might require us to go back to ring-3. */
8010 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
8011 if (rc != VINF_SUCCESS)
8012 return rc;
8013
8014#ifndef IEM_VERIFICATION_MODE_FULL
8015 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8016 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8017 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8018 {
8019 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8020 RTGCPHYS GCPhysApicBase;
8021 GCPhysApicBase = pMixedCtx->msrApicBase;
8022 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8023
8024 /* Unalias any existing mapping. */
8025 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8026 AssertRCReturn(rc, rc);
8027
8028 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8029 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
8030 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8031 AssertRCReturn(rc, rc);
8032
8033 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8034 }
8035#endif /* !IEM_VERIFICATION_MODE_FULL */
8036
8037 /* Load the guest state bits, we can handle longjmps/getting preempted here. */
8038 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8039
8040 /*
8041 * Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
8042 * we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
8043 */
8044 if (TRPMHasTrap(pVCpu))
8045 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8046 else if (!pVCpu->hm.s.Event.fPending)
8047 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8048
8049 /*
8050 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8051 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8052 */
8053 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
8054 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8055 {
8056 Assert(rc == VINF_EM_RESET);
8057 return rc;
8058 }
8059
8060 /*
8061 * No longjmps to ring-3 from this point on!!!
8062 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8063 * This also disables flushing of the R0-logger instance (if any).
8064 */
8065 VMMRZCallRing3Disable(pVCpu);
8066
8067 /*
8068 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8069 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8070 *
8071 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8072 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8073 *
8074 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8075 * executing guest code.
8076 */
8077 pVmxTransient->uEflags = ASMIntDisableFlags();
8078 if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8079 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8080 {
8081 hmR0VmxClearEventVmcs(pVCpu);
8082 ASMSetFlags(pVmxTransient->uEflags);
8083 VMMRZCallRing3Enable(pVCpu);
8084 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8085 return VINF_EM_RAW_TO_R3;
8086 }
8087 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8088 {
8089 hmR0VmxClearEventVmcs(pVCpu);
8090 ASMSetFlags(pVmxTransient->uEflags);
8091 VMMRZCallRing3Enable(pVCpu);
8092 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8093 return VINF_EM_RAW_INTERRUPT;
8094 }
8095
8096 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8097 pVCpu->hm.s.Event.fPending = false;
8098
8099 return VINF_SUCCESS;
8100}
8101
8102
8103/**
8104 * Prepares to run guest code in VT-x and we've committed to doing so. This
8105 * means there is no backing out to ring-3 or anywhere else at this
8106 * point.
8107 *
8108 * @param pVM Pointer to the VM.
8109 * @param pVCpu Pointer to the VMCPU.
8110 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8111 * out-of-sync. Make sure to update the required fields
8112 * before using them.
8113 * @param pVmxTransient Pointer to the VMX transient structure.
8114 *
8115 * @remarks Called with preemption disabled.
8116 * @remarks No-long-jump zone!!!
8117 */
8118static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8119{
8120 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8121 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8122 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8123
8124 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8125 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8126
8127 /*
8128 * If we are injecting events to a real-on-v86 mode guest, we may have to update
8129 * RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8130 * Reload only the necessary state, the assertion will catch if other parts of the code
8131 * change.
8132 */
8133 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8134 {
8135 hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8136 hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8137 }
8138
8139#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8140 if (!CPUMIsGuestFPUStateActive(pVCpu))
8141 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8142 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8143#endif
8144
8145 if ( pVCpu->hm.s.fUseGuestFpu
8146 && !CPUMIsGuestFPUStateActive(pVCpu))
8147 {
8148 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8149 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8150 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8151 }
8152
8153 /*
8154 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8155 */
8156 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8157 && pVCpu->hm.s.vmx.cMsrs > 0)
8158 {
8159 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8160 }
8161
8162 /*
8163 * Load the host state bits as we may've been preempted (only happens when
8164 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8165 */
8166 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8167 {
8168 /* This ASSUMES that pfnStartVM has been set up already. */
8169 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8170 AssertRC(rc);
8171 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8172 }
8173 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8174
8175 /*
8176 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8177 */
8178 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8179 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8180 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8181
8182 /* Store status of the shared guest-host state at the time of VM-entry. */
8183#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8184 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8185 {
8186 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8187 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8188 }
8189 else
8190#endif
8191 {
8192 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8193 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8194 }
8195 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8196
8197 /*
8198 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8199 */
8200 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8201 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8202
8203 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8204 RTCPUID idCurrentCpu = pCpu->idCpu;
8205 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8206 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8207 {
8208 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8209 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8210 }
8211
8212 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8213 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8214 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8215 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8216
8217 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8218
8219 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8220 to start executing. */
8221
8222 /*
8223 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8224 */
8225 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8226 {
8227 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8228 {
8229 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8230 AssertRC(rc2);
8231 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8232 bool fMsrUpdated = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu),
8233 true /* fUpdateHostMsr */);
8234 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8235 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8236 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8237 }
8238 else
8239 {
8240 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8241 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8242 }
8243 }
8244#ifdef VBOX_STRICT
8245 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8246#endif
8247}
8248
8249
8250/**
8251 * Performs some essential restoration of state after running guest code in
8252 * VT-x.
8253 *
8254 * @param pVM Pointer to the VM.
8255 * @param pVCpu Pointer to the VMCPU.
8256 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8257 * out-of-sync. Make sure to update the required fields
8258 * before using them.
8259 * @param pVmxTransient Pointer to the VMX transient structure.
8260 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8261 *
8262 * @remarks Called with interrupts disabled, and returns with interrups enabled!
8263 *
8264 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8265 * unconditionally when it is safe to do so.
8266 */
8267static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8268{
8269 NOREF(pVM);
8270
8271 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8272
8273 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8274 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8275 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8276 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8277 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8278
8279 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8280 {
8281 /** @todo Find a way to fix hardcoding a guestimate. */
8282 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
8283 + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
8284 }
8285
8286 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8287 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8288 Assert(!(ASMGetFlags() & X86_EFL_IF));
8289 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8290
8291#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8292 if (CPUMIsGuestFPUStateActive(pVCpu))
8293 {
8294 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8295 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8296 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8297 }
8298#endif
8299
8300#if HC_ARCH_BITS == 64
8301 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8302#endif
8303 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8304 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8305 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8306
8307 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8308 uint32_t uExitReason;
8309 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8310 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8311 AssertRC(rc);
8312 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8313 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8314
8315 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8316 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8317 {
8318 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8319 pVmxTransient->fVMEntryFailed));
8320 return;
8321 }
8322
8323 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8324 {
8325 /* Update the guest interruptibility-state from the VMCS. */
8326 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8327#if defined(HMVMX_SYNC_FULL_GUEST_STATE) || defined(HMVMX_SAVE_FULL_GUEST_STATE)
8328 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8329 AssertRC(rc);
8330#endif
8331 /*
8332 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8333 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8334 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8335 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8336 */
8337 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8338 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8339 {
8340 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8341 AssertRC(rc);
8342 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8343 }
8344 }
8345}
8346
8347
8348
8349/**
8350 * Runs the guest code using VT-x the normal way.
8351 *
8352 * @returns VBox status code.
8353 * @param pVM Pointer to the VM.
8354 * @param pVCpu Pointer to the VMCPU.
8355 * @param pCtx Pointer to the guest-CPU context.
8356 *
8357 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8358 */
8359static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8360{
8361 VMXTRANSIENT VmxTransient;
8362 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8363 int rc = VERR_INTERNAL_ERROR_5;
8364 uint32_t cLoops = 0;
8365
8366 for (;; cLoops++)
8367 {
8368 Assert(!HMR0SuspendPending());
8369 HMVMX_ASSERT_CPU_SAFE();
8370
8371 /* Preparatory work for running guest code, this may force us to return
8372 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8373 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8374 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8375 if (rc != VINF_SUCCESS)
8376 break;
8377
8378 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8379 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8380 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8381
8382 /* Restore any residual host-state and save any bits shared between host
8383 and guest into the guest-CPU state. Re-enables interrupts! */
8384 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8385
8386 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8387 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8388 {
8389 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8390 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8391 return rc;
8392 }
8393
8394 /* Handle the VM-exit. */
8395 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8396 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8397 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8398 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8399 HMVMX_START_EXIT_DISPATCH_PROF();
8400#ifdef HMVMX_USE_FUNCTION_TABLE
8401 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8402#else
8403 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8404#endif
8405 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8406 if (rc != VINF_SUCCESS)
8407 break;
8408 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8409 {
8410 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8411 rc = VINF_EM_RAW_INTERRUPT;
8412 break;
8413 }
8414 }
8415
8416 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8417 return rc;
8418}
8419
8420
8421/**
8422 * Single steps guest code using VT-x.
8423 *
8424 * @returns VBox status code.
8425 * @param pVM Pointer to the VM.
8426 * @param pVCpu Pointer to the VMCPU.
8427 * @param pCtx Pointer to the guest-CPU context.
8428 *
8429 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8430 */
8431static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8432{
8433 VMXTRANSIENT VmxTransient;
8434 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8435 int rc = VERR_INTERNAL_ERROR_5;
8436 uint32_t cLoops = 0;
8437 uint16_t uCsStart = pCtx->cs.Sel;
8438 uint64_t uRipStart = pCtx->rip;
8439
8440 for (;; cLoops++)
8441 {
8442 Assert(!HMR0SuspendPending());
8443 HMVMX_ASSERT_CPU_SAFE();
8444
8445 /* Preparatory work for running guest code, this may force us to return
8446 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8447 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8448 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8449 if (rc != VINF_SUCCESS)
8450 break;
8451
8452 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8453 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8454 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8455
8456 /* Restore any residual host-state and save any bits shared between host
8457 and guest into the guest-CPU state. Re-enables interrupts! */
8458 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8459
8460 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8461 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8462 {
8463 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8464 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8465 return rc;
8466 }
8467
8468 /* Handle the VM-exit. */
8469 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8470 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8471 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8472 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8473 HMVMX_START_EXIT_DISPATCH_PROF();
8474#ifdef HMVMX_USE_FUNCTION_TABLE
8475 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8476#else
8477 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8478#endif
8479 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8480 if (rc != VINF_SUCCESS)
8481 break;
8482 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8483 {
8484 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8485 rc = VINF_EM_RAW_INTERRUPT;
8486 break;
8487 }
8488
8489 /*
8490 * Did the RIP change, if so, consider it a single step.
8491 * Otherwise, make sure one of the TFs gets set.
8492 */
8493 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8494 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8495 AssertRCReturn(rc2, rc2);
8496 if ( pCtx->rip != uRipStart
8497 || pCtx->cs.Sel != uCsStart)
8498 {
8499 rc = VINF_EM_DBG_STEPPED;
8500 break;
8501 }
8502 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8503 }
8504
8505 /*
8506 * Clear the X86_EFL_TF if necessary.
8507 */
8508 if (pVCpu->hm.s.fClearTrapFlag)
8509 {
8510 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8511 AssertRCReturn(rc2, rc2);
8512 pVCpu->hm.s.fClearTrapFlag = false;
8513 pCtx->eflags.Bits.u1TF = 0;
8514 }
8515 /** @todo there seems to be issues with the resume flag when the monitor trap
8516 * flag is pending without being used. Seen early in bios init when
8517 * accessing APIC page in prot mode. */
8518
8519 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8520 return rc;
8521}
8522
8523
8524/**
8525 * Runs the guest code using VT-x.
8526 *
8527 * @returns VBox status code.
8528 * @param pVM Pointer to the VM.
8529 * @param pVCpu Pointer to the VMCPU.
8530 * @param pCtx Pointer to the guest-CPU context.
8531 */
8532VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8533{
8534 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8535 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
8536 HMVMX_ASSERT_PREEMPT_SAFE();
8537
8538 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8539
8540 int rc;
8541 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8542 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8543 else
8544 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8545
8546 if (rc == VERR_EM_INTERPRETER)
8547 rc = VINF_EM_RAW_EMULATE_INSTR;
8548 else if (rc == VINF_EM_RESET)
8549 rc = VINF_EM_TRIPLE_FAULT;
8550
8551 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8552 if (RT_FAILURE(rc2))
8553 {
8554 pVCpu->hm.s.u32HMError = rc;
8555 rc = rc2;
8556 }
8557 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8558 return rc;
8559}
8560
8561
8562#ifndef HMVMX_USE_FUNCTION_TABLE
8563DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8564{
8565#ifdef DEBUG_ramshankar
8566# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
8567# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
8568#endif
8569 int rc;
8570 switch (rcReason)
8571 {
8572 case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); */ rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8573 case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); */ rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8574 case VMX_EXIT_IO_INSTR: /* SVVMCS(); */ rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8575 case VMX_EXIT_CPUID: /* SVVMCS(); */ rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8576 case VMX_EXIT_RDTSC: /* SVVMCS(); */ rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8577 case VMX_EXIT_RDTSCP: /* SVVMCS(); */ rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8578 case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8579 case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); */ rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8580 case VMX_EXIT_MOV_CRX: /* SVVMCS(); */ rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8581 case VMX_EXIT_EXT_INT: /* SVVMCS(); */ rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8582 case VMX_EXIT_INT_WINDOW: /* SVVMCS(); */ rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8583 case VMX_EXIT_MWAIT: /* SVVMCS(); */ rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8584 case VMX_EXIT_MONITOR: /* SVVMCS(); */ rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8585 case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); */ rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8586 case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); */ rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8587 case VMX_EXIT_RDMSR: /* SVVMCS(); */ rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8588 case VMX_EXIT_WRMSR: /* SVVMCS(); */ rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8589 case VMX_EXIT_MOV_DRX: /* SVVMCS(); */ rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8590 case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); */ rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8591 case VMX_EXIT_HLT: /* SVVMCS(); */ rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8592 case VMX_EXIT_INVD: /* SVVMCS(); */ rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8593 case VMX_EXIT_INVLPG: /* SVVMCS(); */ rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8594 case VMX_EXIT_RSM: /* SVVMCS(); */ rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8595 case VMX_EXIT_MTF: /* SVVMCS(); */ rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8596 case VMX_EXIT_PAUSE: /* SVVMCS(); */ rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8597 case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8598 case VMX_EXIT_TR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8599 case VMX_EXIT_WBINVD: /* SVVMCS(); */ rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8600 case VMX_EXIT_XSETBV: /* SVVMCS(); */ rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8601 case VMX_EXIT_RDRAND: /* SVVMCS(); */ rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8602 case VMX_EXIT_INVPCID: /* SVVMCS(); */ rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8603 case VMX_EXIT_GETSEC: /* SVVMCS(); */ rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8604 case VMX_EXIT_RDPMC: /* SVVMCS(); */ rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8605
8606 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8607 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8608 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8609 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8610 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8611 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8612 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8613 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8614 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8615
8616 case VMX_EXIT_VMCALL:
8617 case VMX_EXIT_VMCLEAR:
8618 case VMX_EXIT_VMLAUNCH:
8619 case VMX_EXIT_VMPTRLD:
8620 case VMX_EXIT_VMPTRST:
8621 case VMX_EXIT_VMREAD:
8622 case VMX_EXIT_VMRESUME:
8623 case VMX_EXIT_VMWRITE:
8624 case VMX_EXIT_VMXOFF:
8625 case VMX_EXIT_VMXON:
8626 case VMX_EXIT_INVEPT:
8627 case VMX_EXIT_INVVPID:
8628 case VMX_EXIT_VMFUNC:
8629 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8630 break;
8631 default:
8632 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8633 break;
8634 }
8635 return rc;
8636}
8637#endif
8638
8639#ifdef DEBUG
8640/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8641# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8642 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8643
8644# define HMVMX_ASSERT_PREEMPT_CPUID() \
8645 do \
8646 { \
8647 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8648 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8649 } while (0)
8650
8651# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8652 do { \
8653 AssertPtr(pVCpu); \
8654 AssertPtr(pMixedCtx); \
8655 AssertPtr(pVmxTransient); \
8656 Assert(pVmxTransient->fVMEntryFailed == false); \
8657 Assert(ASMIntAreEnabled()); \
8658 HMVMX_ASSERT_PREEMPT_SAFE(); \
8659 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8660 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8661 HMVMX_ASSERT_PREEMPT_SAFE(); \
8662 if (VMMR0IsLogFlushDisabled(pVCpu)) \
8663 HMVMX_ASSERT_PREEMPT_CPUID(); \
8664 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8665 } while (0)
8666
8667# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8668 do { \
8669 Log4Func(("\n")); \
8670 } while (0)
8671#else /* Release builds */
8672# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8673 do { \
8674 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8675 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
8676 } while (0)
8677# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
8678#endif
8679
8680
8681/**
8682 * Advances the guest RIP after reading it from the VMCS.
8683 *
8684 * @returns VBox status code.
8685 * @param pVCpu Pointer to the VMCPU.
8686 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8687 * out-of-sync. Make sure to update the required fields
8688 * before using them.
8689 * @param pVmxTransient Pointer to the VMX transient structure.
8690 *
8691 * @remarks No-long-jump zone!!!
8692 */
8693DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8694{
8695 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8696 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8697 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8698 AssertRCReturn(rc, rc);
8699
8700 pMixedCtx->rip += pVmxTransient->cbInstr;
8701 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
8702
8703 /*
8704 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
8705 * pending debug exception field as it takes care of priority of events.
8706 *
8707 * See Intel spec. 32.2.1 "Debug Exceptions".
8708 */
8709 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
8710
8711 return rc;
8712}
8713
8714
8715/**
8716 * Tries to determine what part of the guest-state VT-x has deemed as invalid
8717 * and update error record fields accordingly.
8718 *
8719 * @return VMX_IGS_* return codes.
8720 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
8721 * wrong with the guest state.
8722 *
8723 * @param pVM Pointer to the VM.
8724 * @param pVCpu Pointer to the VMCPU.
8725 * @param pCtx Pointer to the guest-CPU state.
8726 */
8727static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8728{
8729#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
8730#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
8731 uError = (err); \
8732 break; \
8733 } else do { } while (0)
8734
8735 int rc;
8736 uint32_t uError = VMX_IGS_ERROR;
8737 uint32_t u32Val;
8738 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
8739
8740 do
8741 {
8742 /*
8743 * CR0.
8744 */
8745 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8746 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8747 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8748 See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8749 if (fUnrestrictedGuest)
8750 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
8751
8752 uint32_t u32GuestCR0;
8753 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
8754 AssertRCBreak(rc);
8755 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8756 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8757 if ( !fUnrestrictedGuest
8758 && (u32GuestCR0 & X86_CR0_PG)
8759 && !(u32GuestCR0 & X86_CR0_PE))
8760 {
8761 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8762 }
8763
8764 /*
8765 * CR4.
8766 */
8767 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8768 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8769
8770 uint32_t u32GuestCR4;
8771 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
8772 AssertRCBreak(rc);
8773 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8774 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8775
8776 /*
8777 * IA32_DEBUGCTL MSR.
8778 */
8779 uint64_t u64Val;
8780 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8781 AssertRCBreak(rc);
8782 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8783 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8784 {
8785 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8786 }
8787 uint64_t u64DebugCtlMsr = u64Val;
8788
8789#ifdef VBOX_STRICT
8790 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
8791 AssertRCBreak(rc);
8792 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
8793#endif
8794 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8795
8796 /*
8797 * RIP and RFLAGS.
8798 */
8799 uint32_t u32Eflags;
8800#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8801 if (HMVMX_IS_64BIT_HOST_MODE())
8802 {
8803 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8804 AssertRCBreak(rc);
8805 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8806 if ( !fLongModeGuest
8807 || !pCtx->cs.Attr.n.u1Long)
8808 {
8809 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8810 }
8811 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8812 * must be identical if the "IA32e mode guest" VM-entry control is 1
8813 * and CS.L is 1. No check applies if the CPU supports 64
8814 * linear-address bits. */
8815
8816 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8817 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8818 AssertRCBreak(rc);
8819 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8820 VMX_IGS_RFLAGS_RESERVED);
8821 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8822 u32Eflags = u64Val;
8823 }
8824 else
8825#endif
8826 {
8827 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8828 AssertRCBreak(rc);
8829 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8830 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8831 }
8832
8833 if ( fLongModeGuest
8834 || ( fUnrestrictedGuest
8835 && !(u32GuestCR0 & X86_CR0_PE)))
8836 {
8837 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8838 }
8839
8840 uint32_t u32EntryInfo;
8841 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8842 AssertRCBreak(rc);
8843 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
8844 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8845 {
8846 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8847 }
8848
8849 /*
8850 * 64-bit checks.
8851 */
8852#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8853 if (HMVMX_IS_64BIT_HOST_MODE())
8854 {
8855 if ( fLongModeGuest
8856 && !fUnrestrictedGuest)
8857 {
8858 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
8859 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
8860 }
8861
8862 if ( !fLongModeGuest
8863 && (u32GuestCR4 & X86_CR4_PCIDE))
8864 {
8865 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8866 }
8867
8868 /** @todo CR3 field must be such that bits 63:52 and bits in the range
8869 * 51:32 beyond the processor's physical-address width are 0. */
8870
8871 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8872 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8873 {
8874 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8875 }
8876
8877 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8878 AssertRCBreak(rc);
8879 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8880
8881 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8882 AssertRCBreak(rc);
8883 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8884 }
8885#endif
8886
8887 /*
8888 * PERF_GLOBAL MSR.
8889 */
8890 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8891 {
8892 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8893 AssertRCBreak(rc);
8894 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8895 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8896 }
8897
8898 /*
8899 * PAT MSR.
8900 */
8901 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8902 {
8903 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8904 AssertRCBreak(rc);
8905 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8906 for (unsigned i = 0; i < 8; i++)
8907 {
8908 uint8_t u8Val = (u64Val & 0x7);
8909 if ( u8Val != 0 /* UC */
8910 || u8Val != 1 /* WC */
8911 || u8Val != 4 /* WT */
8912 || u8Val != 5 /* WP */
8913 || u8Val != 6 /* WB */
8914 || u8Val != 7 /* UC- */)
8915 {
8916 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8917 }
8918 u64Val >>= 3;
8919 }
8920 }
8921
8922 /*
8923 * EFER MSR.
8924 */
8925 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8926 {
8927 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8928 AssertRCBreak(rc);
8929 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8930 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8931 HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8932 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8933 HMVMX_CHECK_BREAK( fUnrestrictedGuest
8934 || (u64Val & MSR_K6_EFER_LMA) == (u32GuestCR0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8935 }
8936
8937 /*
8938 * Segment registers.
8939 */
8940 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8941 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8942 if (!(u32Eflags & X86_EFL_VM))
8943 {
8944 /* CS */
8945 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8946 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8947 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8948 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8949 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8950 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8951 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8952 /* CS cannot be loaded with NULL in protected mode. */
8953 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8954 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8955 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
8956 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8957 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
8958 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8959 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8960 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8961 else
8962 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8963
8964 /* SS */
8965 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8966 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8967 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8968 if ( !(pCtx->cr0 & X86_CR0_PE)
8969 || pCtx->cs.Attr.n.u4Type == 3)
8970 {
8971 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8972 }
8973 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8974 {
8975 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8976 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8977 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8978 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8979 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8980 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8981 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8982 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8983 }
8984
8985 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8986 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8987 {
8988 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8989 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8990 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8991 || pCtx->ds.Attr.n.u4Type > 11
8992 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8993 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8994 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8995 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8996 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8997 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8998 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8999 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9000 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9001 }
9002 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9003 {
9004 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9005 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9006 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9007 || pCtx->es.Attr.n.u4Type > 11
9008 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9009 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9010 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9011 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9012 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9013 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9014 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9015 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9016 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9017 }
9018 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9019 {
9020 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9021 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9022 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9023 || pCtx->fs.Attr.n.u4Type > 11
9024 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9025 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9026 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9027 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9028 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9029 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9030 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9031 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9032 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9033 }
9034 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9035 {
9036 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9037 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9038 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9039 || pCtx->gs.Attr.n.u4Type > 11
9040 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9041 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9042 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9043 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9044 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9045 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9046 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9047 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9048 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9049 }
9050 /* 64-bit capable CPUs. */
9051#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9052 if (HMVMX_IS_64BIT_HOST_MODE())
9053 {
9054 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9055 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9056 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9057 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9058 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9059 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9060 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9061 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9062 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9063 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9064 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9065 }
9066#endif
9067 }
9068 else
9069 {
9070 /* V86 mode checks. */
9071 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9072 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9073 {
9074 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9075 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9076 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9077 }
9078 else
9079 {
9080 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9081 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9082 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9083 }
9084
9085 /* CS */
9086 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9087 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9088 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9089 /* SS */
9090 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9091 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9092 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9093 /* DS */
9094 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9095 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9096 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9097 /* ES */
9098 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9099 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9100 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9101 /* FS */
9102 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9103 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9104 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9105 /* GS */
9106 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9107 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9108 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9109 /* 64-bit capable CPUs. */
9110#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9111 if (HMVMX_IS_64BIT_HOST_MODE())
9112 {
9113 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9114 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9115 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9116 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9117 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9118 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9119 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9120 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9121 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9122 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9123 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9124 }
9125#endif
9126 }
9127
9128 /*
9129 * TR.
9130 */
9131 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9132 /* 64-bit capable CPUs. */
9133#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9134 if (HMVMX_IS_64BIT_HOST_MODE())
9135 {
9136 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9137 }
9138#endif
9139 if (fLongModeGuest)
9140 {
9141 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9142 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9143 }
9144 else
9145 {
9146 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9147 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9148 VMX_IGS_TR_ATTR_TYPE_INVALID);
9149 }
9150 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9151 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9152 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9153 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9154 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9155 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9156 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9157 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9158
9159 /*
9160 * GDTR and IDTR.
9161 */
9162#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9163 if (HMVMX_IS_64BIT_HOST_MODE())
9164 {
9165 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9166 AssertRCBreak(rc);
9167 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9168
9169 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9170 AssertRCBreak(rc);
9171 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9172 }
9173#endif
9174
9175 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9176 AssertRCBreak(rc);
9177 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9178
9179 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9180 AssertRCBreak(rc);
9181 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9182
9183 /*
9184 * Guest Non-Register State.
9185 */
9186 /* Activity State. */
9187 uint32_t u32ActivityState;
9188 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9189 AssertRCBreak(rc);
9190 HMVMX_CHECK_BREAK( !u32ActivityState
9191 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9192 VMX_IGS_ACTIVITY_STATE_INVALID);
9193 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9194 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9195 uint32_t u32IntrState;
9196 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9197 AssertRCBreak(rc);
9198 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9199 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9200 {
9201 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9202 }
9203
9204 /** @todo Activity state and injecting interrupts. Left as a todo since we
9205 * currently don't use activity states but ACTIVE. */
9206
9207 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9208 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9209
9210 /* Guest interruptibility-state. */
9211 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9212 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9213 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9214 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9215 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9216 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9217 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9218 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9219 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9220 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9221 {
9222 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9223 {
9224 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9225 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9226 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9227 }
9228 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9229 {
9230 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9231 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9232 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9233 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9234 }
9235 }
9236 /** @todo Assumes the processor is not in SMM. */
9237 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9238 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9239 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9240 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9241 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9242 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9243 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9244 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9245 {
9246 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9247 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9248 }
9249
9250 /* Pending debug exceptions. */
9251 if (HMVMX_IS_64BIT_HOST_MODE())
9252 {
9253 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9254 AssertRCBreak(rc);
9255 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9256 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9257 u32Val = u64Val; /* For pending debug exceptions checks below. */
9258 }
9259 else
9260 {
9261 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9262 AssertRCBreak(rc);
9263 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9264 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9265 }
9266
9267 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9268 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9269 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9270 {
9271 if ( (u32Eflags & X86_EFL_TF)
9272 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9273 {
9274 /* Bit 14 is PendingDebug.BS. */
9275 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9276 }
9277 if ( !(u32Eflags & X86_EFL_TF)
9278 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9279 {
9280 /* Bit 14 is PendingDebug.BS. */
9281 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9282 }
9283 }
9284
9285 /* VMCS link pointer. */
9286 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9287 AssertRCBreak(rc);
9288 if (u64Val != UINT64_C(0xffffffffffffffff))
9289 {
9290 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9291 /** @todo Bits beyond the processor's physical-address width MBZ. */
9292 /** @todo 32-bit located in memory referenced by value of this field (as a
9293 * physical address) must contain the processor's VMCS revision ID. */
9294 /** @todo SMM checks. */
9295 }
9296
9297 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9298 * not using Nested Paging? */
9299 if ( pVM->hm.s.fNestedPaging
9300 && !fLongModeGuest
9301 && CPUMIsGuestInPAEModeEx(pCtx))
9302 {
9303 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9304 AssertRCBreak(rc);
9305 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9306
9307 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9308 AssertRCBreak(rc);
9309 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9310
9311 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9312 AssertRCBreak(rc);
9313 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9314
9315 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9316 AssertRCBreak(rc);
9317 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9318 }
9319
9320 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9321 if (uError == VMX_IGS_ERROR)
9322 uError = VMX_IGS_REASON_NOT_FOUND;
9323 } while (0);
9324
9325 pVCpu->hm.s.u32HMError = uError;
9326 return uError;
9327
9328#undef HMVMX_ERROR_BREAK
9329#undef HMVMX_CHECK_BREAK
9330}
9331
9332/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9333/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9334/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9335
9336/** @name VM-exit handlers.
9337 * @{
9338 */
9339
9340/**
9341 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9342 */
9343HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9344{
9345 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9346 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9347 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9348 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9349 return VINF_SUCCESS;
9350 return VINF_EM_RAW_INTERRUPT;
9351}
9352
9353
9354/**
9355 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9356 */
9357HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9358{
9359 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9360 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9361
9362 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9363 AssertRCReturn(rc, rc);
9364
9365 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9366 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9367 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9368 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9369
9370 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9371 {
9372 /*
9373 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9374 * anything we inject is not going to cause a VM-exit directly for the event being injected.
9375 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9376 *
9377 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9378 */
9379 VMXDispatchHostNmi();
9380 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9381 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9382 return VINF_SUCCESS;
9383 }
9384
9385 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9386 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9387 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9388 {
9389 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9390 return VINF_SUCCESS;
9391 }
9392 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9393 {
9394 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9395 return rc;
9396 }
9397
9398 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9399 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9400 switch (uIntType)
9401 {
9402 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9403 Assert(uVector == X86_XCPT_DB);
9404 /* no break */
9405 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9406 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9407 /* no break */
9408 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9409 {
9410 switch (uVector)
9411 {
9412 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9413 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9414 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9415 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9416 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9417 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9418#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9419 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9420 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9421 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9422 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9423 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9424 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9425 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9426 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9427 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9428 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9429#endif
9430 default:
9431 {
9432 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9433 AssertRCReturn(rc, rc);
9434
9435 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9436 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9437 {
9438 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9439 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9440 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9441
9442 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9443 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9444 AssertRCReturn(rc, rc);
9445 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9446 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9447 0 /* GCPtrFaultAddress */);
9448 AssertRCReturn(rc, rc);
9449 }
9450 else
9451 {
9452 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9453 pVCpu->hm.s.u32HMError = uVector;
9454 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9455 }
9456 break;
9457 }
9458 }
9459 break;
9460 }
9461
9462 default:
9463 {
9464 pVCpu->hm.s.u32HMError = uExitIntInfo;
9465 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9466 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9467 break;
9468 }
9469 }
9470 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9471 return rc;
9472}
9473
9474
9475/**
9476 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9477 */
9478HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9479{
9480 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9481
9482 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9483 hmR0VmxClearIntWindowExitVmcs(pVCpu);
9484
9485 /* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
9486 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9487 return VINF_SUCCESS;
9488}
9489
9490
9491/**
9492 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9493 */
9494HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9495{
9496 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9497 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9498 HMVMX_RETURN_UNEXPECTED_EXIT();
9499}
9500
9501
9502/**
9503 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9504 */
9505HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9506{
9507 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9508 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9509 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9510}
9511
9512
9513/**
9514 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9515 */
9516HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9517{
9518 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9519 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9520 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9521}
9522
9523
9524/**
9525 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
9526 */
9527HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9528{
9529 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9530 PVM pVM = pVCpu->CTX_SUFF(pVM);
9531 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9532 if (RT_LIKELY(rc == VINF_SUCCESS))
9533 {
9534 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9535 Assert(pVmxTransient->cbInstr == 2);
9536 }
9537 else
9538 {
9539 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
9540 rc = VERR_EM_INTERPRETER;
9541 }
9542 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
9543 return rc;
9544}
9545
9546
9547/**
9548 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
9549 */
9550HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9551{
9552 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9553 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
9554 AssertRCReturn(rc, rc);
9555
9556 if (pMixedCtx->cr4 & X86_CR4_SMXE)
9557 return VINF_EM_RAW_EMULATE_INSTR;
9558
9559 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
9560 HMVMX_RETURN_UNEXPECTED_EXIT();
9561}
9562
9563
9564/**
9565 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
9566 */
9567HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9568{
9569 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9570 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9571 AssertRCReturn(rc, rc);
9572
9573 PVM pVM = pVCpu->CTX_SUFF(pVM);
9574 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9575 if (RT_LIKELY(rc == VINF_SUCCESS))
9576 {
9577 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9578 Assert(pVmxTransient->cbInstr == 2);
9579 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9580 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9581 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9582 }
9583 else
9584 {
9585 AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
9586 rc = VERR_EM_INTERPRETER;
9587 }
9588 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9589 return rc;
9590}
9591
9592
9593/**
9594 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
9595 */
9596HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9597{
9598 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9599 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9600 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
9601 AssertRCReturn(rc, rc);
9602
9603 PVM pVM = pVCpu->CTX_SUFF(pVM);
9604 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
9605 if (RT_LIKELY(rc == VINF_SUCCESS))
9606 {
9607 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9608 Assert(pVmxTransient->cbInstr == 3);
9609 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9610 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9611 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9612 }
9613 else
9614 {
9615 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9616 rc = VERR_EM_INTERPRETER;
9617 }
9618 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9619 return rc;
9620}
9621
9622
9623/**
9624 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9625 */
9626HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9627{
9628 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9629 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9630 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9631 AssertRCReturn(rc, rc);
9632
9633 PVM pVM = pVCpu->CTX_SUFF(pVM);
9634 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9635 if (RT_LIKELY(rc == VINF_SUCCESS))
9636 {
9637 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9638 Assert(pVmxTransient->cbInstr == 2);
9639 }
9640 else
9641 {
9642 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9643 rc = VERR_EM_INTERPRETER;
9644 }
9645 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9646 return rc;
9647}
9648
9649
9650/**
9651 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9652 */
9653HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9654{
9655 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9656 PVM pVM = pVCpu->CTX_SUFF(pVM);
9657 Assert(!pVM->hm.s.fNestedPaging);
9658
9659 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9660 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9661 AssertRCReturn(rc, rc);
9662
9663 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9664 rc = VBOXSTRICTRC_VAL(rc2);
9665 if (RT_LIKELY(rc == VINF_SUCCESS))
9666 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9667 else
9668 {
9669 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9670 pVmxTransient->uExitQualification, rc));
9671 }
9672 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9673 return rc;
9674}
9675
9676
9677/**
9678 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9679 */
9680HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9681{
9682 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9683 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9684 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9685 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9686 AssertRCReturn(rc, rc);
9687
9688 PVM pVM = pVCpu->CTX_SUFF(pVM);
9689 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9690 if (RT_LIKELY(rc == VINF_SUCCESS))
9691 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9692 else
9693 {
9694 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
9695 rc = VERR_EM_INTERPRETER;
9696 }
9697 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
9698 return rc;
9699}
9700
9701
9702/**
9703 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9704 */
9705HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9706{
9707 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9708 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9709 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9710 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9711 AssertRCReturn(rc, rc);
9712
9713 PVM pVM = pVCpu->CTX_SUFF(pVM);
9714 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9715 rc = VBOXSTRICTRC_VAL(rc2);
9716 if (RT_LIKELY( rc == VINF_SUCCESS
9717 || rc == VINF_EM_HALT))
9718 {
9719 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9720 AssertRCReturn(rc3, rc3);
9721
9722 if ( rc == VINF_EM_HALT
9723 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
9724 {
9725 rc = VINF_SUCCESS;
9726 }
9727 }
9728 else
9729 {
9730 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
9731 rc = VERR_EM_INTERPRETER;
9732 }
9733 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
9734 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
9735 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
9736 return rc;
9737}
9738
9739
9740/**
9741 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
9742 */
9743HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9744{
9745 /*
9746 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
9747 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
9748 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
9749 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
9750 */
9751 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9752 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9753 HMVMX_RETURN_UNEXPECTED_EXIT();
9754}
9755
9756
9757/**
9758 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
9759 */
9760HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9761{
9762 /*
9763 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
9764 * root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
9765 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
9766 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
9767 */
9768 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9769 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9770 HMVMX_RETURN_UNEXPECTED_EXIT();
9771}
9772
9773
9774/**
9775 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9776 */
9777HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9778{
9779 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9780 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9781 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9782 HMVMX_RETURN_UNEXPECTED_EXIT();
9783}
9784
9785
9786/**
9787 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9788 */
9789HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9790{
9791 /*
9792 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9793 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9794 * See Intel spec. 25.3 "Other Causes of VM-exits".
9795 */
9796 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9797 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9798 HMVMX_RETURN_UNEXPECTED_EXIT();
9799}
9800
9801
9802/**
9803 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9804 * VM-exit.
9805 */
9806HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9807{
9808 /*
9809 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9810 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9811 *
9812 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9813 * See Intel spec. "23.8 Restrictions on VMX operation".
9814 */
9815 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9816 return VINF_SUCCESS;
9817}
9818
9819
9820/**
9821 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9822 * VM-exit.
9823 */
9824HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9825{
9826 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9827 return VINF_EM_RESET;
9828}
9829
9830
9831/**
9832 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9833 */
9834HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9835{
9836 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9837 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9838 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9839 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9840 AssertRCReturn(rc, rc);
9841
9842 pMixedCtx->rip++;
9843 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9844 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9845 rc = VINF_SUCCESS;
9846 else
9847 rc = VINF_EM_HALT;
9848
9849 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9850 return rc;
9851}
9852
9853
9854/**
9855 * VM-exit handler for instructions that result in a #UD exception delivered to
9856 * the guest.
9857 */
9858HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9859{
9860 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9861 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9862 return VINF_SUCCESS;
9863}
9864
9865
9866/**
9867 * VM-exit handler for expiry of the VMX preemption timer.
9868 */
9869HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9870{
9871 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9872
9873 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9874 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9875
9876 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9877 PVM pVM = pVCpu->CTX_SUFF(pVM);
9878 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9879 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9880 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9881}
9882
9883
9884/**
9885 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9886 */
9887HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9888{
9889 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9890
9891 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9892 /** @todo check if XSETBV is supported by the recompiler. */
9893 return VERR_EM_INTERPRETER;
9894}
9895
9896
9897/**
9898 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9899 */
9900HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9901{
9902 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9903
9904 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9905 /** @todo implement EMInterpretInvpcid() */
9906 return VERR_EM_INTERPRETER;
9907}
9908
9909
9910/**
9911 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9912 * Error VM-exit.
9913 */
9914HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9915{
9916 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9917 AssertRCReturn(rc, rc);
9918
9919 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9920 NOREF(uInvalidReason);
9921
9922#ifdef VBOX_STRICT
9923 uint32_t uIntrState;
9924 HMVMXHCUINTREG uHCReg;
9925 uint64_t u64Val;
9926 uint32_t u32Val;
9927
9928 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9929 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9930 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
9931 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9932 AssertRCReturn(rc, rc);
9933
9934 Log4(("uInvalidReason %u\n", uInvalidReason));
9935 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
9936 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9937 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9938 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9939
9940 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9941 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9942 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9943 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9944 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9945 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9946 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9947 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9948 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9949 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9950 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9951 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9952#else
9953 NOREF(pVmxTransient);
9954#endif
9955
9956 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9957 return VERR_VMX_INVALID_GUEST_STATE;
9958}
9959
9960
9961/**
9962 * VM-exit handler for VM-entry failure due to an MSR-load
9963 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9964 */
9965HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9966{
9967 NOREF(pVmxTransient);
9968 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
9969 HMVMX_RETURN_UNEXPECTED_EXIT();
9970}
9971
9972
9973/**
9974 * VM-exit handler for VM-entry failure due to a machine-check event
9975 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9976 */
9977HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9978{
9979 NOREF(pVmxTransient);
9980 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
9981 HMVMX_RETURN_UNEXPECTED_EXIT();
9982}
9983
9984
9985/**
9986 * VM-exit handler for all undefined reasons. Should never ever happen.. in
9987 * theory.
9988 */
9989HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9990{
9991 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9992 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
9993 return VERR_VMX_UNDEFINED_EXIT_CODE;
9994}
9995
9996
9997/**
9998 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9999 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
10000 * Conditional VM-exit.
10001 */
10002HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10003{
10004 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10005
10006 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
10007 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
10008 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
10009 return VERR_EM_INTERPRETER;
10010 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10011 HMVMX_RETURN_UNEXPECTED_EXIT();
10012}
10013
10014
10015/**
10016 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
10017 */
10018HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10019{
10020 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10021
10022 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
10023 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
10024 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
10025 return VERR_EM_INTERPRETER;
10026 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10027 HMVMX_RETURN_UNEXPECTED_EXIT();
10028}
10029
10030
10031/**
10032 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
10033 */
10034HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10035{
10036 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10037
10038 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
10039 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10040 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10041 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10042 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10043 {
10044 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10045 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10046 }
10047 AssertRCReturn(rc, rc);
10048 Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
10049
10050#ifdef VBOX_STRICT
10051 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
10052 {
10053 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10054 {
10055 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10056 HMVMX_RETURN_UNEXPECTED_EXIT();
10057 }
10058# if HC_ARCH_BITS == 64
10059 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
10060 && hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10061 {
10062 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10063 HMVMX_RETURN_UNEXPECTED_EXIT();
10064 }
10065# endif
10066 }
10067#endif
10068
10069 PVM pVM = pVCpu->CTX_SUFF(pVM);
10070 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10071 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
10072 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
10073 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
10074
10075 if (RT_LIKELY(rc == VINF_SUCCESS))
10076 {
10077 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10078 Assert(pVmxTransient->cbInstr == 2);
10079 }
10080 return rc;
10081}
10082
10083
10084/**
10085 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10086 */
10087HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10088{
10089 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10090 PVM pVM = pVCpu->CTX_SUFF(pVM);
10091 int rc = VINF_SUCCESS;
10092
10093 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10094 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10095 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10096 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10097 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10098 {
10099 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10100 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10101 }
10102 AssertRCReturn(rc, rc);
10103 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10104
10105 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10106 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10107 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10108
10109 if (RT_LIKELY(rc == VINF_SUCCESS))
10110 {
10111 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10112
10113 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
10114 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10115 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10116 {
10117 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10118 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10119 EMInterpretWrmsr() changes it. */
10120 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10121 }
10122 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10123 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10124
10125 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10126 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10127 {
10128 switch (pMixedCtx->ecx)
10129 {
10130 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10131 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10132 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10133 case MSR_K8_FS_BASE: /* no break */
10134 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10135 default:
10136 {
10137 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10138 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10139#if HC_ARCH_BITS == 64
10140 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10141 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10142#endif
10143 break;
10144 }
10145 }
10146 }
10147#ifdef VBOX_STRICT
10148 else
10149 {
10150 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10151 switch (pMixedCtx->ecx)
10152 {
10153 case MSR_IA32_SYSENTER_CS:
10154 case MSR_IA32_SYSENTER_EIP:
10155 case MSR_IA32_SYSENTER_ESP:
10156 case MSR_K8_FS_BASE:
10157 case MSR_K8_GS_BASE:
10158 {
10159 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10160 HMVMX_RETURN_UNEXPECTED_EXIT();
10161 }
10162
10163 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10164 default:
10165 {
10166 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10167 {
10168 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10169 pMixedCtx->ecx));
10170 HMVMX_RETURN_UNEXPECTED_EXIT();
10171 }
10172
10173#if HC_ARCH_BITS == 64
10174 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10175 {
10176 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10177 HMVMX_RETURN_UNEXPECTED_EXIT();
10178 }
10179#endif
10180 break;
10181 }
10182 }
10183 }
10184#endif /* VBOX_STRICT */
10185 }
10186 return rc;
10187}
10188
10189
10190/**
10191 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10192 */
10193HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10194{
10195 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10196
10197 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10198 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10199 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10200 return VERR_EM_INTERPRETER;
10201 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10202 HMVMX_RETURN_UNEXPECTED_EXIT();
10203}
10204
10205
10206/**
10207 * VM-exit handler for when the TPR value is lowered below the specified
10208 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10209 */
10210HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10211{
10212 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10213 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10214
10215 /*
10216 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10217 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10218 * resume guest execution.
10219 */
10220 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10221 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10222 return VINF_SUCCESS;
10223}
10224
10225
10226/**
10227 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10228 * VM-exit.
10229 *
10230 * @retval VINF_SUCCESS when guest execution can continue.
10231 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10232 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10233 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10234 * recompiler.
10235 */
10236HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10237{
10238 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10239 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10240 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10241 AssertRCReturn(rc, rc);
10242
10243 const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
10244 const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10245 PVM pVM = pVCpu->CTX_SUFF(pVM);
10246 switch (uAccessType)
10247 {
10248 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10249 {
10250#if 0
10251 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10252 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10253#else
10254 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10255 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10256 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10257#endif
10258 AssertRCReturn(rc, rc);
10259
10260 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10261 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10262 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10263 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
10264
10265 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10266 {
10267 case 0: /* CR0 */
10268 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10269 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10270 break;
10271 case 2: /* CR2 */
10272 /* Nothing to do here, CR2 it's not part of the VMCS. */
10273 break;
10274 case 3: /* CR3 */
10275 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10276 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10277 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10278 break;
10279 case 4: /* CR4 */
10280 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10281 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10282 break;
10283 case 8: /* CR8 */
10284 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10285 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10286 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10287 break;
10288 default:
10289 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10290 break;
10291 }
10292
10293 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10294 break;
10295 }
10296
10297 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10298 {
10299 /* EMInterpretCRxRead() requires EFER MSR, CS. */
10300 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10301 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10302 AssertRCReturn(rc, rc);
10303 Assert( !pVM->hm.s.fNestedPaging
10304 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10305 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10306
10307 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10308 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10309 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10310
10311 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10312 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10313 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10314 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10315 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10316 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10317 break;
10318 }
10319
10320 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10321 {
10322 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10323 AssertRCReturn(rc, rc);
10324 rc = EMInterpretCLTS(pVM, pVCpu);
10325 AssertRCReturn(rc, rc);
10326 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10327 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10328 Log4(("CRX CLTS write rc=%d\n", rc));
10329 break;
10330 }
10331
10332 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10333 {
10334 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10335 AssertRCReturn(rc, rc);
10336 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10337 if (RT_LIKELY(rc == VINF_SUCCESS))
10338 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10339 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10340 Log4(("CRX LMSW write rc=%d\n", rc));
10341 break;
10342 }
10343
10344 default:
10345 {
10346 AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
10347 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10348 }
10349 }
10350
10351 /* Validate possible error codes. */
10352 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
10353 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10354 if (RT_SUCCESS(rc))
10355 {
10356 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10357 AssertRCReturn(rc2, rc2);
10358 }
10359
10360 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10361 return rc;
10362}
10363
10364
10365/**
10366 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10367 * VM-exit.
10368 */
10369HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10370{
10371 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10372 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10373
10374 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10375 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10376 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10377 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10378 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10379 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10380 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10381 AssertRCReturn(rc2, rc2);
10382
10383 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10384 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10385 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10386 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10387 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10388 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10389 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
10390 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
10391
10392 /* I/O operation lookup arrays. */
10393 static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10394 static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10395
10396 VBOXSTRICTRC rcStrict;
10397 const uint32_t cbValue = s_aIOSizes[uIOWidth];
10398 const uint32_t cbInstr = pVmxTransient->cbInstr;
10399 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
10400 PVM pVM = pVCpu->CTX_SUFF(pVM);
10401 if (fIOString)
10402 {
10403#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
10404 /*
10405 * INS/OUTS - I/O String instruction.
10406 *
10407 * Use instruction-information if available, otherwise fall back on
10408 * interpreting the instruction.
10409 */
10410 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10411 AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
10412 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10413 {
10414 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10415 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10416 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10417 AssertRCReturn(rc2, rc2);
10418 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
10419 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10420 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10421 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10422 if (fIOWrite)
10423 {
10424 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10425 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10426 }
10427 else
10428 {
10429 /*
10430 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10431 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10432 * See Intel Instruction spec. for "INS".
10433 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10434 */
10435 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10436 }
10437 }
10438 else
10439 {
10440 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10441 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10442 AssertRCReturn(rc2, rc2);
10443 rcStrict = IEMExecOne(pVCpu);
10444 }
10445 /** @todo IEM needs to be setting these flags somehow. */
10446 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10447 fUpdateRipAlready = true;
10448#else
10449 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10450 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
10451 if (RT_SUCCESS(rcStrict))
10452 {
10453 if (fIOWrite)
10454 {
10455 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10456 (DISCPUMODE)pDis->uAddrMode, cbValue);
10457 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10458 }
10459 else
10460 {
10461 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10462 (DISCPUMODE)pDis->uAddrMode, cbValue);
10463 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10464 }
10465 }
10466 else
10467 {
10468 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10469 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10470 }
10471#endif
10472 }
10473 else
10474 {
10475 /*
10476 * IN/OUT - I/O instruction.
10477 */
10478 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10479 const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
10480 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10481 if (fIOWrite)
10482 {
10483 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
10484 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10485 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10486 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
10487 }
10488 else
10489 {
10490 uint32_t u32Result = 0;
10491 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
10492 if (IOM_SUCCESS(rcStrict))
10493 {
10494 /* Save result of I/O IN instr. in AL/AX/EAX. */
10495 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
10496 }
10497 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10498 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10499 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
10500 }
10501 }
10502
10503 if (IOM_SUCCESS(rcStrict))
10504 {
10505 if (!fUpdateRipAlready)
10506 {
10507 pMixedCtx->rip += cbInstr;
10508 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10509 }
10510
10511 /*
10512 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
10513 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
10514 */
10515 if (fIOString)
10516 {
10517 /** @todo Single-step for INS/OUTS with REP prefix? */
10518 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
10519 }
10520 else if (fStepping)
10521 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
10522
10523 /*
10524 * If any I/O breakpoints are armed, we need to check if one triggered
10525 * and take appropriate action.
10526 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
10527 */
10528 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10529 AssertRCReturn(rc2, rc2);
10530
10531 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
10532 * execution engines about whether hyper BPs and such are pending. */
10533 uint32_t const uDr7 = pMixedCtx->dr[7];
10534 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
10535 && X86_DR7_ANY_RW_IO(uDr7)
10536 && (pMixedCtx->cr4 & X86_CR4_DE))
10537 || DBGFBpIsHwIoArmed(pVM)))
10538 {
10539 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
10540
10541 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
10542 VMMRZCallRing3Disable(pVCpu);
10543 HM_DISABLE_PREEMPT_IF_NEEDED();
10544
10545 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /*fDr6*/);
10546
10547 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
10548 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
10549 {
10550 /* Raise #DB. */
10551 if (fIsGuestDbgActive)
10552 ASMSetDR6(pMixedCtx->dr[6]);
10553 if (pMixedCtx->dr[7] != uDr7)
10554 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10555
10556 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
10557 }
10558 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
10559 else if ( rcStrict2 != VINF_SUCCESS
10560 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
10561 rcStrict = rcStrict2;
10562
10563 HM_RESTORE_PREEMPT_IF_NEEDED();
10564 VMMRZCallRing3Enable(pVCpu);
10565 }
10566 }
10567
10568#ifdef DEBUG
10569 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10570 Assert(!fIOWrite);
10571 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10572 Assert(fIOWrite);
10573 else
10574 {
10575 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
10576 * statuses, that the VMM device and some others may return. See
10577 * IOM_SUCCESS() for guidance. */
10578 AssertMsg( RT_FAILURE(rcStrict)
10579 || rcStrict == VINF_SUCCESS
10580 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
10581 || rcStrict == VINF_EM_DBG_BREAKPOINT
10582 || rcStrict == VINF_EM_RAW_GUEST_TRAP
10583 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10584 }
10585#endif
10586
10587 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
10588 return VBOXSTRICTRC_TODO(rcStrict);
10589}
10590
10591
10592/**
10593 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
10594 * VM-exit.
10595 */
10596HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10597{
10598 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10599
10600 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
10601 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10602 AssertRCReturn(rc, rc);
10603 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
10604 {
10605 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
10606 AssertRCReturn(rc, rc);
10607 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
10608 {
10609 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
10610
10611 /* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
10612 if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
10613 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
10614 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
10615 {
10616 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
10617 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
10618
10619 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
10620 Assert(!pVCpu->hm.s.Event.fPending);
10621 pVCpu->hm.s.Event.fPending = true;
10622 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
10623 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
10624 AssertRCReturn(rc, rc);
10625 if (fErrorCodeValid)
10626 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
10627 else
10628 pVCpu->hm.s.Event.u32ErrCode = 0;
10629 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
10630 && uVector == X86_XCPT_PF)
10631 {
10632 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
10633 }
10634
10635 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
10636 }
10637 }
10638 }
10639
10640 /** @todo Emulate task switch someday, currently just going back to ring-3 for
10641 * emulation. */
10642 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10643 return VERR_EM_INTERPRETER;
10644}
10645
10646
10647/**
10648 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
10649 */
10650HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10651{
10652 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10653 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
10654 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
10655 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10656 AssertRCReturn(rc, rc);
10657 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
10658 return VINF_EM_DBG_STEPPED;
10659}
10660
10661
10662/**
10663 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10664 */
10665HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10666{
10667 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10668
10669 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10670 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10671 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10672 return VINF_SUCCESS;
10673 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10674 return rc;
10675
10676#if 0
10677 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
10678 * just sync the whole thing. */
10679 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10680#else
10681 /* Aggressive state sync. for now. */
10682 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10683 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10684 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10685#endif
10686 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10687 AssertRCReturn(rc, rc);
10688
10689 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
10690 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
10691 switch (uAccessType)
10692 {
10693 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
10694 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
10695 {
10696 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
10697 && VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
10698 {
10699 AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
10700 }
10701
10702 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
10703 GCPhys &= PAGE_BASE_GC_MASK;
10704 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
10705 PVM pVM = pVCpu->CTX_SUFF(pVM);
10706 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
10707 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
10708
10709 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
10710 (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
10711 CPUMCTX2CORE(pMixedCtx), GCPhys);
10712 rc = VBOXSTRICTRC_VAL(rc2);
10713 Log4(("ApicAccess rc=%d\n", rc));
10714 if ( rc == VINF_SUCCESS
10715 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10716 || rc == VERR_PAGE_NOT_PRESENT)
10717 {
10718 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10719 | HM_CHANGED_GUEST_RSP
10720 | HM_CHANGED_GUEST_RFLAGS
10721 | HM_CHANGED_VMX_GUEST_APIC_STATE);
10722 rc = VINF_SUCCESS;
10723 }
10724 break;
10725 }
10726
10727 default:
10728 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
10729 rc = VINF_EM_RAW_EMULATE_INSTR;
10730 break;
10731 }
10732
10733 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
10734 return rc;
10735}
10736
10737
10738/**
10739 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
10740 * VM-exit.
10741 */
10742HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10743{
10744 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10745
10746 /* We should -not- get this VM-exit if the guest's debug registers were active. */
10747 if (pVmxTransient->fWasGuestDebugStateActive)
10748 {
10749 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10750 HMVMX_RETURN_UNEXPECTED_EXIT();
10751 }
10752
10753 int rc = VERR_INTERNAL_ERROR_5;
10754 if ( !DBGFIsStepping(pVCpu)
10755 && !pVCpu->hm.s.fSingleInstruction
10756 && !pVmxTransient->fWasHyperDebugStateActive)
10757 {
10758 /* Don't intercept MOV DRx and #DB any more. */
10759 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
10760 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10761 AssertRCReturn(rc, rc);
10762
10763 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10764 {
10765#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10766 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
10767 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
10768 AssertRCReturn(rc, rc);
10769#endif
10770 }
10771
10772 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
10773 VMMRZCallRing3Disable(pVCpu);
10774 HM_DISABLE_PREEMPT_IF_NEEDED();
10775
10776 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
10777 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
10778 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
10779
10780 HM_RESTORE_PREEMPT_IF_NEEDED();
10781 VMMRZCallRing3Enable(pVCpu);
10782
10783#ifdef VBOX_WITH_STATISTICS
10784 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10785 AssertRCReturn(rc, rc);
10786 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10787 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10788 else
10789 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10790#endif
10791 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
10792 return VINF_SUCCESS;
10793 }
10794
10795 /*
10796 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
10797 * Update the segment registers and DR7 from the CPU.
10798 */
10799 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10800 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10801 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10802 AssertRCReturn(rc, rc);
10803 Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10804
10805 PVM pVM = pVCpu->CTX_SUFF(pVM);
10806 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10807 {
10808 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10809 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
10810 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
10811 if (RT_SUCCESS(rc))
10812 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10813 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10814 }
10815 else
10816 {
10817 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10818 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
10819 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
10820 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10821 }
10822
10823 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10824 if (RT_SUCCESS(rc))
10825 {
10826 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10827 AssertRCReturn(rc2, rc2);
10828 }
10829 return rc;
10830}
10831
10832
10833/**
10834 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10835 * Conditional VM-exit.
10836 */
10837HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10838{
10839 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10840 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10841
10842 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10843 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10844 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10845 return VINF_SUCCESS;
10846 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10847 return rc;
10848
10849 RTGCPHYS GCPhys = 0;
10850 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10851
10852#if 0
10853 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10854#else
10855 /* Aggressive state sync. for now. */
10856 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10857 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10858 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10859#endif
10860 AssertRCReturn(rc, rc);
10861
10862 /*
10863 * If we succeed, resume guest execution.
10864 * If we fail in interpreting the instruction because we couldn't get the guest physical address
10865 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10866 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10867 * weird case. See @bugref{6043}.
10868 */
10869 PVM pVM = pVCpu->CTX_SUFF(pVM);
10870 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10871 rc = VBOXSTRICTRC_VAL(rc2);
10872 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10873 if ( rc == VINF_SUCCESS
10874 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10875 || rc == VERR_PAGE_NOT_PRESENT)
10876 {
10877 /* Successfully handled MMIO operation. */
10878 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10879 | HM_CHANGED_GUEST_RSP
10880 | HM_CHANGED_GUEST_RFLAGS
10881 | HM_CHANGED_VMX_GUEST_APIC_STATE);
10882 rc = VINF_SUCCESS;
10883 }
10884 return rc;
10885}
10886
10887
10888/**
10889 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10890 * VM-exit.
10891 */
10892HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10893{
10894 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10895 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10896
10897 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10898 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10899 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10900 return VINF_SUCCESS;
10901 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10902 return rc;
10903
10904 RTGCPHYS GCPhys = 0;
10905 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10906 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10907#if 0
10908 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10909#else
10910 /* Aggressive state sync. for now. */
10911 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10912 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10913 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10914#endif
10915 AssertRCReturn(rc, rc);
10916
10917 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10918 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10919
10920 RTGCUINT uErrorCode = 0;
10921 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10922 uErrorCode |= X86_TRAP_PF_ID;
10923 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10924 uErrorCode |= X86_TRAP_PF_RW;
10925 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10926 uErrorCode |= X86_TRAP_PF_P;
10927
10928 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10929
10930 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10931 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10932
10933 /* Handle the pagefault trap for the nested shadow table. */
10934 PVM pVM = pVCpu->CTX_SUFF(pVM);
10935 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10936 TRPMResetTrap(pVCpu);
10937
10938 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10939 if ( rc == VINF_SUCCESS
10940 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10941 || rc == VERR_PAGE_NOT_PRESENT)
10942 {
10943 /* Successfully synced our nested page tables. */
10944 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10945 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10946 | HM_CHANGED_GUEST_RSP
10947 | HM_CHANGED_GUEST_RFLAGS);
10948 return VINF_SUCCESS;
10949 }
10950
10951 Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
10952 return rc;
10953}
10954
10955/** @} */
10956
10957/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10958/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10959/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10960
10961/** @name VM-exit exception handlers.
10962 * @{
10963 */
10964
10965/**
10966 * VM-exit exception handler for #MF (Math Fault: floating point exception).
10967 */
10968static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10969{
10970 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10971 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10972
10973 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10974 AssertRCReturn(rc, rc);
10975
10976 if (!(pMixedCtx->cr0 & X86_CR0_NE))
10977 {
10978 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
10979 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
10980
10981 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
10982 * provides VM-exit instruction length. If this causes problem later,
10983 * disassemble the instruction like it's done on AMD-V. */
10984 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10985 AssertRCReturn(rc2, rc2);
10986 return rc;
10987 }
10988
10989 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10990 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10991 return rc;
10992}
10993
10994
10995/**
10996 * VM-exit exception handler for #BP (Breakpoint exception).
10997 */
10998static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10999{
11000 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11001 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
11002
11003 /** @todo Try optimize this by not saving the entire guest state unless
11004 * really needed. */
11005 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11006 AssertRCReturn(rc, rc);
11007
11008 PVM pVM = pVCpu->CTX_SUFF(pVM);
11009 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11010 if (rc == VINF_EM_RAW_GUEST_TRAP)
11011 {
11012 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11013 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11014 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11015 AssertRCReturn(rc, rc);
11016
11017 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11018 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11019 }
11020
11021 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
11022 return rc;
11023}
11024
11025
11026/**
11027 * VM-exit exception handler for #DB (Debug exception).
11028 */
11029static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11030{
11031 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11032 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
11033 Log6(("XcptDB\n"));
11034
11035 /*
11036 * Get the DR6-like values from the exit qualification and pass it to DBGF
11037 * for processing.
11038 */
11039 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11040 AssertRCReturn(rc, rc);
11041
11042 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
11043 uint64_t uDR6 = X86_DR6_INIT_VAL;
11044 uDR6 |= ( pVmxTransient->uExitQualification
11045 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
11046
11047 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
11048 if (rc == VINF_EM_RAW_GUEST_TRAP)
11049 {
11050 /*
11051 * The exception was for the guest. Update DR6, DR7.GD and
11052 * IA32_DEBUGCTL.LBR before forwarding it.
11053 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
11054 */
11055 VMMRZCallRing3Disable(pVCpu);
11056 HM_DISABLE_PREEMPT_IF_NEEDED();
11057
11058 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
11059 pMixedCtx->dr[6] |= uDR6;
11060 if (CPUMIsGuestDebugStateActive(pVCpu))
11061 ASMSetDR6(pMixedCtx->dr[6]);
11062
11063 HM_RESTORE_PREEMPT_IF_NEEDED();
11064 VMMRZCallRing3Enable(pVCpu);
11065
11066 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11067 AssertRCReturn(rc, rc);
11068
11069 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
11070 pMixedCtx->dr[7] &= ~X86_DR7_GD;
11071
11072 /* Paranoia. */
11073 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
11074 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
11075
11076 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
11077 AssertRCReturn(rc, rc);
11078
11079 /*
11080 * Raise #DB in the guest.
11081 *
11082 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
11083 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
11084 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
11085 *
11086 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11087 */
11088 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11089 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11090 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11091 AssertRCReturn(rc, rc);
11092 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11093 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11094 return VINF_SUCCESS;
11095 }
11096
11097 /*
11098 * Not a guest trap, must be a hypervisor related debug event then.
11099 * Update DR6 in case someone is interested in it.
11100 */
11101 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11102 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11103 CPUMSetHyperDR6(pVCpu, uDR6);
11104
11105 return rc;
11106}
11107
11108
11109/**
11110 * VM-exit exception handler for #NM (Device-not-available exception: floating
11111 * point exception).
11112 */
11113static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11114{
11115 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11116
11117 /* We require CR0 and EFER. EFER is always up-to-date. */
11118 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11119 AssertRCReturn(rc, rc);
11120
11121 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11122 VMMRZCallRing3Disable(pVCpu);
11123 HM_DISABLE_PREEMPT_IF_NEEDED();
11124
11125 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11126 if (pVmxTransient->fWasGuestFPUStateActive)
11127 {
11128 rc = VINF_EM_RAW_GUEST_TRAP;
11129 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11130 }
11131 else
11132 {
11133#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11134 Assert(!pVmxTransient->fWasGuestFPUStateActive);
11135#endif
11136 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11137 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11138 }
11139
11140 HM_RESTORE_PREEMPT_IF_NEEDED();
11141 VMMRZCallRing3Enable(pVCpu);
11142
11143 if (rc == VINF_SUCCESS)
11144 {
11145 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11146 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11147 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11148 pVCpu->hm.s.fUseGuestFpu = true;
11149 }
11150 else
11151 {
11152 /* Forward #NM to the guest. */
11153 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11154 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11155 AssertRCReturn(rc, rc);
11156 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11157 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
11158 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11159 }
11160
11161 return VINF_SUCCESS;
11162}
11163
11164
11165/**
11166 * VM-exit exception handler for #GP (General-protection exception).
11167 *
11168 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11169 */
11170static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11171{
11172 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11173 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11174
11175 int rc = VERR_INTERNAL_ERROR_5;
11176 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11177 {
11178#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11179 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11180 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11181 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11182 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11183 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11184 AssertRCReturn(rc, rc);
11185 Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntErrorCode,
11186 pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
11187 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11188 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11189 return rc;
11190#else
11191 /* We don't intercept #GP. */
11192 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11193 NOREF(pVmxTransient);
11194 return VERR_VMX_UNEXPECTED_EXCEPTION;
11195#endif
11196 }
11197
11198 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11199 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11200
11201 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11202 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11203 AssertRCReturn(rc, rc);
11204
11205 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11206 uint32_t cbOp = 0;
11207 PVM pVM = pVCpu->CTX_SUFF(pVM);
11208 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11209 if (RT_SUCCESS(rc))
11210 {
11211 rc = VINF_SUCCESS;
11212 Assert(cbOp == pDis->cbInstr);
11213 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11214 switch (pDis->pCurInstr->uOpcode)
11215 {
11216 case OP_CLI:
11217 {
11218 pMixedCtx->eflags.Bits.u1IF = 0;
11219 pMixedCtx->rip += pDis->cbInstr;
11220 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11221 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11222 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11223 break;
11224 }
11225
11226 case OP_STI:
11227 {
11228 pMixedCtx->eflags.Bits.u1IF = 1;
11229 pMixedCtx->rip += pDis->cbInstr;
11230 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11231 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11232 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11233 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11234 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11235 break;
11236 }
11237
11238 case OP_HLT:
11239 {
11240 rc = VINF_EM_HALT;
11241 pMixedCtx->rip += pDis->cbInstr;
11242 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11243 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11244 break;
11245 }
11246
11247 case OP_POPF:
11248 {
11249 Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11250 uint32_t cbParm;
11251 uint32_t uMask;
11252 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11253 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11254 {
11255 cbParm = 4;
11256 uMask = 0xffffffff;
11257 }
11258 else
11259 {
11260 cbParm = 2;
11261 uMask = 0xffff;
11262 }
11263
11264 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
11265 RTGCPTR GCPtrStack = 0;
11266 X86EFLAGS Eflags;
11267 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11268 &GCPtrStack);
11269 if (RT_SUCCESS(rc))
11270 {
11271 Assert(sizeof(Eflags.u32) >= cbParm);
11272 Eflags.u32 = 0;
11273 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11274 }
11275 if (RT_FAILURE(rc))
11276 {
11277 rc = VERR_EM_INTERPRETER;
11278 break;
11279 }
11280 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11281 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11282 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11283 pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
11284 pMixedCtx->esp += cbParm;
11285 pMixedCtx->esp &= uMask;
11286 pMixedCtx->rip += pDis->cbInstr;
11287 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11288 | HM_CHANGED_GUEST_RSP
11289 | HM_CHANGED_GUEST_RFLAGS);
11290 /* Generate a pending-debug exception when stepping over POPF regardless of how POPF modifies EFLAGS.TF. */
11291 if (fStepping)
11292 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11293
11294 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11295 break;
11296 }
11297
11298 case OP_PUSHF:
11299 {
11300 uint32_t cbParm;
11301 uint32_t uMask;
11302 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11303 {
11304 cbParm = 4;
11305 uMask = 0xffffffff;
11306 }
11307 else
11308 {
11309 cbParm = 2;
11310 uMask = 0xffff;
11311 }
11312
11313 /* Get the stack pointer & push the contents of eflags onto the stack. */
11314 RTGCPTR GCPtrStack = 0;
11315 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11316 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11317 if (RT_FAILURE(rc))
11318 {
11319 rc = VERR_EM_INTERPRETER;
11320 break;
11321 }
11322 X86EFLAGS Eflags = pMixedCtx->eflags;
11323 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11324 Eflags.Bits.u1RF = 0;
11325 Eflags.Bits.u1VM = 0;
11326
11327 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11328 if (RT_FAILURE(rc))
11329 {
11330 rc = VERR_EM_INTERPRETER;
11331 break;
11332 }
11333 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11334 pMixedCtx->esp -= cbParm;
11335 pMixedCtx->esp &= uMask;
11336 pMixedCtx->rip += pDis->cbInstr;
11337 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP);
11338 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11339 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11340 break;
11341 }
11342
11343 case OP_IRET:
11344 {
11345 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11346 * instruction reference. */
11347 RTGCPTR GCPtrStack = 0;
11348 uint32_t uMask = 0xffff;
11349 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11350 uint16_t aIretFrame[3];
11351 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
11352 {
11353 rc = VERR_EM_INTERPRETER;
11354 break;
11355 }
11356 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11357 &GCPtrStack);
11358 if (RT_SUCCESS(rc))
11359 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11360 if (RT_FAILURE(rc))
11361 {
11362 rc = VERR_EM_INTERPRETER;
11363 break;
11364 }
11365 pMixedCtx->eip = 0;
11366 pMixedCtx->ip = aIretFrame[0];
11367 pMixedCtx->cs.Sel = aIretFrame[1];
11368 pMixedCtx->cs.ValidSel = aIretFrame[1];
11369 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11370 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11371 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11372 pMixedCtx->sp += sizeof(aIretFrame);
11373 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11374 | HM_CHANGED_GUEST_SEGMENT_REGS
11375 | HM_CHANGED_GUEST_RSP
11376 | HM_CHANGED_GUEST_RFLAGS);
11377 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
11378 if (fStepping)
11379 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11380 Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11381 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11382 break;
11383 }
11384
11385 case OP_INT:
11386 {
11387 uint16_t uVector = pDis->Param1.uValue & 0xff;
11388 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
11389 /* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11390 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11391 break;
11392 }
11393
11394 case OP_INTO:
11395 {
11396 if (pMixedCtx->eflags.Bits.u1OF)
11397 {
11398 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
11399 /* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11400 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11401 }
11402 break;
11403 }
11404
11405 default:
11406 {
11407 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
11408 EMCODETYPE_SUPERVISOR);
11409 rc = VBOXSTRICTRC_VAL(rc2);
11410 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
11411 /** @todo We have to set pending-debug exceptions here when the guest is
11412 * single-stepping depending on the instruction that was interpreted. */
11413 Log4(("#GP rc=%Rrc\n", rc));
11414 break;
11415 }
11416 }
11417 }
11418 else
11419 rc = VERR_EM_INTERPRETER;
11420
11421 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
11422 ("#GP Unexpected rc=%Rrc\n", rc));
11423 return rc;
11424}
11425
11426
11427#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11428/**
11429 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
11430 * the exception reported in the VMX transient structure back into the VM.
11431 *
11432 * @remarks Requires uExitIntInfo in the VMX transient structure to be
11433 * up-to-date.
11434 */
11435static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11436{
11437 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11438
11439 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
11440 hmR0VmxCheckExitDueToEventDelivery(). */
11441 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11442 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11443 AssertRCReturn(rc, rc);
11444 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
11445
11446 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11447 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11448 return VINF_SUCCESS;
11449}
11450#endif
11451
11452
11453/**
11454 * VM-exit exception handler for #PF (Page-fault exception).
11455 */
11456static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11457{
11458 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11459 PVM pVM = pVCpu->CTX_SUFF(pVM);
11460 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11461 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11462 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11463 AssertRCReturn(rc, rc);
11464
11465#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
11466 if (pVM->hm.s.fNestedPaging)
11467 {
11468 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11469 if (RT_LIKELY(!pVmxTransient->fVectoringPF))
11470 {
11471 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11472 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11473 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11474 }
11475 else
11476 {
11477 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11478 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11479 Log4(("Pending #DF due to vectoring #PF. NP\n"));
11480 }
11481 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11482 return rc;
11483 }
11484#else
11485 Assert(!pVM->hm.s.fNestedPaging);
11486 NOREF(pVM);
11487#endif
11488
11489 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11490 AssertRCReturn(rc, rc);
11491
11492 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
11493 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
11494
11495 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
11496 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
11497 (RTGCPTR)pVmxTransient->uExitQualification);
11498
11499 Log4(("#PF: rc=%Rrc\n", rc));
11500 if (rc == VINF_SUCCESS)
11501 {
11502 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
11503 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
11504 * memory? We don't update the whole state here... */
11505 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11506 | HM_CHANGED_GUEST_RSP
11507 | HM_CHANGED_GUEST_RFLAGS
11508 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11509 TRPMResetTrap(pVCpu);
11510 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
11511 return rc;
11512 }
11513 else if (rc == VINF_EM_RAW_GUEST_TRAP)
11514 {
11515 if (!pVmxTransient->fVectoringPF)
11516 {
11517 /* It's a guest page fault and needs to be reflected to the guest. */
11518 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
11519 TRPMResetTrap(pVCpu);
11520 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
11521 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11522 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11523 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
11524 }
11525 else
11526 {
11527 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11528 TRPMResetTrap(pVCpu);
11529 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
11530 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11531 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
11532 }
11533
11534 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11535 return VINF_SUCCESS;
11536 }
11537
11538 TRPMResetTrap(pVCpu);
11539 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
11540 return rc;
11541}
11542
11543/** @} */
11544
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette